def reduce(self, remote=False): """ Reduction process using job submission. @param remote: If True, the job will be submitted to a compute node """ self._process_data(self.data) try: self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data)) instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce') proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce') log_dir = os.path.join(proposal_shared_dir, "reduction_log") if not os.path.exists(log_dir): os.makedirs(log_dir) # Allow for an alternate output directory, if defined if len(self.conf.dev_output_dir.strip()) > 0: proposal_shared_dir = self.conf.dev_output_dir logging.info("Using output directory: %s" % proposal_shared_dir) # Look for run summary script summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument) if os.path.exists(summary_script) == True: summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal)) cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output logging.debug("Run summary subprocess started: " + cmd) subprocess.call(cmd, shell=True) logging.debug("Run summary subprocess completed, see " + summary_output) # Look for auto-reduction script reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument) if os.path.exists(reduce_script_path) == False: self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data)) return # Run the reduction out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log") out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err") if remote: job_handling.remote_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) else: job_handling.local_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) # Determine error condition success, status_data = job_handling.determine_success_local(self.conf, out_err) self.data.update(status_data) if success: if os.path.isfile(out_err): os.remove(out_err) self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data)) else: self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data)) except: logging.error("reduce: %s" % sys.exc_value) self.data["error"] = "Reduction: %s " % sys.exc_value self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
def reduce(self, remote=False): """ Reduction process using job submission. @param remote: If True, the job will be submitted to a compute node """ self._process_data(self.data) try: self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data)) instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce') proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce') log_dir = os.path.join(proposal_shared_dir, "reduction_log") if not os.path.exists(log_dir): os.makedirs(log_dir) # Allow for an alternate output directory, if defined if len(self.conf.dev_output_dir.strip()) > 0: proposal_shared_dir = self.conf.dev_output_dir logging.info("Using output directory: %s" % proposal_shared_dir) # Look for run summary script summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument) if os.path.exists(summary_script) == True: summary_output = os.path.join( proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal)) cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output logging.debug("Run summary subprocess started: " + cmd) subprocess.call(cmd, shell=True) logging.debug("Run summary subprocess completed, see " + summary_output) # Look for auto-reduction script reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument) if os.path.exists(reduce_script_path) == False: self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data)) return # Run the reduction out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log") out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err") if remote: job_handling.remote_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) else: job_handling.local_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) # Determine error condition success, status_data = job_handling.determine_success_local( self.conf, out_err) self.data.update(status_data) if success: if os.path.isfile(out_err): os.remove(out_err) self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data)) else: self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data)) except: logging.error("reduce: %s" % sys.exc_value) self.data["error"] = "Reduction: %s " % sys.exc_value self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data))
def reduce(self, remote=False): """ Reduction process using job submission. @param remote: If True, the job will be submitted to a compute node """ self._process_data(self.data) try: self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data)) instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce') proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce') log_dir = os.path.join(proposal_shared_dir, "reduction_log") if not os.path.exists(log_dir): os.makedirs(log_dir) # Allow for an alternate output directory, if defined if len(self.conf.dev_output_dir.strip()) > 0: proposal_shared_dir = self.conf.dev_output_dir logging.info("Using output directory: %s" % proposal_shared_dir) # Look for run summary script summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument) if os.path.exists(summary_script) == True: summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal)) cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output logging.debug("Run summary subprocess started: " + cmd) subprocess.call(cmd, shell=True) logging.debug("Run summary subprocess completed, see " + summary_output) # Look for auto-reduction script reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument) if os.path.exists(reduce_script_path) == False: self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data)) return monitor_user = {'username': self.conf.amq_user, 'password': self.conf.amq_pwd} # Run the reduction out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log") out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err") if remote: job_handling.remote_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) else: job_handling.local_submission(self.conf, reduce_script_path, self.data_file, proposal_shared_dir, out_log, out_err) # If the reduction succeeded, upload the images we might find in the reduction directory success = not os.path.isfile(out_err) or os.stat(out_err).st_size == 0 if not success: # Go through each line and report the error message. # If we can't fine the actual error, report the last line last_line = None error_line = None fp = file(out_err, "r") for l in fp.readlines(): if len(l.replace('-', '').strip()) > 0: last_line = l.strip() result = re.search('Error: ([\w ]+)$', l) if result is not None: error_line = result.group(1) if error_line is None: error_line = last_line for item in self.exceptions: if re.search(item, error_line): success = True self.data["information"] = error_line logging.error("Reduction error ignored: %s" % error_line) if not success: self.data["error"] = "REDUCTION: %s" % error_line self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data)) if success: if os.path.isfile(out_err): os.remove(out_err) self.send('/queue/' + self.conf.reduction_complete , json.dumps(self.data)) # Send image to the web monitor if len(self.conf.web_monitor_url.strip()) > 0: url_template = string.Template(self.conf.web_monitor_url) url = url_template.substitute(instrument=self.instrument, run_number=self.run_number) pattern = self.instrument + "_" + self.run_number + "*" for dirpath, dirnames, filenames in os.walk(proposal_shared_dir): listing = glob.glob(os.path.join(dirpath, pattern)) for filepath in listing: f, e = os.path.splitext(filepath) if e.startswith(os.extsep): e = e[len(os.extsep):] if e == "png" or e == "jpg" or filepath.endswith("plot_data.dat"): files = {'file': open(filepath, 'rb')} # Post the file if it's small enough if len(files) != 0 and os.path.getsize(filepath) < self.conf.max_image_size: request = requests.post(url, data=monitor_user, files=files, verify=False) logging.info("Submitted %s [status: %s]" % (filepath, request.status_code)) except: logging.error("reduce: %s" % sys.exc_value) self.data["error"] = "Reduction: %s " % sys.exc_value self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))