def reduce(self, remote=False):
"""
Reduction process using job submission.
@param remote: If True, the job will be submitted to a compute node
"""
self._process_data(self.data)
try:
self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data))
instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce')
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
log_dir = os.path.join(proposal_shared_dir, "reduction_log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Allow for an alternate output directory, if defined
if len(self.conf.dev_output_dir.strip()) > 0:
proposal_shared_dir = self.conf.dev_output_dir
logging.info("Using output directory: %s" % proposal_shared_dir)
# Look for run summary script
summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument)
if os.path.exists(summary_script) == True:
summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal))
cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output
logging.debug("Run summary subprocess started: " + cmd)
subprocess.call(cmd, shell=True)
logging.debug("Run summary subprocess completed, see " + summary_output)
# Look for auto-reduction script
reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument)
if os.path.exists(reduce_script_path) == False:
self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data))
return
# Run the reduction
out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log")
out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err")
if remote:
job_handling.remote_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
else:
job_handling.local_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
# Determine error condition
success, status_data = job_handling.determine_success_local(self.conf, out_err)
self.data.update(status_data)
if success:
if os.path.isfile(out_err):
os.remove(out_err)
self.send('/queue/' + self.conf.reduction_complete, json.dumps(self.data))
else:
self.send('/queue/' + self.conf.reduction_error, json.dumps(self.data))
except:
logging.error("reduce: %s" % sys.exc_value)
self.data["error"] = "Reduction: %s " % sys.exc_value
self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
def reduce(self, remote=False):
"""
Reduction process using job submission.
@param remote: If True, the job will be submitted to a compute node
"""
self._process_data(self.data)
try:
self.send('/queue/' + self.conf.reduction_started,
json.dumps(self.data))
instrument_shared_dir = os.path.join('/', self.facility,
self.instrument, 'shared',
'autoreduce')
proposal_shared_dir = os.path.join('/', self.facility,
self.instrument, self.proposal,
'shared', 'autoreduce')
log_dir = os.path.join(proposal_shared_dir, "reduction_log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Allow for an alternate output directory, if defined
if len(self.conf.dev_output_dir.strip()) > 0:
proposal_shared_dir = self.conf.dev_output_dir
logging.info("Using output directory: %s" % proposal_shared_dir)
# Look for run summary script
summary_script = os.path.join(instrument_shared_dir,
"sumRun_%s.py" % self.instrument)
if os.path.exists(summary_script) == True:
summary_output = os.path.join(
proposal_shared_dir,
"%s_%s_runsummary.csv" % (self.instrument, self.proposal))
cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output
logging.debug("Run summary subprocess started: " + cmd)
subprocess.call(cmd, shell=True)
logging.debug("Run summary subprocess completed, see " +
summary_output)
# Look for auto-reduction script
reduce_script_path = os.path.join(instrument_shared_dir,
"reduce_%s.py" % self.instrument)
if os.path.exists(reduce_script_path) == False:
self.send('/queue/' + self.conf.reduction_disabled,
json.dumps(self.data))
return
# Run the reduction
out_log = os.path.join(log_dir,
os.path.basename(self.data_file) + ".log")
out_err = os.path.join(log_dir,
os.path.basename(self.data_file) + ".err")
if remote:
job_handling.remote_submission(self.conf, reduce_script_path,
self.data_file,
proposal_shared_dir, out_log,
out_err)
else:
job_handling.local_submission(self.conf, reduce_script_path,
self.data_file,
proposal_shared_dir, out_log,
out_err)
# Determine error condition
success, status_data = job_handling.determine_success_local(
self.conf, out_err)
self.data.update(status_data)
if success:
if os.path.isfile(out_err):
os.remove(out_err)
self.send('/queue/' + self.conf.reduction_complete,
json.dumps(self.data))
else:
self.send('/queue/' + self.conf.reduction_error,
json.dumps(self.data))
except:
logging.error("reduce: %s" % sys.exc_value)
self.data["error"] = "Reduction: %s " % sys.exc_value
self.send('/queue/' + self.conf.reduction_error,
json.dumps(self.data))
def reduce(self, remote=False):
"""
Reduction process using job submission.
@param remote: If True, the job will be submitted to a compute node
"""
self._process_data(self.data)
try:
self.send('/queue/' + self.conf.reduction_started, json.dumps(self.data))
instrument_shared_dir = os.path.join('/', self.facility, self.instrument, 'shared', 'autoreduce')
proposal_shared_dir = os.path.join('/', self.facility, self.instrument, self.proposal, 'shared', 'autoreduce')
log_dir = os.path.join(proposal_shared_dir, "reduction_log")
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Allow for an alternate output directory, if defined
if len(self.conf.dev_output_dir.strip()) > 0:
proposal_shared_dir = self.conf.dev_output_dir
logging.info("Using output directory: %s" % proposal_shared_dir)
# Look for run summary script
summary_script = os.path.join(instrument_shared_dir, "sumRun_%s.py" % self.instrument)
if os.path.exists(summary_script) == True:
summary_output = os.path.join(proposal_shared_dir, "%s_%s_runsummary.csv" % (self.instrument, self.proposal))
cmd = "python " + summary_script + " " + self.instrument + " " + self.data_file + " " + summary_output
logging.debug("Run summary subprocess started: " + cmd)
subprocess.call(cmd, shell=True)
logging.debug("Run summary subprocess completed, see " + summary_output)
# Look for auto-reduction script
reduce_script_path = os.path.join(instrument_shared_dir, "reduce_%s.py" % self.instrument)
if os.path.exists(reduce_script_path) == False:
self.send('/queue/' + self.conf.reduction_disabled, json.dumps(self.data))
return
monitor_user = {'username': self.conf.amq_user, 'password': self.conf.amq_pwd}
# Run the reduction
out_log = os.path.join(log_dir, os.path.basename(self.data_file) + ".log")
out_err = os.path.join(log_dir, os.path.basename(self.data_file) + ".err")
if remote:
job_handling.remote_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
else:
job_handling.local_submission(self.conf, reduce_script_path,
self.data_file, proposal_shared_dir,
out_log, out_err)
# If the reduction succeeded, upload the images we might find in the reduction directory
success = not os.path.isfile(out_err) or os.stat(out_err).st_size == 0
if not success:
# Go through each line and report the error message.
# If we can't fine the actual error, report the last line
last_line = None
error_line = None
fp = file(out_err, "r")
for l in fp.readlines():
if len(l.replace('-', '').strip()) > 0:
last_line = l.strip()
result = re.search('Error: ([\w ]+)$', l)
if result is not None:
error_line = result.group(1)
if error_line is None:
error_line = last_line
for item in self.exceptions:
if re.search(item, error_line):
success = True
self.data["information"] = error_line
logging.error("Reduction error ignored: %s" % error_line)
if not success:
self.data["error"] = "REDUCTION: %s" % error_line
self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
if success:
if os.path.isfile(out_err):
os.remove(out_err)
self.send('/queue/' + self.conf.reduction_complete , json.dumps(self.data))
# Send image to the web monitor
if len(self.conf.web_monitor_url.strip()) > 0:
url_template = string.Template(self.conf.web_monitor_url)
url = url_template.substitute(instrument=self.instrument, run_number=self.run_number)
pattern = self.instrument + "_" + self.run_number + "*"
for dirpath, dirnames, filenames in os.walk(proposal_shared_dir):
listing = glob.glob(os.path.join(dirpath, pattern))
for filepath in listing:
f, e = os.path.splitext(filepath)
if e.startswith(os.extsep):
e = e[len(os.extsep):]
if e == "png" or e == "jpg" or filepath.endswith("plot_data.dat"):
files = {'file': open(filepath, 'rb')}
# Post the file if it's small enough
if len(files) != 0 and os.path.getsize(filepath) < self.conf.max_image_size:
request = requests.post(url, data=monitor_user, files=files, verify=False)
logging.info("Submitted %s [status: %s]" % (filepath,
request.status_code))
except:
logging.error("reduce: %s" % sys.exc_value)
self.data["error"] = "Reduction: %s " % sys.exc_value
self.send('/queue/' + self.conf.reduction_error , json.dumps(self.data))
