def hold_message(self, destination, data, headers):
logger.debug("holding thread")
data_dict = json.loads(data)
self.update_child_process_list()
if not self.should_proceed(
data_dict): # wait while the run shouldn't proceed
reactor.callLater(10, self.hold_message, destination, data,
headers)
return
if self.should_cancel(data_dict):
self.cancel_run(data_dict)
return
print_dict = data_dict.copy()
print_dict.pop("reduction_script")
logger.debug("Calling: %s %s %s %s" %
("python", MISC['post_process_directory'], destination,
print_dict))
self._client.ack(headers['message-id'],
headers['subscription']) # Remove message from queue
proc = subprocess.Popen(
["python", MISC['post_process_directory'], destination, data])
self.add_process(proc, data_dict)
def on_message(self, headers, data):
destination = headers['destination']
logger.debug("Received frame destination: " + destination)
logger.debug("Recieved frame priority: " + headers["priority"])
self.update_child_process_list()
data_dict = json.loads(data)
if "cancel" in data_dict and data_dict["cancel"]:
self.add_cancel(data_dict)
return
self.hold_message(destination, data, headers)
def on_message(self, headers, data):
self._client.ack(headers['message-id'], headers['subscription']) # Remove message from queue
destination = headers['destination']
logger.debug("Received frame destination: " + destination)
logger.debug("Recieved frame priority: " + headers["priority"])
self.updateChildProcessList()
data_dict = json.loads(data)
if "cancel" in data_dict and data_dict["cancel"]:
self.addCancel(data_dict)
return
self.holdMessage(destination, data)
def holdMessage(self, destination, data):
logger.debug("holding thread")
data_dict = json.loads(data)
self.updateChildProcessList()
if not self.shouldProceed(data_dict): # wait while the run shouldn't proceed
reactor.callLater(10, self.holdMessage, destination, data)
return
if self.shouldCancel(data_dict):
self.cancelRun(data_dict)
return
print_dict = data_dict.copy()
print_dict.pop("reduction_script")
logger.debug("Calling: %s %s %s %s" % ("python", "/usr/bin/PostProcessAdmin.py", destination, print_dict))
proc = subprocess.Popen(["python", "/usr/bin/PostProcessAdmin.py", destination, data])
self.addProc(proc, data_dict)
def __init__(self, data, conf, connection):
logger.debug("json data: " + prettify(data))
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
self.client = connection
self.reduction_log_stream = cStringIO.StringIO()
self.admin_log_stream = cStringIO.StringIO()
try:
if data.has_key('data'):
self.data_file = windows_to_linux_path(str(data['data']), self.conf["temp_root_directory"])
logger.debug("data_file: %s" % self.data_file)
else:
raise ValueError("data is missing")
if data.has_key('facility'):
self.facility = str(data['facility']).upper()
logger.debug("facility: %s" % self.facility)
else:
raise ValueError("facility is missing")
if data.has_key('instrument'):
self.instrument = str(data['instrument']).upper()
logger.debug("instrument: %s" % self.instrument)
else:
raise ValueError("instrument is missing")
if data.has_key('rb_number'):
self.proposal = str(data['rb_number']).upper()
logger.debug("rb_number: %s" % self.proposal)
else:
raise ValueError("rb_number is missing")
if data.has_key('run_number'):
self.run_number = str(int(data['run_number'])) # Cast to int to remove trailing zeros
logger.debug("run_number: %s" % str(self.run_number))
else:
raise ValueError("run_number is missing")
if data.has_key('reduction_script'):
self.reduction_script = data['reduction_script']
logger.debug("reduction_script: %s ..." % self.reduction_script[:50])
else:
raise ValueError("reduction_script is missing")
if data.has_key('reduction_arguments'):
self.reduction_arguments = data['reduction_arguments']
logger.debug("reduction_arguments: %s" % self.reduction_arguments)
else:
raise ValueError("reduction_arguments is missing")
except ValueError:
logger.info('JSON data error', exc_info=True)
raise
def reduce(self):
logger.debug("In reduce() method")
try:
logger.debug("Calling: " + self.conf['reduction_started'] + "\n" + prettify(self.data))
self.client.send(self.conf['reduction_started'], json.dumps(self.data))
# specify instrument directory
cycle = re.match(r'.*cycle_(\d\d_\d).*', self.data_file.lower()).group(1)
if self.instrument in (self.conf["ceph_instruments"] + self.conf["excitation_instruments"]):
cycle = re.sub('[_]', '', cycle)
instrument_dir = self.conf["ceph_directory"] % (self.instrument, self.proposal, self.run_number)
if self.instrument in self.conf["excitation_instruments"]:
#Excitations would like to remove the run number folder at the end
instrument_dir = instrument_dir[:instrument_dir.rfind('/')+1]
else:
instrument_dir = self.conf["archive_directory"] % (self.instrument, cycle, self.proposal, self.run_number)
# specify directories where autoreduction output will go
reduce_result_dir = self.conf["temp_root_directory"] + instrument_dir
if self.instrument not in self.conf["excitation_instruments"]:
run_output_dir = os.path.join(self.conf["temp_root_directory"], instrument_dir[:instrument_dir.rfind('/')+1])
else:
run_output_dir = reduce_result_dir
log_dir = reduce_result_dir + "/reduction_log/"
log_and_err_name = "RB" + self.proposal + "Run" + self.run_number
script_out = os.path.join(log_dir, log_and_err_name + "Script.out")
mantid_log = os.path.join(log_dir, log_and_err_name + "Mantid.log")
final_result_dir = reduce_result_dir[len(self.conf["temp_root_directory"]):] # strip the temp path off the front of the temp directory to get the final archive directory
final_log_dir = log_dir[len(self.conf["temp_root_directory"]):]
# test for access to result paths
try:
shouldBeWritable = [reduce_result_dir, log_dir, final_result_dir, final_log_dir]
shouldBeReadable = [self.data_file]
# try to make directories which should exist
for path in filter( lambda p: not os.path.isdir(p), shouldBeWritable ):
os.makedirs(path)
doesNotExist = lambda path : not os.access(path, os.F_OK)
notReadable = lambda path : not os.access(path, os.R_OK)
notWritable = lambda path : not os.access(path, os.W_OK)
# we want write access to these directories, plus the final output paths
if filter(notWritable, shouldBeWritable) != []:
failPath = filter(notWritable, shouldBeWritable)[0]
problem = "does not exist" if doesNotExist(failPath) else "no write access"
raise Exception("Couldn't write to %s - %s" % (failPath, problem))
if filter(notReadable, shouldBeReadable) != []:
failPath = filter(notReadable, shouldBeReadable)[0]
problem = "does not exist" if doesNotExist(failPath) else "no read access"
raise Exception("Couldn't read %s - %s" % (failPath, problem))
except Exception as e:
# if we can't access now, we should abort the run, and tell the server that it should be re-run at a later time
self.data["message"] = "Permission error: %s" % e
self.data["retry_in"] = 6 * 60 * 60 # 6 hours
raise e
self.data['reduction_data'] = []
if "message" not in self.data:
self.data["message"] = ""
logger.info("----------------")
logger.info("Reduction script: %s ..." % self.reduction_script[:50])
logger.info("Result dir: %s" % reduce_result_dir)
logger.info("Run Output dir: %s" % run_output_dir)
logger.info("Log dir: %s" % log_dir)
logger.info("Out log: %s" % script_out)
logger.info("----------------")
logger.info("Reduction subprocess started.")
try:
with channels_redirected(script_out, mantid_log, self.reduction_log_stream):
# Load reduction script as a module. This works as long as reduce.py makes no assumption that it is in the same directory as reduce_vars,
# i.e., either it does not import it at all, or adds its location to os.path explicitly.
reduce_script = imp.new_module('reducescript')
exec self.reduction_script in reduce_script.__dict__ # loads the string as a module into reduce_script
try:
skip_numbers = reduce_script.SKIP_RUNS
except:
skip_numbers = []
pass
if self.data['run_number'] not in skip_numbers:
reduce_script = self.replace_variables(reduce_script)
out_directories = reduce_script.main(input_file=str(self.data_file), output_dir=str(reduce_result_dir))
else:
self.data['message'] = "Run has been skipped in script"
except Exception as e:
with open(script_out, "a") as f:
f.writelines(str(e) + "\n")
f.write(traceback.format_exc())
self.copy_temp_directory(reduce_result_dir, final_result_dir)
self.delete_temp_directory(reduce_result_dir)
errorStr = "Error in user reduction script: %s - %s" % (type(e).__name__, e)
raise Exception(errorStr) # parent except block will discard exception type, so format the type as a string now
logger.info("Reduction subprocess completed.")
logger.info("Additional save directories: %s" % out_directories)
self.copy_temp_directory(reduce_result_dir, final_result_dir)
# If the reduce script specified some additional save directories, copy to there first
if out_directories:
if type(out_directories) is str:
self.copy_temp_directory(reduce_result_dir, out_directories)
elif type(out_directories) is list:
for out_dir in out_directories:
if type(out_dir) is str:
self.copy_temp_directory(reduce_result_dir, out_dir)
else:
self.log_and_message("Optional output directories of reduce.py must be strings: %s" % out_dir)
else:
self.log_and_message("Optional output directories of reduce.py must be a string or list of stings: %s" % out_directories)
# no longer a need for the temp directory used for temporary storing of reduction results
self.delete_temp_directory(reduce_result_dir)
except Exception as e:
self.data["message"] = "REDUCTION Error: %s " % e
self.data['reduction_log'] = self.reduction_log_stream.getvalue()
self.data["admin_log"] = self.admin_log_stream.getvalue()
if self.data["message"] != "":
# This means an error has been produced somewhere
try:
self._send_error_and_log()
except Exception as e:
logger.info("Failed to send to queue! - %s - %s" % (e, repr(e)))
finally:
logger.info("Reduction job failed")
else:
# reduction has successfully completed
self.client.send(self.conf['reduction_complete'], json.dumps(self.data))
print("\nCalling: " + self.conf['reduction_complete'] + "\n" + prettify(self.data) + "\n")
logger.info("Reduction job successfully complete")
def __init__(self, data, connection):
logger.debug("json data: " + prettify(data))
data["information"] = socket.gethostname()
self.data = data
self.client = connection
self.reduction_log_stream = cStringIO.StringIO()
self.admin_log_stream = cStringIO.StringIO()
try:
if 'data' in data:
self.data_file = windows_to_linux_path(
str(data['data']), MISC["temp_root_directory"])
logger.debug("data_file: %s" % self.data_file)
else:
raise ValueError("data is missing")
if 'facility' in data:
self.facility = str(data['facility']).upper()
logger.debug("facility: %s" % self.facility)
else:
raise ValueError("facility is missing")
if 'instrument' in data:
self.instrument = str(data['instrument']).upper()
logger.debug("instrument: %s" % self.instrument)
else:
raise ValueError("instrument is missing")
if 'rb_number' in data:
self.proposal = str(data['rb_number']).upper()
logger.debug("rb_number: %s" % self.proposal)
else:
raise ValueError("rb_number is missing")
if 'run_number' in data:
self.run_number = str(int(data['run_number'])
) # Cast to int to remove trailing zeros
logger.debug("run_number: %s" % str(self.run_number))
else:
raise ValueError("run_number is missing")
if 'reduction_script' in data:
self.reduction_script = data['reduction_script']
logger.debug("reduction_script: %s ..." %
self.reduction_script[:50])
else:
raise ValueError("reduction_script is missing")
if 'reduction_arguments' in data:
self.reduction_arguments = data['reduction_arguments']
logger.debug("reduction_arguments: %s" %
self.reduction_arguments)
else:
raise ValueError("reduction_arguments is missing")
except ValueError:
logger.info('JSON data error', exc_info=True)
raise
def reduce(self):
try:
logger.debug("Calling: " + ACTIVEMQ['reduction_started'] + "\n" +
prettify(self.data))
self.client.send(ACTIVEMQ['reduction_started'],
json.dumps(self.data))
# Specify instrument directory
cycle = re.match(r'.*cycle_(\d\d_\d).*',
self.data_file.lower()).group(1)
if self.instrument in (MISC["ceph_instruments"] +
MISC["excitation_instruments"]):
instrument_dir = MISC["ceph_directory"] % (
self.instrument, self.proposal, self.run_number)
if self.instrument in MISC["excitation_instruments"]:
# Excitations would like to remove the run number folder at the end
instrument_dir = instrument_dir[:instrument_dir.
rfind('/') + 1]
else:
instrument_dir = MISC["archive_directory"] % (
self.instrument, cycle, self.proposal, self.run_number)
# Specify directories where autoreduction output will go
reduce_result_dir = MISC["temp_root_directory"] + instrument_dir
if self.instrument not in MISC["excitation_instruments"]:
run_output_dir = os.path.join(
MISC["temp_root_directory"],
instrument_dir[:instrument_dir.rfind('/') + 1])
else:
run_output_dir = reduce_result_dir
if 'run_description' in self.data:
logger.info("DESCRIPTION: " + self.data["run_description"])
log_dir = reduce_result_dir + "/reduction_log/"
log_and_err_name = "RB" + self.proposal + "Run" + self.run_number
script_out = os.path.join(log_dir, log_and_err_name + "Script.out")
mantid_log = os.path.join(log_dir, log_and_err_name + "Mantid.log")
# strip the temp path off the front of the temp directory to get the final archive directory
final_result_dir = reduce_result_dir[
len(MISC["temp_root_directory"]):]
final_log_dir = log_dir[len(MISC["temp_root_directory"]):]
if 'overwrite' in self.data:
if not self.data["overwrite"]:
logger.info('Don\'t want to overwrite previous data')
path_parts = final_result_dir.split('/')
new_path = '/'
for part in path_parts:
if part != 'autoreduced' and part != '':
new_path = new_path + part + '/'
maximum = 0
for folder in os.listdir(new_path):
if folder.startswith('autoreduced'):
number = folder.replace('autoreduced', '')
if number != '':
number = int(number) + 1
if number > maximum:
maximum = number
else:
maximum = 1
if maximum == 0:
new_path = new_path + 'autoreduced'
else:
new_path = new_path + 'autoreduced' + str(
maximum) + '/'
final_result_dir = new_path
final_log_dir = new_path + 'reduction_log/'
logger.info('Final Result Directory = ' + final_result_dir)
logger.info('Final Log Directory = ' + final_log_dir)
# test for access to result paths
try:
should_be_writable = [
reduce_result_dir, log_dir, final_result_dir, final_log_dir
]
should_be_readable = [self.data_file]
# try to make directories which should exist
for path in filter(lambda p: not os.path.isdir(p),
should_be_writable):
os.makedirs(path)
does_not_exist = lambda path: not os.access(path, os.F_OK)
not_readable = lambda path: not os.access(path, os.R_OK)
not_writable = lambda path: not os.access(path, os.W_OK)
# we want write access to these directories, plus the final output paths
if len(filter(not_writable, should_be_writable)) > 0:
fail_path = filter(not_writable, should_be_writable)[0]
problem = "does not exist" if does_not_exist(
fail_path) else "no write access"
raise Exception("Couldn't write to %s - %s" %
(fail_path, problem))
if len(filter(not_readable, should_be_readable)) > 0:
fail_path = filter(not_readable, should_be_readable)[0]
problem = "does not exist" if does_not_exist(
fail_path) else "no read access"
raise Exception("Couldn't read %s - %s" %
(fail_path, problem))
except Exception as exp:
# if we can't access now, we should abort the run, and tell the server that it should be
# re-run at a later time
self.data["message"] = "Permission error: %s" % exp
# 6 hours
self.data["retry_in"] = 6 * 60 * 60
logger.error(traceback.format_exc())
raise exp
self.data['reduction_data'] = []
if "message" not in self.data:
self.data["message"] = ""
logger.info("----------------")
logger.info("Reduction script: %s ..." %
self.reduction_script[:50])
logger.info("Result dir: %s" % reduce_result_dir)
logger.info("Run Output dir: %s" % run_output_dir)
logger.info("Log dir: %s" % log_dir)
logger.info("Out log: %s" % script_out)
logger.info("Datafile: %s" % self.data_file)
logger.info("----------------")
logger.info("Reduction subprocess started.")
logger.info(reduce_result_dir)
out_directories = None
try:
with channels_redirected(script_out, mantid_log,
self.reduction_log_stream):
"""
Load reduction script as a module. This works as long as reduce.py makes no assumption that it is
in the same directory as reduce_vars, i.e., either it does not import it at all, or adds its
location to os.path explicitly.
"""
# Append the Mantid path to the system path such that we can use Mantid to run the user's script
sys.path.append(MISC["mantid_path"])
reduce_script_location = self._load_reduction_script(
self.instrument)
reduce_script = imp.load_source('reducescript',
reduce_script_location)
try:
skip_numbers = reduce_script.SKIP_RUNS
except:
skip_numbers = []
pass
if self.data['run_number'] not in skip_numbers:
reduce_script = self.replace_variables(reduce_script)
with timeout(MISC["script_timeout"]):
out_directories = reduce_script.main(
input_file=str(self.data_file),
output_dir=str(reduce_result_dir))
else:
self.data['message'] = "Run has been skipped in script"
except Exception as exp:
with open(script_out, "a") as f:
f.writelines(str(exp) + "\n")
f.write(traceback.format_exc())
self.copy_temp_directory(reduce_result_dir, final_result_dir)
self.delete_temp_directory(reduce_result_dir)
error_str = "Error in user reduction script: %s - %s" % (
type(exp).__name__, exp)
logger.error(traceback.format_exc())
# Parent except block will discard exception type, so format the type as a string now
raise Exception(error_str)
logger.info("Reduction subprocess completed.")
logger.info("Additional save directories: %s" % out_directories)
self.copy_temp_directory(reduce_result_dir, final_result_dir)
# If the reduce script specified some additional save directories, copy to there first
if out_directories:
if type(out_directories) is str:
self.copy_temp_directory(reduce_result_dir,
out_directories)
elif type(out_directories) is list:
for out_dir in out_directories:
if type(out_dir) is str:
self.copy_temp_directory(reduce_result_dir,
out_dir)
else:
self.log_and_message(
"Optional output directories of reduce.py must be strings: %s"
% out_dir)
else:
self.log_and_message(
"Optional output directories of reduce.py must be a string or list of stings: "
"%s" % out_directories)
# no longer a need for the temp directory used for temporary storing of reduction results
self.delete_temp_directory(reduce_result_dir)
except Exception as exp:
logger.error(traceback.format_exc())
self.data["message"] = "REDUCTION Error: %s " % exp
self.data['reduction_log'] = self.reduction_log_stream.getvalue()
self.data["admin_log"] = self.admin_log_stream.getvalue()
if self.data["message"] != "":
# This means an error has been produced somewhere
try:
self._send_error_and_log()
except Exception:
logger.info("Failed to send to queue! - %s - %s" %
(e, repr(e)))
finally:
logger.info("Reduction job failed")
else:
# reduction has successfully completed
self.client.send(ACTIVEMQ['reduction_complete'],
json.dumps(self.data))
print("\nCalling: " + ACTIVEMQ['reduction_complete'] + "\n" +
prettify(self.data) + "\n")
logger.info("Reduction job successfully complete")
