def should_proceed(self, data_dict):
if data_dict["rb_number"] in self.RBList:
logger.info("Duplicate RB run #" + data_dict["rb_number"] +
", waiting for the first to finish.")
return False
else:
return True
def shouldProceed(self, data_dict):
if data_dict["rb_number"] in self.RBList:
logger.info("Duplicate RB run #" + data_dict["rb_number"] + ", waiting for the first to finish.")
return False
else:
return True
def delete_temp_directory(self, temp_result_dir):
""" Remove temporary working directory """
logger.info("Remove temp dir %s " % temp_result_dir)
try:
shutil.rmtree(temp_result_dir, ignore_errors=True)
except Exception as e:
logger.info("Unable to remove temporary directory %s - %s" % temp_result_dir)
def __init__(self, data, connection):
logger.debug("json data: " + prettify(data))
data["information"] = socket.gethostname()
self.data = data
self.client = connection
self.reduction_log_stream = cStringIO.StringIO()
self.admin_log_stream = cStringIO.StringIO()
try:
if 'data' in data:
self.data_file = windows_to_linux_path(
str(data['data']), MISC["temp_root_directory"])
logger.debug("data_file: %s" % self.data_file)
else:
raise ValueError("data is missing")
if 'facility' in data:
self.facility = str(data['facility']).upper()
logger.debug("facility: %s" % self.facility)
else:
raise ValueError("facility is missing")
if 'instrument' in data:
self.instrument = str(data['instrument']).upper()
logger.debug("instrument: %s" % self.instrument)
else:
raise ValueError("instrument is missing")
if 'rb_number' in data:
self.proposal = str(data['rb_number']).upper()
logger.debug("rb_number: %s" % self.proposal)
else:
raise ValueError("rb_number is missing")
if 'run_number' in data:
self.run_number = str(int(data['run_number'])
) # Cast to int to remove trailing zeros
logger.debug("run_number: %s" % str(self.run_number))
else:
raise ValueError("run_number is missing")
if 'reduction_script' in data:
self.reduction_script = data['reduction_script']
logger.debug("reduction_script: %s ..." %
self.reduction_script[:50])
else:
raise ValueError("reduction_script is missing")
if 'reduction_arguments' in data:
self.reduction_arguments = data['reduction_arguments']
logger.debug("reduction_arguments: %s" %
self.reduction_arguments)
else:
raise ValueError("reduction_arguments is missing")
except ValueError:
logger.info('JSON data error', exc_info=True)
raise
def delete_temp_directory(temp_result_dir):
""" Remove temporary working directory """
logger.info("Remove temp dir %s " % temp_result_dir)
try:
shutil.rmtree(temp_result_dir, ignore_errors=True)
except:
logger.info("Unable to remove temporary directory %s - %s" %
temp_result_dir)
def __init__(self, data, conf, connection):
logger.debug("json data: " + prettify(data))
data["information"] = socket.gethostname()
self.data = data
self.conf = conf
self.client = connection
self.reduction_log_stream = cStringIO.StringIO()
self.admin_log_stream = cStringIO.StringIO()
try:
if data.has_key('data'):
self.data_file = windows_to_linux_path(str(data['data']), self.conf["temp_root_directory"])
logger.debug("data_file: %s" % self.data_file)
else:
raise ValueError("data is missing")
if data.has_key('facility'):
self.facility = str(data['facility']).upper()
logger.debug("facility: %s" % self.facility)
else:
raise ValueError("facility is missing")
if data.has_key('instrument'):
self.instrument = str(data['instrument']).upper()
logger.debug("instrument: %s" % self.instrument)
else:
raise ValueError("instrument is missing")
if data.has_key('rb_number'):
self.proposal = str(data['rb_number']).upper()
logger.debug("rb_number: %s" % self.proposal)
else:
raise ValueError("rb_number is missing")
if data.has_key('run_number'):
self.run_number = str(int(data['run_number'])) # Cast to int to remove trailing zeros
logger.debug("run_number: %s" % str(self.run_number))
else:
raise ValueError("run_number is missing")
if data.has_key('reduction_script'):
self.reduction_script = data['reduction_script']
logger.debug("reduction_script: %s ..." % self.reduction_script[:50])
else:
raise ValueError("reduction_script is missing")
if data.has_key('reduction_arguments'):
self.reduction_arguments = data['reduction_arguments']
logger.debug("reduction_arguments: %s" % self.reduction_arguments)
else:
raise ValueError("reduction_arguments is missing")
except ValueError:
logger.info('JSON data error', exc_info=True)
raise
def run(self):
brokers = [(self.config['brokers'].split(':')[0],int(self.config['brokers'].split(':')[1]))]
connection = stomp.Connection(host_and_ports=brokers, use_ssl=True, ssl_version=3)
connection.set_listener(self.consumer_name, Listener(connection, self.config))
connection.start()
connection.connect(self.config['amq_user'], self.config['amq_pwd'], wait=False, header={'activemq.prefetchSize': '1'})
for queue in self.config['amq_queues']:
connection.subscribe(destination=queue, id=1, ack='client-individual', header={'activemq.prefetchSize': '1'})
logger.info("[%s] Subscribing to %s" % (self.consumer_name, queue))
def copy_temp_directory(self, temp_result_dir, copy_destination):
""" Method that copies the temporary files held in results_directory to CEPH/archive, replacing old data if it
exists"""
# EXCITATION instrument are treated as a special case because they done what run number subfolders
if os.path.isdir(copy_destination) and self.instrument not in self.conf["excitation_instruments"]:
self._remove_directory(copy_destination)
self.data['reduction_data'].append(linux_to_windows_path(copy_destination))
logger.info("Moving %s to %s" % (temp_result_dir, copy_destination))
try:
self._copy_tree(temp_result_dir, copy_destination)
except Exception as e:
self.log_and_message("Unable to copy to %s - %s" % (copy_destination, e))
def copy_temp_directory(self, temp_result_dir, copy_destination):
""" Method that copies the temporary files held in results_directory to CEPH/archive, replacing old data if it
exists"""
# EXCITATION instrument are treated as a special case because they done what run number subfolders
if os.path.isdir(copy_destination) and self.instrument not in MISC[
"excitation_instruments"]:
self._remove_directory(copy_destination)
self.data['reduction_data'].append(
linux_to_windows_path(copy_destination))
logger.info("Moving %s to %s" % (temp_result_dir, copy_destination))
try:
self._copy_tree(temp_result_dir, copy_destination)
except Exception as exp:
self.log_and_message("Unable to copy to %s - %s" %
(copy_destination, exp))
def run(self):
logger.info('Called run ' + ACTIVEMQ['brokers'].split(':')[0] + ' ' +
ACTIVEMQ['brokers'].split(':')[1])
brokers = [(ACTIVEMQ['brokers'].split(':')[0],
int(ACTIVEMQ['brokers'].split(':')[1]))]
connection = stomp.Connection(host_and_ports=brokers, use_ssl=False)
connection.set_listener(self.consumer_name, Listener(connection))
logger.info("Starting ActiveMQ Connection to " + ACTIVEMQ['brokers'])
connection.start()
logger.info("Completed ActiveMQ Connection")
connection.connect(ACTIVEMQ['amq_user'],
ACTIVEMQ['amq_pwd'],
wait=False,
header={'activemq.prefetchSize': '1'})
for queue in ACTIVEMQ['amq_queues']:
connection.subscribe(destination=queue,
id='1',
ack='client-individual',
header={'activemq.prefetchSize': '1'})
logger.info("[%s] Subscribing to %s" % (self.consumer_name, queue))
logger.info("Successfully subscribed to all of the queues")
def main():
try:
config = json.load(open('/etc/autoreduce/post_process_consumer.conf'))
except:
logger.info("Can't open post_process_consumer.conf")
sys.exit()
logger.info("Start post process asynchronous listener!")
reactor.callWhenRunning(Consumer(config).run)
reactor.run()
logger.info("Stop post process asynchronous listener!")
self._remove_with_wait(False, full_path)
self._remove_with_wait(True, directory)
except Exception as exp:
self.log_and_message(
"Unable to remove existing directory %s - %s" %
(directory, exp))
if __name__ == "__main__":
brokers = []
brokers.append((ACTIVEMQ['brokers'].split(':')[0],
int(ACTIVEMQ['brokers'].split(':')[1])))
stomp_connection = stomp.Connection(host_and_ports=brokers, use_ssl=False)
json_data = None
try:
logger.info("PostProcessAdmin Connecting to ActiveMQ")
stomp_connection.start()
stomp_connection.connect(ACTIVEMQ['amq_user'],
ACTIVEMQ['amq_pwd'],
wait=True,
header={
'activemq.prefetchSize': '1',
})
logger.info("PostProcessAdmin Successfully Connected to ActiveMQ")
destination, message = sys.argv[1:3]
print("destination: " + destination)
print("message: " + prettify(message))
json_data = json.loads(message)
try:
def log_and_message(self, message):
"""Helper function to add text to the outgoing activemq message and to the info logs """
logger.info(message)
if self.data["message"] == "":
# Only send back first message as there is a char limit
self.data["message"] = message
def _send_error_and_log(self):
logger.info("\nCalling " + ACTIVEMQ['reduction_error'] + " --- " +
prettify(self.data))
self.client.send(ACTIVEMQ['reduction_error'], json.dumps(self.data))
def log_and_message(self, msg):
"""Helper function to add text to the outgoing activemq message and to the info logs """
logger.info(msg)
if self.data["message"] == "":
# Only send back first message as there is a char limit
self.data["message"] = msg
def _send_error_and_log(self):
logger.info("\nCalling " + self.conf['reduction_error'] + " --- " + prettify(self.data))
self.client.send(self.conf['reduction_error'], json.dumps(self.data))
def reduce(self):
logger.debug("In reduce() method")
try:
logger.debug("Calling: " + self.conf['reduction_started'] + "\n" + prettify(self.data))
self.client.send(self.conf['reduction_started'], json.dumps(self.data))
# specify instrument directory
cycle = re.match(r'.*cycle_(\d\d_\d).*', self.data_file.lower()).group(1)
if self.instrument in (self.conf["ceph_instruments"] + self.conf["excitation_instruments"]):
cycle = re.sub('[_]', '', cycle)
instrument_dir = self.conf["ceph_directory"] % (self.instrument, self.proposal, self.run_number)
if self.instrument in self.conf["excitation_instruments"]:
#Excitations would like to remove the run number folder at the end
instrument_dir = instrument_dir[:instrument_dir.rfind('/')+1]
else:
instrument_dir = self.conf["archive_directory"] % (self.instrument, cycle, self.proposal, self.run_number)
# specify directories where autoreduction output will go
reduce_result_dir = self.conf["temp_root_directory"] + instrument_dir
if self.instrument not in self.conf["excitation_instruments"]:
run_output_dir = os.path.join(self.conf["temp_root_directory"], instrument_dir[:instrument_dir.rfind('/')+1])
else:
run_output_dir = reduce_result_dir
log_dir = reduce_result_dir + "/reduction_log/"
log_and_err_name = "RB" + self.proposal + "Run" + self.run_number
script_out = os.path.join(log_dir, log_and_err_name + "Script.out")
mantid_log = os.path.join(log_dir, log_and_err_name + "Mantid.log")
final_result_dir = reduce_result_dir[len(self.conf["temp_root_directory"]):] # strip the temp path off the front of the temp directory to get the final archive directory
final_log_dir = log_dir[len(self.conf["temp_root_directory"]):]
# test for access to result paths
try:
shouldBeWritable = [reduce_result_dir, log_dir, final_result_dir, final_log_dir]
shouldBeReadable = [self.data_file]
# try to make directories which should exist
for path in filter( lambda p: not os.path.isdir(p), shouldBeWritable ):
os.makedirs(path)
doesNotExist = lambda path : not os.access(path, os.F_OK)
notReadable = lambda path : not os.access(path, os.R_OK)
notWritable = lambda path : not os.access(path, os.W_OK)
# we want write access to these directories, plus the final output paths
if filter(notWritable, shouldBeWritable) != []:
failPath = filter(notWritable, shouldBeWritable)[0]
problem = "does not exist" if doesNotExist(failPath) else "no write access"
raise Exception("Couldn't write to %s - %s" % (failPath, problem))
if filter(notReadable, shouldBeReadable) != []:
failPath = filter(notReadable, shouldBeReadable)[0]
problem = "does not exist" if doesNotExist(failPath) else "no read access"
raise Exception("Couldn't read %s - %s" % (failPath, problem))
except Exception as e:
# if we can't access now, we should abort the run, and tell the server that it should be re-run at a later time
self.data["message"] = "Permission error: %s" % e
self.data["retry_in"] = 6 * 60 * 60 # 6 hours
raise e
self.data['reduction_data'] = []
if "message" not in self.data:
self.data["message"] = ""
logger.info("----------------")
logger.info("Reduction script: %s ..." % self.reduction_script[:50])
logger.info("Result dir: %s" % reduce_result_dir)
logger.info("Run Output dir: %s" % run_output_dir)
logger.info("Log dir: %s" % log_dir)
logger.info("Out log: %s" % script_out)
logger.info("----------------")
logger.info("Reduction subprocess started.")
try:
with channels_redirected(script_out, mantid_log, self.reduction_log_stream):
# Load reduction script as a module. This works as long as reduce.py makes no assumption that it is in the same directory as reduce_vars,
# i.e., either it does not import it at all, or adds its location to os.path explicitly.
reduce_script = imp.new_module('reducescript')
exec self.reduction_script in reduce_script.__dict__ # loads the string as a module into reduce_script
try:
skip_numbers = reduce_script.SKIP_RUNS
except:
skip_numbers = []
pass
if self.data['run_number'] not in skip_numbers:
reduce_script = self.replace_variables(reduce_script)
out_directories = reduce_script.main(input_file=str(self.data_file), output_dir=str(reduce_result_dir))
else:
self.data['message'] = "Run has been skipped in script"
except Exception as e:
with open(script_out, "a") as f:
f.writelines(str(e) + "\n")
f.write(traceback.format_exc())
self.copy_temp_directory(reduce_result_dir, final_result_dir)
self.delete_temp_directory(reduce_result_dir)
errorStr = "Error in user reduction script: %s - %s" % (type(e).__name__, e)
raise Exception(errorStr) # parent except block will discard exception type, so format the type as a string now
logger.info("Reduction subprocess completed.")
logger.info("Additional save directories: %s" % out_directories)
self.copy_temp_directory(reduce_result_dir, final_result_dir)
# If the reduce script specified some additional save directories, copy to there first
if out_directories:
if type(out_directories) is str:
self.copy_temp_directory(reduce_result_dir, out_directories)
elif type(out_directories) is list:
for out_dir in out_directories:
if type(out_dir) is str:
self.copy_temp_directory(reduce_result_dir, out_dir)
else:
self.log_and_message("Optional output directories of reduce.py must be strings: %s" % out_dir)
else:
self.log_and_message("Optional output directories of reduce.py must be a string or list of stings: %s" % out_directories)
# no longer a need for the temp directory used for temporary storing of reduction results
self.delete_temp_directory(reduce_result_dir)
except Exception as e:
self.data["message"] = "REDUCTION Error: %s " % e
self.data['reduction_log'] = self.reduction_log_stream.getvalue()
self.data["admin_log"] = self.admin_log_stream.getvalue()
if self.data["message"] != "":
# This means an error has been produced somewhere
try:
self._send_error_and_log()
except Exception as e:
logger.info("Failed to send to queue! - %s - %s" % (e, repr(e)))
finally:
logger.info("Reduction job failed")
else:
# reduction has successfully completed
self.client.send(self.conf['reduction_complete'], json.dumps(self.data))
print("\nCalling: " + self.conf['reduction_complete'] + "\n" + prettify(self.data) + "\n")
logger.info("Reduction job successfully complete")
destination, message = sys.argv[1:3]
print("destination: " + destination)
print("message: " + prettify(message))
data = json.loads(message)
try:
pp = PostProcessAdmin(data, conf, connection)
log_stream_handler = logging.StreamHandler(pp.admin_log_stream)
logger.addHandler(log_stream_handler)
if destination == '/queue/ReductionPending':
pp.reduce()
except ValueError as e:
data["error"] = str(e)
logger.info("JSON data error: " + prettify(data))
raise
except Exception as e:
logger.info("PostProcessAdmin error: %s" % e)
raise
finally:
try:
logger.removeHandler(log_stream_handler)
except:
pass
except Exception as er:
logger.info("Something went wrong: " + str(er))
try:
def main():
logger.info("Start post process asynchronous listener!")
reactor.callWhenRunning(Consumer().run)
reactor.run()
logger.info("Stop post process asynchronous listener!")
def reduce(self):
try:
logger.debug("Calling: " + ACTIVEMQ['reduction_started'] + "\n" +
prettify(self.data))
self.client.send(ACTIVEMQ['reduction_started'],
json.dumps(self.data))
# Specify instrument directory
cycle = re.match(r'.*cycle_(\d\d_\d).*',
self.data_file.lower()).group(1)
if self.instrument in (MISC["ceph_instruments"] +
MISC["excitation_instruments"]):
instrument_dir = MISC["ceph_directory"] % (
self.instrument, self.proposal, self.run_number)
if self.instrument in MISC["excitation_instruments"]:
# Excitations would like to remove the run number folder at the end
instrument_dir = instrument_dir[:instrument_dir.
rfind('/') + 1]
else:
instrument_dir = MISC["archive_directory"] % (
self.instrument, cycle, self.proposal, self.run_number)
# Specify directories where autoreduction output will go
reduce_result_dir = MISC["temp_root_directory"] + instrument_dir
if self.instrument not in MISC["excitation_instruments"]:
run_output_dir = os.path.join(
MISC["temp_root_directory"],
instrument_dir[:instrument_dir.rfind('/') + 1])
else:
run_output_dir = reduce_result_dir
if 'run_description' in self.data:
logger.info("DESCRIPTION: " + self.data["run_description"])
log_dir = reduce_result_dir + "/reduction_log/"
log_and_err_name = "RB" + self.proposal + "Run" + self.run_number
script_out = os.path.join(log_dir, log_and_err_name + "Script.out")
mantid_log = os.path.join(log_dir, log_and_err_name + "Mantid.log")
# strip the temp path off the front of the temp directory to get the final archive directory
final_result_dir = reduce_result_dir[
len(MISC["temp_root_directory"]):]
final_log_dir = log_dir[len(MISC["temp_root_directory"]):]
if 'overwrite' in self.data:
if not self.data["overwrite"]:
logger.info('Don\'t want to overwrite previous data')
path_parts = final_result_dir.split('/')
new_path = '/'
for part in path_parts:
if part != 'autoreduced' and part != '':
new_path = new_path + part + '/'
maximum = 0
for folder in os.listdir(new_path):
if folder.startswith('autoreduced'):
number = folder.replace('autoreduced', '')
if number != '':
number = int(number) + 1
if number > maximum:
maximum = number
else:
maximum = 1
if maximum == 0:
new_path = new_path + 'autoreduced'
else:
new_path = new_path + 'autoreduced' + str(
maximum) + '/'
final_result_dir = new_path
final_log_dir = new_path + 'reduction_log/'
logger.info('Final Result Directory = ' + final_result_dir)
logger.info('Final Log Directory = ' + final_log_dir)
# test for access to result paths
try:
should_be_writable = [
reduce_result_dir, log_dir, final_result_dir, final_log_dir
]
should_be_readable = [self.data_file]
# try to make directories which should exist
for path in filter(lambda p: not os.path.isdir(p),
should_be_writable):
os.makedirs(path)
does_not_exist = lambda path: not os.access(path, os.F_OK)
not_readable = lambda path: not os.access(path, os.R_OK)
not_writable = lambda path: not os.access(path, os.W_OK)
# we want write access to these directories, plus the final output paths
if len(filter(not_writable, should_be_writable)) > 0:
fail_path = filter(not_writable, should_be_writable)[0]
problem = "does not exist" if does_not_exist(
fail_path) else "no write access"
raise Exception("Couldn't write to %s - %s" %
(fail_path, problem))
if len(filter(not_readable, should_be_readable)) > 0:
fail_path = filter(not_readable, should_be_readable)[0]
problem = "does not exist" if does_not_exist(
fail_path) else "no read access"
raise Exception("Couldn't read %s - %s" %
(fail_path, problem))
except Exception as exp:
# if we can't access now, we should abort the run, and tell the server that it should be
# re-run at a later time
self.data["message"] = "Permission error: %s" % exp
# 6 hours
self.data["retry_in"] = 6 * 60 * 60
logger.error(traceback.format_exc())
raise exp
self.data['reduction_data'] = []
if "message" not in self.data:
self.data["message"] = ""
logger.info("----------------")
logger.info("Reduction script: %s ..." %
self.reduction_script[:50])
logger.info("Result dir: %s" % reduce_result_dir)
logger.info("Run Output dir: %s" % run_output_dir)
logger.info("Log dir: %s" % log_dir)
logger.info("Out log: %s" % script_out)
logger.info("Datafile: %s" % self.data_file)
logger.info("----------------")
logger.info("Reduction subprocess started.")
logger.info(reduce_result_dir)
out_directories = None
try:
with channels_redirected(script_out, mantid_log,
self.reduction_log_stream):
"""
Load reduction script as a module. This works as long as reduce.py makes no assumption that it is
in the same directory as reduce_vars, i.e., either it does not import it at all, or adds its
location to os.path explicitly.
"""
# Append the Mantid path to the system path such that we can use Mantid to run the user's script
sys.path.append(MISC["mantid_path"])
reduce_script_location = self._load_reduction_script(
self.instrument)
reduce_script = imp.load_source('reducescript',
reduce_script_location)
try:
skip_numbers = reduce_script.SKIP_RUNS
except:
skip_numbers = []
pass
if self.data['run_number'] not in skip_numbers:
reduce_script = self.replace_variables(reduce_script)
with timeout(MISC["script_timeout"]):
out_directories = reduce_script.main(
input_file=str(self.data_file),
output_dir=str(reduce_result_dir))
else:
self.data['message'] = "Run has been skipped in script"
except Exception as exp:
with open(script_out, "a") as f:
f.writelines(str(exp) + "\n")
f.write(traceback.format_exc())
self.copy_temp_directory(reduce_result_dir, final_result_dir)
self.delete_temp_directory(reduce_result_dir)
error_str = "Error in user reduction script: %s - %s" % (
type(exp).__name__, exp)
logger.error(traceback.format_exc())
# Parent except block will discard exception type, so format the type as a string now
raise Exception(error_str)
logger.info("Reduction subprocess completed.")
logger.info("Additional save directories: %s" % out_directories)
self.copy_temp_directory(reduce_result_dir, final_result_dir)
# If the reduce script specified some additional save directories, copy to there first
if out_directories:
if type(out_directories) is str:
self.copy_temp_directory(reduce_result_dir,
out_directories)
elif type(out_directories) is list:
for out_dir in out_directories:
if type(out_dir) is str:
self.copy_temp_directory(reduce_result_dir,
out_dir)
else:
self.log_and_message(
"Optional output directories of reduce.py must be strings: %s"
% out_dir)
else:
self.log_and_message(
"Optional output directories of reduce.py must be a string or list of stings: "
"%s" % out_directories)
# no longer a need for the temp directory used for temporary storing of reduction results
self.delete_temp_directory(reduce_result_dir)
except Exception as exp:
logger.error(traceback.format_exc())
self.data["message"] = "REDUCTION Error: %s " % exp
self.data['reduction_log'] = self.reduction_log_stream.getvalue()
self.data["admin_log"] = self.admin_log_stream.getvalue()
if self.data["message"] != "":
# This means an error has been produced somewhere
try:
self._send_error_and_log()
except Exception:
logger.info("Failed to send to queue! - %s - %s" %
(e, repr(e)))
finally:
logger.info("Reduction job failed")
else:
# reduction has successfully completed
self.client.send(ACTIVEMQ['reduction_complete'],
json.dumps(self.data))
print("\nCalling: " + ACTIVEMQ['reduction_complete'] + "\n" +
prettify(self.data) + "\n")
logger.info("Reduction job successfully complete")
