def hold_message(self, destination, data, headers): logger.debug("holding thread") data_dict = json.loads(data) self.update_child_process_list() if not self.should_proceed( data_dict): # wait while the run shouldn't proceed reactor.callLater(10, self.hold_message, destination, data, headers) return if self.should_cancel(data_dict): self.cancel_run(data_dict) return print_dict = data_dict.copy() print_dict.pop("reduction_script") logger.debug("Calling: %s %s %s %s" % ("python", MISC['post_process_directory'], destination, print_dict)) self._client.ack(headers['message-id'], headers['subscription']) # Remove message from queue proc = subprocess.Popen( ["python", MISC['post_process_directory'], destination, data]) self.add_process(proc, data_dict)
def on_message(self, headers, data): destination = headers['destination'] logger.debug("Received frame destination: " + destination) logger.debug("Recieved frame priority: " + headers["priority"]) self.update_child_process_list() data_dict = json.loads(data) if "cancel" in data_dict and data_dict["cancel"]: self.add_cancel(data_dict) return self.hold_message(destination, data, headers)
def on_message(self, headers, data): self._client.ack(headers['message-id'], headers['subscription']) # Remove message from queue destination = headers['destination'] logger.debug("Received frame destination: " + destination) logger.debug("Recieved frame priority: " + headers["priority"]) self.updateChildProcessList() data_dict = json.loads(data) if "cancel" in data_dict and data_dict["cancel"]: self.addCancel(data_dict) return self.holdMessage(destination, data)
def holdMessage(self, destination, data): logger.debug("holding thread") data_dict = json.loads(data) self.updateChildProcessList() if not self.shouldProceed(data_dict): # wait while the run shouldn't proceed reactor.callLater(10, self.holdMessage, destination, data) return if self.shouldCancel(data_dict): self.cancelRun(data_dict) return print_dict = data_dict.copy() print_dict.pop("reduction_script") logger.debug("Calling: %s %s %s %s" % ("python", "/usr/bin/PostProcessAdmin.py", destination, print_dict)) proc = subprocess.Popen(["python", "/usr/bin/PostProcessAdmin.py", destination, data]) self.addProc(proc, data_dict)
def __init__(self, data, conf, connection): logger.debug("json data: " + prettify(data)) data["information"] = socket.gethostname() self.data = data self.conf = conf self.client = connection self.reduction_log_stream = cStringIO.StringIO() self.admin_log_stream = cStringIO.StringIO() try: if data.has_key('data'): self.data_file = windows_to_linux_path(str(data['data']), self.conf["temp_root_directory"]) logger.debug("data_file: %s" % self.data_file) else: raise ValueError("data is missing") if data.has_key('facility'): self.facility = str(data['facility']).upper() logger.debug("facility: %s" % self.facility) else: raise ValueError("facility is missing") if data.has_key('instrument'): self.instrument = str(data['instrument']).upper() logger.debug("instrument: %s" % self.instrument) else: raise ValueError("instrument is missing") if data.has_key('rb_number'): self.proposal = str(data['rb_number']).upper() logger.debug("rb_number: %s" % self.proposal) else: raise ValueError("rb_number is missing") if data.has_key('run_number'): self.run_number = str(int(data['run_number'])) # Cast to int to remove trailing zeros logger.debug("run_number: %s" % str(self.run_number)) else: raise ValueError("run_number is missing") if data.has_key('reduction_script'): self.reduction_script = data['reduction_script'] logger.debug("reduction_script: %s ..." % self.reduction_script[:50]) else: raise ValueError("reduction_script is missing") if data.has_key('reduction_arguments'): self.reduction_arguments = data['reduction_arguments'] logger.debug("reduction_arguments: %s" % self.reduction_arguments) else: raise ValueError("reduction_arguments is missing") except ValueError: logger.info('JSON data error', exc_info=True) raise
def reduce(self): logger.debug("In reduce() method") try: logger.debug("Calling: " + self.conf['reduction_started'] + "\n" + prettify(self.data)) self.client.send(self.conf['reduction_started'], json.dumps(self.data)) # specify instrument directory cycle = re.match(r'.*cycle_(\d\d_\d).*', self.data_file.lower()).group(1) if self.instrument in (self.conf["ceph_instruments"] + self.conf["excitation_instruments"]): cycle = re.sub('[_]', '', cycle) instrument_dir = self.conf["ceph_directory"] % (self.instrument, self.proposal, self.run_number) if self.instrument in self.conf["excitation_instruments"]: #Excitations would like to remove the run number folder at the end instrument_dir = instrument_dir[:instrument_dir.rfind('/')+1] else: instrument_dir = self.conf["archive_directory"] % (self.instrument, cycle, self.proposal, self.run_number) # specify directories where autoreduction output will go reduce_result_dir = self.conf["temp_root_directory"] + instrument_dir if self.instrument not in self.conf["excitation_instruments"]: run_output_dir = os.path.join(self.conf["temp_root_directory"], instrument_dir[:instrument_dir.rfind('/')+1]) else: run_output_dir = reduce_result_dir log_dir = reduce_result_dir + "/reduction_log/" log_and_err_name = "RB" + self.proposal + "Run" + self.run_number script_out = os.path.join(log_dir, log_and_err_name + "Script.out") mantid_log = os.path.join(log_dir, log_and_err_name + "Mantid.log") final_result_dir = reduce_result_dir[len(self.conf["temp_root_directory"]):] # strip the temp path off the front of the temp directory to get the final archive directory final_log_dir = log_dir[len(self.conf["temp_root_directory"]):] # test for access to result paths try: shouldBeWritable = [reduce_result_dir, log_dir, final_result_dir, final_log_dir] shouldBeReadable = [self.data_file] # try to make directories which should exist for path in filter( lambda p: not os.path.isdir(p), shouldBeWritable ): os.makedirs(path) doesNotExist = lambda path : not os.access(path, os.F_OK) notReadable = lambda path : not os.access(path, os.R_OK) notWritable = lambda path : not os.access(path, os.W_OK) # we want write access to these directories, plus the final output paths if filter(notWritable, shouldBeWritable) != []: failPath = filter(notWritable, shouldBeWritable)[0] problem = "does not exist" if doesNotExist(failPath) else "no write access" raise Exception("Couldn't write to %s - %s" % (failPath, problem)) if filter(notReadable, shouldBeReadable) != []: failPath = filter(notReadable, shouldBeReadable)[0] problem = "does not exist" if doesNotExist(failPath) else "no read access" raise Exception("Couldn't read %s - %s" % (failPath, problem)) except Exception as e: # if we can't access now, we should abort the run, and tell the server that it should be re-run at a later time self.data["message"] = "Permission error: %s" % e self.data["retry_in"] = 6 * 60 * 60 # 6 hours raise e self.data['reduction_data'] = [] if "message" not in self.data: self.data["message"] = "" logger.info("----------------") logger.info("Reduction script: %s ..." % self.reduction_script[:50]) logger.info("Result dir: %s" % reduce_result_dir) logger.info("Run Output dir: %s" % run_output_dir) logger.info("Log dir: %s" % log_dir) logger.info("Out log: %s" % script_out) logger.info("----------------") logger.info("Reduction subprocess started.") try: with channels_redirected(script_out, mantid_log, self.reduction_log_stream): # Load reduction script as a module. This works as long as reduce.py makes no assumption that it is in the same directory as reduce_vars, # i.e., either it does not import it at all, or adds its location to os.path explicitly. reduce_script = imp.new_module('reducescript') exec self.reduction_script in reduce_script.__dict__ # loads the string as a module into reduce_script try: skip_numbers = reduce_script.SKIP_RUNS except: skip_numbers = [] pass if self.data['run_number'] not in skip_numbers: reduce_script = self.replace_variables(reduce_script) out_directories = reduce_script.main(input_file=str(self.data_file), output_dir=str(reduce_result_dir)) else: self.data['message'] = "Run has been skipped in script" except Exception as e: with open(script_out, "a") as f: f.writelines(str(e) + "\n") f.write(traceback.format_exc()) self.copy_temp_directory(reduce_result_dir, final_result_dir) self.delete_temp_directory(reduce_result_dir) errorStr = "Error in user reduction script: %s - %s" % (type(e).__name__, e) raise Exception(errorStr) # parent except block will discard exception type, so format the type as a string now logger.info("Reduction subprocess completed.") logger.info("Additional save directories: %s" % out_directories) self.copy_temp_directory(reduce_result_dir, final_result_dir) # If the reduce script specified some additional save directories, copy to there first if out_directories: if type(out_directories) is str: self.copy_temp_directory(reduce_result_dir, out_directories) elif type(out_directories) is list: for out_dir in out_directories: if type(out_dir) is str: self.copy_temp_directory(reduce_result_dir, out_dir) else: self.log_and_message("Optional output directories of reduce.py must be strings: %s" % out_dir) else: self.log_and_message("Optional output directories of reduce.py must be a string or list of stings: %s" % out_directories) # no longer a need for the temp directory used for temporary storing of reduction results self.delete_temp_directory(reduce_result_dir) except Exception as e: self.data["message"] = "REDUCTION Error: %s " % e self.data['reduction_log'] = self.reduction_log_stream.getvalue() self.data["admin_log"] = self.admin_log_stream.getvalue() if self.data["message"] != "": # This means an error has been produced somewhere try: self._send_error_and_log() except Exception as e: logger.info("Failed to send to queue! - %s - %s" % (e, repr(e))) finally: logger.info("Reduction job failed") else: # reduction has successfully completed self.client.send(self.conf['reduction_complete'], json.dumps(self.data)) print("\nCalling: " + self.conf['reduction_complete'] + "\n" + prettify(self.data) + "\n") logger.info("Reduction job successfully complete")
def __init__(self, data, connection): logger.debug("json data: " + prettify(data)) data["information"] = socket.gethostname() self.data = data self.client = connection self.reduction_log_stream = cStringIO.StringIO() self.admin_log_stream = cStringIO.StringIO() try: if 'data' in data: self.data_file = windows_to_linux_path( str(data['data']), MISC["temp_root_directory"]) logger.debug("data_file: %s" % self.data_file) else: raise ValueError("data is missing") if 'facility' in data: self.facility = str(data['facility']).upper() logger.debug("facility: %s" % self.facility) else: raise ValueError("facility is missing") if 'instrument' in data: self.instrument = str(data['instrument']).upper() logger.debug("instrument: %s" % self.instrument) else: raise ValueError("instrument is missing") if 'rb_number' in data: self.proposal = str(data['rb_number']).upper() logger.debug("rb_number: %s" % self.proposal) else: raise ValueError("rb_number is missing") if 'run_number' in data: self.run_number = str(int(data['run_number']) ) # Cast to int to remove trailing zeros logger.debug("run_number: %s" % str(self.run_number)) else: raise ValueError("run_number is missing") if 'reduction_script' in data: self.reduction_script = data['reduction_script'] logger.debug("reduction_script: %s ..." % self.reduction_script[:50]) else: raise ValueError("reduction_script is missing") if 'reduction_arguments' in data: self.reduction_arguments = data['reduction_arguments'] logger.debug("reduction_arguments: %s" % self.reduction_arguments) else: raise ValueError("reduction_arguments is missing") except ValueError: logger.info('JSON data error', exc_info=True) raise
def reduce(self): try: logger.debug("Calling: " + ACTIVEMQ['reduction_started'] + "\n" + prettify(self.data)) self.client.send(ACTIVEMQ['reduction_started'], json.dumps(self.data)) # Specify instrument directory cycle = re.match(r'.*cycle_(\d\d_\d).*', self.data_file.lower()).group(1) if self.instrument in (MISC["ceph_instruments"] + MISC["excitation_instruments"]): instrument_dir = MISC["ceph_directory"] % ( self.instrument, self.proposal, self.run_number) if self.instrument in MISC["excitation_instruments"]: # Excitations would like to remove the run number folder at the end instrument_dir = instrument_dir[:instrument_dir. rfind('/') + 1] else: instrument_dir = MISC["archive_directory"] % ( self.instrument, cycle, self.proposal, self.run_number) # Specify directories where autoreduction output will go reduce_result_dir = MISC["temp_root_directory"] + instrument_dir if self.instrument not in MISC["excitation_instruments"]: run_output_dir = os.path.join( MISC["temp_root_directory"], instrument_dir[:instrument_dir.rfind('/') + 1]) else: run_output_dir = reduce_result_dir if 'run_description' in self.data: logger.info("DESCRIPTION: " + self.data["run_description"]) log_dir = reduce_result_dir + "/reduction_log/" log_and_err_name = "RB" + self.proposal + "Run" + self.run_number script_out = os.path.join(log_dir, log_and_err_name + "Script.out") mantid_log = os.path.join(log_dir, log_and_err_name + "Mantid.log") # strip the temp path off the front of the temp directory to get the final archive directory final_result_dir = reduce_result_dir[ len(MISC["temp_root_directory"]):] final_log_dir = log_dir[len(MISC["temp_root_directory"]):] if 'overwrite' in self.data: if not self.data["overwrite"]: logger.info('Don\'t want to overwrite previous data') path_parts = final_result_dir.split('/') new_path = '/' for part in path_parts: if part != 'autoreduced' and part != '': new_path = new_path + part + '/' maximum = 0 for folder in os.listdir(new_path): if folder.startswith('autoreduced'): number = folder.replace('autoreduced', '') if number != '': number = int(number) + 1 if number > maximum: maximum = number else: maximum = 1 if maximum == 0: new_path = new_path + 'autoreduced' else: new_path = new_path + 'autoreduced' + str( maximum) + '/' final_result_dir = new_path final_log_dir = new_path + 'reduction_log/' logger.info('Final Result Directory = ' + final_result_dir) logger.info('Final Log Directory = ' + final_log_dir) # test for access to result paths try: should_be_writable = [ reduce_result_dir, log_dir, final_result_dir, final_log_dir ] should_be_readable = [self.data_file] # try to make directories which should exist for path in filter(lambda p: not os.path.isdir(p), should_be_writable): os.makedirs(path) does_not_exist = lambda path: not os.access(path, os.F_OK) not_readable = lambda path: not os.access(path, os.R_OK) not_writable = lambda path: not os.access(path, os.W_OK) # we want write access to these directories, plus the final output paths if len(filter(not_writable, should_be_writable)) > 0: fail_path = filter(not_writable, should_be_writable)[0] problem = "does not exist" if does_not_exist( fail_path) else "no write access" raise Exception("Couldn't write to %s - %s" % (fail_path, problem)) if len(filter(not_readable, should_be_readable)) > 0: fail_path = filter(not_readable, should_be_readable)[0] problem = "does not exist" if does_not_exist( fail_path) else "no read access" raise Exception("Couldn't read %s - %s" % (fail_path, problem)) except Exception as exp: # if we can't access now, we should abort the run, and tell the server that it should be # re-run at a later time self.data["message"] = "Permission error: %s" % exp # 6 hours self.data["retry_in"] = 6 * 60 * 60 logger.error(traceback.format_exc()) raise exp self.data['reduction_data'] = [] if "message" not in self.data: self.data["message"] = "" logger.info("----------------") logger.info("Reduction script: %s ..." % self.reduction_script[:50]) logger.info("Result dir: %s" % reduce_result_dir) logger.info("Run Output dir: %s" % run_output_dir) logger.info("Log dir: %s" % log_dir) logger.info("Out log: %s" % script_out) logger.info("Datafile: %s" % self.data_file) logger.info("----------------") logger.info("Reduction subprocess started.") logger.info(reduce_result_dir) out_directories = None try: with channels_redirected(script_out, mantid_log, self.reduction_log_stream): """ Load reduction script as a module. This works as long as reduce.py makes no assumption that it is in the same directory as reduce_vars, i.e., either it does not import it at all, or adds its location to os.path explicitly. """ # Append the Mantid path to the system path such that we can use Mantid to run the user's script sys.path.append(MISC["mantid_path"]) reduce_script_location = self._load_reduction_script( self.instrument) reduce_script = imp.load_source('reducescript', reduce_script_location) try: skip_numbers = reduce_script.SKIP_RUNS except: skip_numbers = [] pass if self.data['run_number'] not in skip_numbers: reduce_script = self.replace_variables(reduce_script) with timeout(MISC["script_timeout"]): out_directories = reduce_script.main( input_file=str(self.data_file), output_dir=str(reduce_result_dir)) else: self.data['message'] = "Run has been skipped in script" except Exception as exp: with open(script_out, "a") as f: f.writelines(str(exp) + "\n") f.write(traceback.format_exc()) self.copy_temp_directory(reduce_result_dir, final_result_dir) self.delete_temp_directory(reduce_result_dir) error_str = "Error in user reduction script: %s - %s" % ( type(exp).__name__, exp) logger.error(traceback.format_exc()) # Parent except block will discard exception type, so format the type as a string now raise Exception(error_str) logger.info("Reduction subprocess completed.") logger.info("Additional save directories: %s" % out_directories) self.copy_temp_directory(reduce_result_dir, final_result_dir) # If the reduce script specified some additional save directories, copy to there first if out_directories: if type(out_directories) is str: self.copy_temp_directory(reduce_result_dir, out_directories) elif type(out_directories) is list: for out_dir in out_directories: if type(out_dir) is str: self.copy_temp_directory(reduce_result_dir, out_dir) else: self.log_and_message( "Optional output directories of reduce.py must be strings: %s" % out_dir) else: self.log_and_message( "Optional output directories of reduce.py must be a string or list of stings: " "%s" % out_directories) # no longer a need for the temp directory used for temporary storing of reduction results self.delete_temp_directory(reduce_result_dir) except Exception as exp: logger.error(traceback.format_exc()) self.data["message"] = "REDUCTION Error: %s " % exp self.data['reduction_log'] = self.reduction_log_stream.getvalue() self.data["admin_log"] = self.admin_log_stream.getvalue() if self.data["message"] != "": # This means an error has been produced somewhere try: self._send_error_and_log() except Exception: logger.info("Failed to send to queue! - %s - %s" % (e, repr(e))) finally: logger.info("Reduction job failed") else: # reduction has successfully completed self.client.send(ACTIVEMQ['reduction_complete'], json.dumps(self.data)) print("\nCalling: " + ACTIVEMQ['reduction_complete'] + "\n" + prettify(self.data) + "\n") logger.info("Reduction job successfully complete")