def _construct_and_send_skipped(self, rb_number, reason, message: Message): """ Construct a message and send to the skipped reduction queue :param rb_number: The RB Number associated with the reduction job :param reason: The error that caused the run to be skipped """ self._logger.warning("Skipping non-integer RB number: %s", rb_number) msg = 'Reduction Skipped: {}. Assuming run number to be ' \ 'a calibration run.'.format(reason) message.message = msg skipped_queue = ACTIVEMQ_SETTINGS.reduction_skipped self._client.send_message(skipped_queue, message)
def send_message_onwards(self, reduction_run, message: Message, instrument): """ Sends the message onwards, either for processing, if validation is OK and instrument isn't paused or skips it if either of those is true. """ # activate instrument if script was found skip_reason = self.find_reason_to_skip_run(reduction_run, message, instrument) if skip_reason is not None: message.message = skip_reason self.reduction_skipped(reduction_run, message) else: self.activate_db_inst(instrument) self.do_reduction(reduction_run, message)
def main(): """ Main method. """ queue_client = QueueClient() try: logger.info("PostProcessAdmin Connecting to ActiveMQ") queue_client.connect() logger.info("PostProcessAdmin Successfully Connected to ActiveMQ") destination, data = sys.argv[1:3] # pylint: disable=unbalanced-tuple-unpacking message = Message() message.populate(data) logger.info("destination: %s", destination) logger.info("message: %s", message.serialize(limit_reduction_script=True)) try: post_proc = PostProcessAdmin(message, queue_client) log_stream_handler = logging.StreamHandler( post_proc.admin_log_stream) logger.addHandler(log_stream_handler) if destination == '/queue/ReductionPending': post_proc.reduce() except ValueError as exp: message.message = str( exp) # Note: I believe this should be .message logger.info("Message data error: %s", message.serialize(limit_reduction_script=True)) raise except Exception as exp: logger.info("PostProcessAdmin error: %s", str(exp)) raise finally: try: logger.removeHandler(log_stream_handler) except: pass except Exception as exp: logger.info("Something went wrong: %s", str(exp)) try: queue_client.send(ACTIVEMQ_SETTINGS.reduction_error, message) logger.info("Called %s ---- %s", ACTIVEMQ_SETTINGS.reduction_error, message.serialize(limit_reduction_script=True)) finally: sys.exit()
def main(): """ This is the entrypoint when a reduction is started. This is run in a subprocess from ReductionProcessManager, and the required parameters to perform the reduction are passed as process arguments. Additionally, the resulting Message is written to a temporary file which the parent process reads back to mark the result of the reduction run in the DB. """ data, temp_output_file = sys.argv[1], sys.argv[2] try: message = Message() message.populate(data) except ValueError as exp: logger.error("Could not populate message from data: %s", str(exp)) raise try: reduction = ReductionRunner(message) except Exception as exp: message.message = str(exp) logger.info("Message data error: %s", message.serialize(limit_reduction_script=True)) raise log_stream_handler = logging.StreamHandler(reduction.admin_log_stream) logger.addHandler(log_stream_handler) try: reduction.reduce() # write out the reduction message with open(temp_output_file, "w") as out_file: out_file.write(reduction.message.serialize()) except Exception as exp: logger.info("ReductionRunner error: %s", str(exp)) raise finally: logger.removeHandler(log_stream_handler)
def data_ready(self, message: Message): """ Called when destination queue was data_ready. Updates the reduction run in the database. """ self._logger.info("Data ready for processing run %s on %s", message.run_number, message.instrument) if not validate_rb_number(message.rb_number): # rb_number is invalid so send message to skip queue and early return message.message = f"Found non-integer RB number: {message.rb_number}" self._logger.warning("%s. Skipping %s%s.", message.message, message.instrument, message.run_number) message.rb_number = 0 run_no = str(message.run_number) instrument = self._get_and_activate_db_inst(message.instrument) status = self._utils.status.get_skipped() if instrument.is_paused \ else self._utils.status.get_queued() # This must be done before looking up the run version to make sure # the record exists experiment = db_access.get_experiment(message.rb_number, create=True) run_version = db_access.find_highest_run_version(run_number=run_no, experiment=experiment) run_version += 1 message.run_version = run_version # Get the script text for the current instrument. If the script text # is null then send to # error queue script_text = self._utils. \ instrument_variable.get_current_script_text(instrument.name)[0] if script_text is None: self.reduction_error(message) raise InvalidStateException( "Script text for current instrument is null") # Make the new reduction run with the information collected so far # and add it into the database reduction_run = db_records.create_reduction_run_record( experiment=experiment, instrument=instrument, message=message, run_version=run_version, script_text=script_text, status=status) db_access.save_record(reduction_run) # Create a new data location entry which has a foreign key linking # it to the current # reduction run. The file path itself will point to a datafile # (e.g. "\isis\inst$\NDXWISH\Instrument\data\cycle_17_1\WISH00038774 # .nxs") data_location = self._data_model.DataLocation( file_path=message.data, reduction_run_id=reduction_run.id) db_access.save_record(data_location) # We now need to create all of the variables for the run such that # the script can run # through in the desired way self._logger.info('Creating variables for run') variables = self._utils.instrument_variable.create_variables_for_run( reduction_run) if not variables: self._logger.warning( "No instrument variables found on %s for run %s", instrument.name, message.run_number) self._logger.info('Getting script and arguments') reduction_script, arguments = self._utils.reduction_run. \ get_script_and_arguments(reduction_run) message.reduction_script = reduction_script message.reduction_arguments = arguments # Make sure the RB number is valid try: message.validate("/queue/DataReady") except RuntimeError as validation_err: self._logger.error("Validation error from handler: %s", str(validation_err)) self._client.send_message('/queue/ReductionSkipped', message) return if instrument.is_paused: self._logger.info("Run %s has been skipped", message.run_number) else: self._client.send_message('/queue/ReductionPending', message) self._logger.info("Run %s ready for reduction", message.run_number)