def _construct_and_send_skipped(self, rb_number, reason, message: Message):
"""
Construct a message and send to the skipped reduction queue
:param rb_number: The RB Number associated with the reduction job
:param reason: The error that caused the run to be skipped
"""
self._logger.warning("Skipping non-integer RB number: %s", rb_number)
msg = 'Reduction Skipped: {}. Assuming run number to be ' \
'a calibration run.'.format(reason)
message.message = msg
skipped_queue = ACTIVEMQ_SETTINGS.reduction_skipped
self._client.send_message(skipped_queue, message)
def send_message_onwards(self, reduction_run, message: Message,
instrument):
"""
Sends the message onwards, either for processing, if validation is OK and instrument isn't paused
or skips it if either of those is true.
"""
# activate instrument if script was found
skip_reason = self.find_reason_to_skip_run(reduction_run, message,
instrument)
if skip_reason is not None:
message.message = skip_reason
self.reduction_skipped(reduction_run, message)
else:
self.activate_db_inst(instrument)
self.do_reduction(reduction_run, message)
def main():
""" Main method. """
queue_client = QueueClient()
try:
logger.info("PostProcessAdmin Connecting to ActiveMQ")
queue_client.connect()
logger.info("PostProcessAdmin Successfully Connected to ActiveMQ")
destination, data = sys.argv[1:3] # pylint: disable=unbalanced-tuple-unpacking
message = Message()
message.populate(data)
logger.info("destination: %s", destination)
logger.info("message: %s",
message.serialize(limit_reduction_script=True))
try:
post_proc = PostProcessAdmin(message, queue_client)
log_stream_handler = logging.StreamHandler(
post_proc.admin_log_stream)
logger.addHandler(log_stream_handler)
if destination == '/queue/ReductionPending':
post_proc.reduce()
except ValueError as exp:
message.message = str(
exp) # Note: I believe this should be .message
logger.info("Message data error: %s",
message.serialize(limit_reduction_script=True))
raise
except Exception as exp:
logger.info("PostProcessAdmin error: %s", str(exp))
raise
finally:
try:
logger.removeHandler(log_stream_handler)
except:
pass
except Exception as exp:
logger.info("Something went wrong: %s", str(exp))
try:
queue_client.send(ACTIVEMQ_SETTINGS.reduction_error, message)
logger.info("Called %s ---- %s", ACTIVEMQ_SETTINGS.reduction_error,
message.serialize(limit_reduction_script=True))
finally:
sys.exit()
def main():
"""
This is the entrypoint when a reduction is started. This is run in a subprocess from
ReductionProcessManager, and the required parameters to perform the reduction are passed
as process arguments.
Additionally, the resulting Message is written to a temporary file which the
parent process reads back to mark the result of the reduction run in the DB.
"""
data, temp_output_file = sys.argv[1], sys.argv[2]
try:
message = Message()
message.populate(data)
except ValueError as exp:
logger.error("Could not populate message from data: %s", str(exp))
raise
try:
reduction = ReductionRunner(message)
except Exception as exp:
message.message = str(exp)
logger.info("Message data error: %s",
message.serialize(limit_reduction_script=True))
raise
log_stream_handler = logging.StreamHandler(reduction.admin_log_stream)
logger.addHandler(log_stream_handler)
try:
reduction.reduce()
# write out the reduction message
with open(temp_output_file, "w") as out_file:
out_file.write(reduction.message.serialize())
except Exception as exp:
logger.info("ReductionRunner error: %s", str(exp))
raise
finally:
logger.removeHandler(log_stream_handler)
def data_ready(self, message: Message):
"""
Called when destination queue was data_ready.
Updates the reduction run in the database.
"""
self._logger.info("Data ready for processing run %s on %s",
message.run_number, message.instrument)
if not validate_rb_number(message.rb_number):
# rb_number is invalid so send message to skip queue and early return
message.message = f"Found non-integer RB number: {message.rb_number}"
self._logger.warning("%s. Skipping %s%s.", message.message,
message.instrument, message.run_number)
message.rb_number = 0
run_no = str(message.run_number)
instrument = self._get_and_activate_db_inst(message.instrument)
status = self._utils.status.get_skipped() if instrument.is_paused \
else self._utils.status.get_queued()
# This must be done before looking up the run version to make sure
# the record exists
experiment = db_access.get_experiment(message.rb_number, create=True)
run_version = db_access.find_highest_run_version(run_number=run_no,
experiment=experiment)
run_version += 1
message.run_version = run_version
# Get the script text for the current instrument. If the script text
# is null then send to
# error queue
script_text = self._utils. \
instrument_variable.get_current_script_text(instrument.name)[0]
if script_text is None:
self.reduction_error(message)
raise InvalidStateException(
"Script text for current instrument is null")
# Make the new reduction run with the information collected so far
# and add it into the database
reduction_run = db_records.create_reduction_run_record(
experiment=experiment,
instrument=instrument,
message=message,
run_version=run_version,
script_text=script_text,
status=status)
db_access.save_record(reduction_run)
# Create a new data location entry which has a foreign key linking
# it to the current
# reduction run. The file path itself will point to a datafile
# (e.g. "\isis\inst$\NDXWISH\Instrument\data\cycle_17_1\WISH00038774
# .nxs")
data_location = self._data_model.DataLocation(
file_path=message.data, reduction_run_id=reduction_run.id)
db_access.save_record(data_location)
# We now need to create all of the variables for the run such that
# the script can run
# through in the desired way
self._logger.info('Creating variables for run')
variables = self._utils.instrument_variable.create_variables_for_run(
reduction_run)
if not variables:
self._logger.warning(
"No instrument variables found on %s for run %s",
instrument.name, message.run_number)
self._logger.info('Getting script and arguments')
reduction_script, arguments = self._utils.reduction_run. \
get_script_and_arguments(reduction_run)
message.reduction_script = reduction_script
message.reduction_arguments = arguments
# Make sure the RB number is valid
try:
message.validate("/queue/DataReady")
except RuntimeError as validation_err:
self._logger.error("Validation error from handler: %s",
str(validation_err))
self._client.send_message('/queue/ReductionSkipped', message)
return
if instrument.is_paused:
self._logger.info("Run %s has been skipped", message.run_number)
else:
self._client.send_message('/queue/ReductionPending', message)
self._logger.info("Run %s ready for reduction", message.run_number)
