def getReductionRun(with_variables=True):
instrument = InstrumentUtils().get_instrument("valid")
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(
instrument=instrument,
run_number=1,
experiment=experiment,
run_version=0,
status=StatusUtils().get_queued(),
script=getValidScript("reduce.py"),
)
reduction_run.save()
if with_variables:
variable = RunVariable(
reduction_run=reduction_run, name="test", value="testvalue1", type="text", is_advanced=False
)
variable.save()
reduction_run.run_variables.add(variable)
variable = RunVariable(
reduction_run=reduction_run, name="advanced_test", value="testvalue2", type="text", is_advanced=True
)
variable.save()
reduction_run.run_variables.add(variable)
reduction_run.save()
return reduction_run
def createRetryRun(self, reductionRun, script=None, variables=None, delay=0, username=None):
"""
Create a run ready for re-running based on the run provided.
If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils
# find the previous run version, so we don't create a duplicate
last_version = -1
for run in ReductionRun.objects.filter(experiment=reductionRun.experiment, run_number=reductionRun.run_number):
last_version = max(last_version, run.run_version)
try:
# get the script to use:
script_text = script if script is not None else reductionRun.script
# create the run object and save it
new_job = ReductionRun(
instrument=reductionRun.instrument,
run_number=reductionRun.run_number,
run_name="",
run_version=last_version + 1,
experiment=reductionRun.experiment,
started_by=username,
status=StatusUtils().get_queued(),
script=script_text,
)
new_job.save()
reductionRun.retry_run = new_job
reductionRun.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta(
seconds=delay if delay else 0
)
reductionRun.save()
# copy the previous data locations
for data_location in reductionRun.data_location.all():
new_data_location = DataLocation(file_path=data_location.file_path, reduction_run=new_job)
new_data_location.save()
new_job.data_location.add(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
var.save()
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job
except:
new_job.delete()
raise
def data_ready(self):
# Import within method to prevent cylindrical imports
from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils
logger.info("Data ready for processing run %s on %s" % (str(self._data_dict['run_number']), self._data_dict['instrument']))
instrument = InstrumentUtils().get_instrument(self._data_dict['instrument'])
# Activate the instrument if it is currently set to inactive
if not instrument.is_active:
instrument.is_active = True
instrument.save()
status = StatusUtils().get_skipped() if instrument.is_paused else StatusUtils().get_queued()
last_run = ReductionRun.objects.filter(run_number=self._data_dict['run_number']).order_by('-run_version').first()
highest_version = last_run.run_version if last_run is not None else -1
experiment, experiment_created = Experiment.objects.get_or_create(reference_number=self._data_dict['rb_number'])
if experiment_created:
experiment.save()
script_text = InstrumentVariablesUtils().get_current_script_text(instrument.name)[0]
run_version = highest_version+1
reduction_run = ReductionRun( run_number=self._data_dict['run_number']
, run_version=run_version
, experiment=experiment
, instrument=instrument
, status=status
, script=script_text
)
reduction_run.save()
self._data_dict['run_version'] = reduction_run.run_version
data_location = DataLocation(file_path=self._data_dict['data'], reduction_run=reduction_run)
data_location.save()
variables = InstrumentVariablesUtils().create_variables_for_run(reduction_run)
if not variables:
logger.warning("No instrument variables found on %s for run %s" % (instrument.name, self._data_dict['run_number']))
reduction_script, arguments = ReductionRunUtils().get_script_and_arguments(reduction_run)
self._data_dict['reduction_script'] = reduction_script
self._data_dict['reduction_arguments'] = arguments
if instrument.is_paused:
logger.info("Run %s has been skipped" % self._data_dict['run_number'])
else:
self._client.send('/queue/ReductionPending', json.dumps(self._data_dict), priority=self._priority)
logger.info("Run %s ready for reduction" % self._data_dict['run_number'])
def createReductionRun(self):
instrument = InstrumentUtils().get_instrument("valid")
instrument.save()
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(run_number=self.run_number,
instrument=instrument,
experiment=experiment,
run_version=1,
status=StatusUtils().get_queued(),
script=getValidScript('reduce.py'))
self.run_number += 1
reduction_run.save()
variables = InstrumentVariablesUtils().get_variables_for_run(
reduction_run)
VariableUtils().save_run_variables(variables, reduction_run)
return reduction_run
def createReductionRun(self):
instrument = InstrumentUtils().get_instrument("valid")
instrument.save()
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(
run_number=self.run_number,
instrument=instrument,
experiment=experiment,
run_version=1,
status=StatusUtils().get_queued(),
script=getValidScript("reduce.py"),
)
self.run_number += 1
reduction_run.save()
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
VariableUtils().save_run_variables(variables, reduction_run)
return reduction_run
def test_get_variables_for_run_default_variables(self):
instrument = InstrumentUtils().get_instrument("valid")
experiment = Experiment(reference_number=1)
reduction_run = ReductionRun(run_number=123, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, None, "Not expecting experiment_reference")
self.assertEqual(variables[0].start_run, 1, "Expecting start run to be 1 but was %s" % variables[0].start_run)
def test_get_variables_for_run_experiment_reference(self):
instrument = InstrumentUtils().get_instrument("valid")
variable = InstrumentVariable(
instrument=instrument,
name="test",
value="test",
is_advanced=False,
type="text",
experiment_reference=99999,
)
variable.save()
experiment = Experiment(reference_number=99999)
reduction_run = ReductionRun(run_number=1, instrument=instrument, experiment=experiment, run_version=1, status=StatusUtils().get_queued())
variables = InstrumentVariablesUtils().get_variables_for_run(reduction_run)
self.assertNotEqual(variables, None, "Expecting some variables to be returned")
self.assertNotEqual(variables, [], "Expecting some variables to be returned")
self.assertTrue(len(variables) > 0, 'Expecting at least 1 variable returned')
self.assertEqual(variables[0].experiment_reference, 99999, "Expecting instrument variables to match with experiment reference number")
def getReductionRun(with_variables=True):
instrument = InstrumentUtils().get_instrument('valid')
experiment = Experiment(reference_number=1)
experiment.save()
reduction_run = ReductionRun(instrument=instrument, run_number=1, experiment=experiment, run_version=0, status=StatusUtils().get_queued(), script=getValidScript('reduce.py'))
reduction_run.save()
if with_variables:
variable = RunVariable(reduction_run=reduction_run,name='test',value='testvalue1',type='text',is_advanced=False)
variable.save()
reduction_run.run_variables.add(variable)
variable = RunVariable(reduction_run=reduction_run,name='advanced_test',value='testvalue2',type='text',is_advanced=True)
variable.save()
reduction_run.run_variables.add(variable)
reduction_run.save()
return reduction_run
def createRetryRun(self,
reductionRun,
overwrite=None,
script=None,
variables=None,
delay=0,
username=None,
description=''):
"""
Create a run ready for re-running based on the run provided.
If variables (RunVariable) are provided, copy them and associate them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
from reduction_variables.utils import InstrumentVariablesUtils, VariableUtils
run_last_updated = reductionRun.last_updated
if username == 'super':
username = 1
# find the previous run version, so we don't create a duplicate
last_version = -1
for run in ReductionRun.objects.filter(
experiment=reductionRun.experiment,
run_number=reductionRun.run_number):
last_version = max(last_version, run.run_version)
try:
# get the script to use:
script_text = script if script is not None else reductionRun.script
# create the run object and save it
new_job = ReductionRun(instrument=reductionRun.instrument,
run_number=reductionRun.run_number,
run_name=description,
run_version=last_version + 1,
experiment=reductionRun.experiment,
started_by=username,
status=StatusUtils().get_queued(),
script=script_text,
overwrite=overwrite)
new_job.save()
reductionRun.retry_run = new_job
reductionRun.retry_when = timezone.now().replace(
microsecond=0) + datetime.timedelta(
seconds=delay if delay else 0)
reductionRun.save()
ReductionRun.objects.filter(id=reductionRun.id).update(
last_updated=run_last_updated)
# copy the previous data locations
for data_location in reductionRun.data_location.all():
new_data_location = DataLocation(
file_path=data_location.file_path, reduction_run=new_job)
new_data_location.save()
new_job.data_location.add(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
var.save()
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job
except Exception as e:
import traceback
logger.error(traceback.format_exc())
logger.error(e)
new_job.delete()
raise
def createRetryRun(user_id, reduction_run, overwrite=None, script=None,
variables=None, delay=0, description=''):
"""
Create a run ready for re-running based on the run provided.
If variables (RunVariable) are provided, copy them and associate
them with the new one, otherwise use the previous run's.
If a script (as a string) is supplied then use it, otherwise use the previous run's.
"""
from reduction_variables.utils import InstrumentVariablesUtils
run_last_updated = reduction_run.last_updated
# find the previous run version, so we don't create a duplicate
last_version = -1
# pylint:disable=no-member
previous_run = ReductionRun.objects.filter(experiment=reduction_run.experiment,
run_number=reduction_run.run_number) \
.order_by("-run_version").first()
last_version = previous_run.run_version
# get the script to use:
script_text = script if script is not None else reduction_run.script
# create the run object and save it
new_job = ReductionRun(instrument=reduction_run.instrument,
run_number=reduction_run.run_number,
run_name=description,
run_version=last_version + 1,
experiment=reduction_run.experiment,
started_by=user_id,
status=StatusUtils().get_queued(),
script=script_text,
overwrite=overwrite)
# Check record is safe to save
try:
new_job.full_clean()
# pylint:disable=catching-non-exception
except django.core.exceptions as exception:
LOGGER.error(traceback.format_exc())
LOGGER.error(exception)
raise
# Attempt to save
try:
new_job.save()
except ValueError as exception:
# This usually indicates a F.K. constraint mismatch. Maybe we didn't get a record in?
LOGGER.error(traceback.format_exc())
LOGGER.error(exception)
raise
reduction_run.retry_run = new_job
reduction_run.retry_when = timezone.now().replace(microsecond=0) + datetime.timedelta(
seconds=delay if delay else 0)
reduction_run.save()
# pylint:disable=no-member
ReductionRun.objects.filter(id=reduction_run.id).update(last_updated=run_last_updated)
# copy the previous data locations
# pylint:disable=no-member
for data_location in reduction_run.data_location.all():
new_data_location = DataLocation(file_path=data_location.file_path,
reduction_run=new_job)
new_data_location.save()
new_job.data_location.add(new_data_location)
if variables is not None:
# associate the variables with the new run
for var in variables:
var.reduction_run = new_job
var.save()
else:
# provide variables if they aren't already
InstrumentVariablesUtils().create_variables_for_run(new_job)
return new_job
