def test_convert_variable_to_type(self):
"""
Test: database variables types are successfully recognised and converted into python
single variable types
When: calling convert_variable_to_type with valid arguments
"""
self.assertIsInstance(vu.convert_variable_to_type('text', 'text'), str)
self.assertIsInstance(vu.convert_variable_to_type('1', 'number'), int)
self.assertIsInstance(vu.convert_variable_to_type('1.0', 'number'), float)
self.assertIsInstance(vu.convert_variable_to_type('True', 'boolean'), bool)
self.assertIsInstance(vu.convert_variable_to_type('False', 'boolean'), bool)
def test_convert_variable_to_type_list_types(self):
"""
Test database variables types are successfully recognised and converted into python
for list types
"""
str_list = vu.convert_variable_to_type('[\'s\',\'t\'', 'list_text')
self.assertIsInstance(str_list, list)
self.assertIsInstance(str_list[0], str)
int_list = vu.convert_variable_to_type('1,2', 'list_number')
self.assertIsInstance(int_list, list)
self.assertIsInstance(int_list[0], int)
float_list = vu.convert_variable_to_type('1.0,2.0', 'list_number')
self.assertIsInstance(float_list, list)
self.assertIsInstance(float_list[0], float)
def render_run_variables(request, instrument_name, run_number, run_version=0):
"""
Handles request to view the summary of a run
"""
# pylint:disable=no-member
reduction_run = ReductionRun.objects.get(instrument__name=instrument_name,
run_number=run_number,
run_version=run_version)
vars_kwargs = ReductionRunUtils.make_kwargs_from_runvariables(
reduction_run)
standard_vars = vars_kwargs["standard_vars"]
advanced_vars = vars_kwargs["advanced_vars"]
try:
current_variables = VariableUtils.get_default_variables(
instrument_name)
current_standard_variables = current_variables["standard_vars"]
current_advanced_variables = current_variables["advanced_vars"]
except (FileNotFoundError, ImportError, SyntaxError):
current_standard_variables = {}
current_advanced_variables = {}
context_dictionary = {
'run_number': run_number,
'run_version': run_version,
'standard_variables': standard_vars,
'advanced_variables': advanced_vars,
'current_standard_variables': current_standard_variables,
'current_advanced_variables': current_advanced_variables,
'instrument': reduction_run.instrument,
}
return render(request, 'snippets/run_variables.html', context_dictionary)
def set_with_correct_type(message_reduction_arguments: dict, reduction_args: dict, dict_name: str):
"""
Set the value of the variable with the correct type.
It retrieves the type string of the value in reduce_vars.py (the reduction_args param),
and converts the value to that type.
This is done to enforce type consistency between reduce_vars.py and message_reduction_arguments.
message_reduction_arguments can contain bad types when it gets sent from the web app - some
values are incorrectly strings and they create extra duplicates.
"""
if dict_name in message_reduction_arguments and dict_name in reduction_args:
for name, value in message_reduction_arguments[dict_name].items():
real_type = VariableUtils.get_type_string(reduction_args[dict_name][name])
reduction_args[dict_name][name] = VariableUtils.convert_variable_to_type(value, real_type)
def test_get_type_string(self):
"""
Test: Python types are successfully recognised and converted to database input
When: Calling get_type_string
"""
self.assertEqual(vu.get_type_string('a string'), 'text')
self.assertEqual(vu.get_type_string(1), 'number')
self.assertEqual(vu.get_type_string(1.0), 'number')
self.assertEqual(vu.get_type_string(True), 'boolean')
self.assertEqual(vu.get_type_string([1, 2, 3]), 'list_number')
self.assertEqual(vu.get_type_string(['s', 't', 'r']), 'list_text')
def current_default_variables(request, instrument=None):
"""
Handles request to view default variables
"""
try:
current_variables = VariableUtils.get_default_variables(instrument)
except (FileNotFoundError, ImportError, SyntaxError) as err:
return {"message": str(err)}
standard_vars = current_variables["standard_vars"]
advanced_vars = current_variables["advanced_vars"]
context_dictionary = {
'instrument': instrument,
'standard_variables': standard_vars,
'advanced_variables': advanced_vars,
}
return context_dictionary
def create_run_variables(self, reduction_run, message_reduction_arguments: dict = None) -> List:
"""
Finds the appropriate InstrumentVariables for the given reduction run, and creates
RunVariables from them.
If the run is a re-run, use the previous run's variables.
If instrument variables set for the run's experiment are found, they're used.
Otherwise if variables set for the run's run number exist, they'll be used.
If not, the instrument's default variables will be used.
:param reduction_run: The reduction run object
:param message_reduction_arguments: Preset arguments that will override whatever is in the
reduce_vars.py script. They are passed in from the web app
and have been acquired via direct user input
"""
if message_reduction_arguments is None:
message_reduction_arguments = {}
instrument_name = reduction_run.instrument.name
model = self.model.variable_model
experiment_reference = reduction_run.experiment.reference_number
run_number = reduction_run.run_number
instrument_id = reduction_run.instrument.id
possible_variables = model.InstrumentVariable.objects.filter(Q(experiment_reference=experiment_reference)
| Q(start_run__lte=run_number),
instrument_id=instrument_id)
reduce_vars = ReductionScript(instrument_name, 'reduce_vars.py')
reduce_vars_module = reduce_vars.load()
final_reduction_arguments = self.merge_arguments(message_reduction_arguments, reduce_vars_module)
variables = self.find_or_make_variables(possible_variables,
instrument_id,
final_reduction_arguments,
run_number=run_number)
logging.info('Creating RunVariables')
# Create run variables from these instrument variables, and return them.
return VariableUtils.save_run_variables(variables, reduction_run)
def get_script_arguments(run_variables):
"""
Converts the RunVariables that have been created into Python kwargs which can
be passed as the script parameters at runtime.
"""
standard_vars, advanced_vars = {}, {}
for run_variable in run_variables:
variable = run_variable.variable
value = VariableUtils.convert_variable_to_type(
variable.value, variable.type)
if variable.is_advanced:
advanced_vars[variable.name] = value
else:
standard_vars[variable.name] = value
arguments = {
'standard_vars': standard_vars,
'advanced_vars': advanced_vars
}
return arguments
def test_convert_variable_unknown_type(self):
"""
Test output variable type are unchanged if the target type is unrecognised
"""
self.assertIsInstance(vu.convert_variable_to_type('value', 'unknown'), str)
self.assertIsInstance(vu.convert_variable_to_type(1, 'unknown'), int)
def test_get_type_string_unknown_type(self):
"""
Test: A value of unknown type is output as database type text
When: Calling get_type_string
"""
self.assertEqual(vu.get_type_string({'key': 'value'}), 'text')
def test_convert_variable_mismatch_type(self):
"""
Test: number type conversion with non number
"""
self.assertIsNone(vu.convert_variable_to_type('string', 'number'))
def runs_list(request, instrument=None):
"""
Render instrument summary
"""
try:
filter_by = request.GET.get('filter', 'run')
instrument_obj = Instrument.objects.get(name=instrument)
except Instrument.DoesNotExist:
return {'message': "Instrument not found."}
try:
sort_by = request.GET.get('sort', 'run')
if sort_by == 'run':
runs = (ReductionRun.objects.only(
'status', 'last_updated', 'run_number', 'run_version',
'run_name').select_related('status').filter(
instrument=instrument_obj).order_by(
'-run_number', 'run_version'))
else:
runs = (ReductionRun.objects.only(
'status', 'last_updated', 'run_number', 'run_version',
'run_name').select_related('status').filter(
instrument=instrument_obj).order_by('-last_updated'))
if len(runs) == 0:
return {'message': "No runs found for instrument."}
try:
current_variables = VariableUtils.get_default_variables(
instrument_obj.name)
error_reason = ""
except FileNotFoundError:
current_variables = {}
error_reason = "reduce_vars.py is missing for this instrument"
except (ImportError, SyntaxError):
current_variables = {}
error_reason = "reduce_vars.py has an import or syntax error"
has_variables = bool(current_variables)
context_dictionary = {
'instrument': instrument_obj,
'instrument_name': instrument_obj.name,
'runs': runs,
'last_instrument_run': runs[0],
'processing': runs.filter(status=STATUS.get_processing()),
'queued': runs.filter(status=STATUS.get_queued()),
'filtering': filter_by,
'sort': sort_by,
'has_variables': has_variables,
'error_reason': error_reason
}
if filter_by == 'experiment':
experiments_and_runs = {}
experiments = Experiment.objects.filter(reduction_runs__instrument=instrument_obj). \
order_by('-reference_number').distinct()
for experiment in experiments:
associated_runs = runs.filter(experiment=experiment). \
order_by('-created')
experiments_and_runs[experiment] = associated_runs
context_dictionary['experiments'] = experiments_and_runs
else:
max_items_per_page = request.GET.get('pagination', 10)
custom_paginator = CustomPaginator(
page_type=sort_by,
query_set=runs,
items_per_page=max_items_per_page,
page_tolerance=3,
current_page=request.GET.get('page', 1))
context_dictionary['paginator'] = custom_paginator
context_dictionary['last_page_index'] = len(
custom_paginator.page_list)
context_dictionary['max_items'] = max_items_per_page
# pylint:disable=broad-except
except Exception as exception:
LOGGER.error(exception)
return {
'message':
"An unexpected error has occurred when loading the instrument."
}
return context_dictionary
def run_summary(_, instrument_name=None, run_number=None, run_version=0):
"""
Render run summary
"""
# pylint:disable=broad-except
try:
history = ReductionRun.objects.filter(
instrument__name=instrument_name, run_number=run_number).order_by(
'-run_version').select_related('status').select_related(
'experiment').select_related('instrument')
if len(history) == 0:
raise ValueError(
f"Could not find any matching runs for instrument {instrument_name} run {run_number}"
)
run = next(run for run in history
if run.run_version == int(run_version))
started_by = started_by_id_to_name(run.started_by)
# run status value of "s" means the run is skipped
is_skipped = run.status.value == "s"
is_rerun = len(history) > 1
location_list = run.reduction_location.all()
reduction_location = None
if location_list:
reduction_location = location_list[0].file_path
if reduction_location and '\\' in reduction_location:
reduction_location = reduction_location.replace('\\', '/')
rb_number = run.experiment.reference_number
try:
current_variables = VariableUtils.get_default_variables(
run.instrument.name)
except (FileNotFoundError, ImportError, SyntaxError):
current_variables = {}
has_reduce_vars = bool(current_variables)
has_run_variables = bool(run.run_variables.count())
context_dictionary = {
'run': run,
'run_number': run_number,
'instrument': instrument_name,
'run_version': run_version,
'is_skipped': is_skipped,
'is_rerun': is_rerun,
'history': history,
'reduction_location': reduction_location,
'started_by': started_by,
'has_reduce_vars': has_reduce_vars,
'has_run_variables': has_run_variables
}
except PermissionDenied:
raise
except Exception as exception:
# Error that we cannot recover from - something wrong with instrument, run, or experiment
LOGGER.error(exception)
return {"message": str(exception)}
if reduction_location:
try:
plot_handler = PlotHandler(
data_filepath=run.data_location.first().file_path,
server_dir=reduction_location,
rb_number=rb_number)
plot_locations = plot_handler.get_plot_file()
if plot_locations:
context_dictionary['plot_locations'] = plot_locations
except Exception as exception:
# Lack of plot images is recoverable - we shouldn't stop the whole page rendering
# if something is wrong with the plot images - but display an error message
err_msg = "Encountered error while retrieving plots for this run"
LOGGER.error("%s. Instrument: %s, run %s. RB Number %s Error: %s",
err_msg, run.instrument.name, run, rb_number,
exception)
context_dictionary["plot_error_message"] = f"{err_msg}."
return context_dictionary
def find_or_make_variables(possible_variables: QuerySet,
instrument_id: int,
reduction_arguments: dict,
run_number: Optional[int] = None,
experiment_reference: Optional[int] = None,
from_webapp=False) -> List:
"""
Find appropriate variables from the possible_variables QuerySet, or make the necessary variables
:param possible_variables: A queryset holding the possible variables to be re-used
:param instrument_id: ID of the instrument object
:param reduction_arguments: A dictionary holding all required reduction arguments
:param run_number: Optional, the run number from which these variables will be active
:param expriment_reference: Optional, the experiment number for which these variables WILL ALWAYS be used.
Variables set for experiment are treated as top-priority.
They can only be changed or deleted from the web app.
They will not be affected by variables for a run range or changing
the values in reduce_vars.
:param from_webapp: If the call is made from the web app we want to ignore the variable's own tracks_script flag
and ALWAYS overwrite the value of the variable with what has been passed in
"""
all_vars: List[Tuple[str, Any, bool]] = [(name, value, False)
for name, value in reduction_arguments["standard_vars"].items()]
all_vars.extend([(name, value, True) for name, value in reduction_arguments["advanced_vars"].items()])
if len(all_vars) == 0:
return []
variables = []
for name, value, is_advanced in all_vars:
script_help_text = InstrumentVariablesUtils.get_help_text(
'standard_vars' if not is_advanced else 'advanced_vars', name, reduction_arguments)
new_value = str(value).replace('[', '').replace(']', '')
new_type = VariableUtils.get_type_string(value)
# Try to find a suitable variable to re-use from the ones that already exist
variable = InstrumentVariablesUtils.find_appropriate_variable(possible_variables, name,
experiment_reference)
# if no suitable variable is found - create a new one
if variable is None:
var_kwargs = {
'name': name,
'value': new_value,
'type': new_type,
'help_text': script_help_text,
'is_advanced': is_advanced,
'instrument_id': instrument_id
}
variable = possible_variables.create(**var_kwargs)
# if the variable was just created then set it to track the script
# and that it starts on the current run
# if it was found already existing just leave it as it is
if experiment_reference:
variable.experiment_reference = experiment_reference
else:
variable.start_run = run_number
variable.tracks_script = not from_webapp
variable.save()
else:
InstrumentVariablesUtils.update_if_necessary(variable, experiment_reference, run_number, new_value,
new_type, script_help_text, from_webapp)
variables.append(variable)
return variables