def open_icat(self):
""" Try to open an ICAT session, if we don't have one already. """
try:
if self.icat is None:
self.icat = ICATCommunication(**self.kwargs)
except Exception as e:
logger.error("Failed to connect to ICAT: %s - %s" %
(type(e).__name, e))
raise ICATConnectionException()
def get_current_and_upcoming_variables(self, instrument_name):
"""
Fetches the instrument variables for:
- The next run number
- Upcoming run numbers
- Upcoming known experiments
as a tuple of (current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment)
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
completed_status = StatusUtils().get_completed()
# First, we find the latest run number to determine what's upcoming.
try:
latest_completed_run_number = ReductionRun.objects.filter(
instrument=instrument, run_version=0,
status=completed_status).order_by(
'-run_number').first().run_number
except AttributeError:
latest_completed_run_number = 1
# Then we find all the upcoming runs and force updating of all subsequent variables.
upcoming_run_variables = InstrumentVariable.objects.filter(
instrument=instrument,
start_run__isnull=False,
start_run__gt=latest_completed_run_number +
1).order_by('start_run')
upcoming_run_numbers = set(
[var.start_run for var in upcoming_run_variables])
[
self.show_variables_for_run(instrument_name, run_number)
for run_number in upcoming_run_numbers
]
# Get the most recent run variables.
current_variables = self.show_variables_for_run(
instrument_name, latest_completed_run_number)
if not current_variables:
# If no variables are saved, we'll use the default ones, and set them while we're at it.
current_variables = self.get_default_variables(instrument_name)
self.set_variables_for_runs(instrument_name, current_variables)
# And then select the variables for all subsequent run numbers; collect the immediate upcoming variables and all subsequent sets.
upcoming_variables_by_run = self.show_variables_for_run(
instrument_name, latest_completed_run_number + 1)
upcoming_variables_by_run += list(
InstrumentVariable.objects.filter(
instrument=instrument,
start_run__in=upcoming_run_numbers).order_by('start_run'))
# Get the upcoming experiments, and then select all variables for these experiments.
upcoming_experiments = []
with ICATCommunication() as icat:
upcoming_experiments = list(
icat.get_upcoming_experiments_for_instrument(instrument_name))
upcoming_variables_by_experiment = InstrumentVariable.objects.filter(
instrument=instrument,
experiment_reference__in=upcoming_experiments).order_by(
'experiment_reference')
return current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment
def open_icat(self):
""" Try to open an ICAT session, if we don't have one already. """
try:
if self.icat is None:
self.icat = ICATCommunication(**self.kwargs)
except Exception as e:
logger.error("Failed to connect to ICAT: %s - %s" % (type(e).__name, e))
raise ICATConnectionException()
def get_current_and_upcoming_variables(self,
instrument_name,
last_run_object=None):
"""
:param instrument_name: The name of the instrument
:param last_run_object: Optionally provide an object of the last run on the instrument
Fetches the instrument variables for:
- The next run number
- Upcoming run numbers
- Upcoming known experiments
as a tuple of
(current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment)
"""
instrument = InstrumentUtils().get_instrument(instrument_name)
completed_status = StatusUtils().get_completed()
# First, we find the latest run number to determine what's upcoming.
try:
if last_run_object and last_run_object.status.value_verbose(
) == 'Completed':
latest_completed_run_number = last_run_object.run_number
else:
latest_completed_run_number = ReductionRun.objects.filter(instrument=instrument,
run_version=0,
status=completed_status)\
.order_by('-run_number').first().run_number
except AttributeError:
latest_completed_run_number = 1
# Then we find all the upcoming runs and force updating of all subsequent variables.
# pylint:disable=no-member
upcoming_run_variables = InstrumentVariable.objects.\
filter(instrument=instrument,
start_run__isnull=False,
start_run__gt=latest_completed_run_number + 1).order_by('start_run')
upcoming_run_numbers = set(
[var.start_run for var in upcoming_run_variables])
# pylint:disable=expression-not-assigned
[
self.show_variables_for_run(instrument_name, run_number)
for run_number in upcoming_run_numbers
]
# Get the most recent run variables.
current_variables = self.show_variables_for_run(
instrument_name, latest_completed_run_number)
if not current_variables:
# If no variables are saved, we'll use the default ones, and set them while we're at it.
current_variables = self.get_default_variables(instrument_name)
self.set_variables_for_runs(instrument_name, current_variables)
# And then select the variables for all subsequent run numbers;
# collect the immediate upcoming variables and all subsequent sets.
upcoming_variables_by_run = self.show_variables_for_run(
instrument_name, latest_completed_run_number + 1)
# pylint:disable=no-member
upcoming_variables_by_run += list(
InstrumentVariable.objects.filter(
instrument=instrument,
start_run__in=upcoming_run_numbers).order_by('start_run'))
# Get the upcoming experiments, and then select all variables for these experiments.
upcoming_experiments = []
with ICATCommunication() as icat:
upcoming_experiments = list(
icat.get_upcoming_experiments_for_instrument(instrument_name))
# pylint:disable=line-too-long,no-member
upcoming_variables_by_experiment = InstrumentVariable.objects.\
filter(instrument=instrument,
experiment_reference__in=upcoming_experiments).order_by('experiment_reference')
return current_variables, upcoming_variables_by_run, upcoming_variables_by_experiment
class ICATCache(object):
"""
A wrapper for ICATCommunication that caches information, and in the case of ICAT failure will try to use this cache.
It stores Cache models in the database, and will get all fields from ICATCommunication if a model is requested but has expired or doesn't exist.
Most of the methods it wraps from ICATCommunication are templated rather than declared explicitly; see below.
"""
def __init__(self, **kwargs):
self.kwargs = kwargs
self.icat = None
self.cache_lifetime = CACHE_LIFETIME
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if self.icat is not None:
self.icat.__exit__(type, value, traceback)
def open_icat(self):
""" Try to open an ICAT session, if we don't have one already. """
try:
if self.icat is None:
self.icat = ICATCommunication(**self.kwargs)
except Exception as e:
logger.error("Failed to connect to ICAT: %s - %s" % (type(e).__name, e))
raise ICATConnectionException()
def is_valid(self, cache_obj):
""" Check whether a cache object is fresh and is not None. """
return cache_obj and cache_obj.created + datetime.timedelta(seconds=self.cache_lifetime) > timezone.now()
def to_list(self, l):
return ",".join(map(str, l))
def cull_invalid(self, list):
""" Removes all objects in the list that have expired. """
rlist = []
for obj in list:
obj.delete() if not self.is_valid(obj) else rlist.append(obj)
return rlist
def update_cache(self, obj_type, obj_id):
"""
Adds an object of type obj_type and id obj_id to the cache - querying ICAT - and returns the object.
E.g., obj_type = InstrumentCache, obj_id = "WISH".
"""
self.open_icat() # Open an ICAT session if we don't have one open.
if obj_type != ExperimentCache:
# Check func_list for the attributes that each model should have, and the corresponding ICATCommunication function to query for it; call it for each, building a dict, and then splice it into the constructor kwargs.
new_obj = obj_type(**{attr:(getattr(self.icat, func)(obj_id) if typ is None else self.to_list(getattr(self.icat, func)(obj_id))) for (func, model, attr, typ) in func_list if model == obj_type})
else:
# In this case, ICATCommunication returns all the ExperimentCache fields in one query, so we splice that into the constructor.
new_obj = obj_type(**{attr:str(val) for attr,val in self.icat.get_experiment_details(obj_id).iteritems() if attr is not "reference_number"})
new_obj.id_name = obj_id
new_obj.save()
return new_obj
def check_cache(self, obj_type, obj_id):
"""
Checks the cache for an object of type obj_type and id obj_id - querying for a new one if there isn't a fresh copy - and returns it.
If ICAT is unavailable, use a local copy if it exists. If we can't use anything, return None.
"""
in_cache = obj_type.objects.filter(id_name = obj_id).order_by("-created")
ret_obj = None
if in_cache:
ret_obj = in_cache[0]
if not self.is_valid(ret_obj):
try:
ret_obj = self.update_cache(obj_type, obj_id)
self.cull_invalid(in_cache)
except ICATConnectionException:
pass
return ret_obj
def get_valid_experiments_for_instruments(self, user_number, instruments):
experiment_dict = {}
user_experiments = set(self.get_associated_experiments(user_number))
for instrument_name in instruments:
instrument_experiments = self.get_valid_experiments_for_instrument(instrument_name)
experiment_dict[instrument_name] = list(user_experiments.intersection(instrument_experiments))
return experiment_dict
def get_experiment_details(self, experiment_number):
experiment = self.check_cache(ExperimentCache, experiment_number)
return experiment.__dict__
class ICATCache(object):
"""
A wrapper for ICATCommunication that caches information, and in the case of ICAT failure will try to use this cache.
It stores Cache models in the database, and will get all fields from ICATCommunication if a model is requested but has expired or doesn't exist.
Most of the methods it wraps from ICATCommunication are templated rather than declared explicitly; see below.
"""
def __init__(self, **kwargs):
self.kwargs = kwargs
self.icat = None
self.cache_lifetime = CACHE_LIFETIME
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if self.icat is not None:
self.icat.__exit__(type, value, traceback)
def open_icat(self):
""" Try to open an ICAT session, if we don't have one already. """
try:
if self.icat is None:
self.icat = ICATCommunication(**self.kwargs)
except Exception as e:
logger.error("Failed to connect to ICAT: %s - %s" %
(type(e).__name, e))
raise ICATConnectionException()
def is_valid(self, cache_obj):
""" Check whether a cache object is fresh and is not None. """
return cache_obj and cache_obj.created + datetime.timedelta(
seconds=self.cache_lifetime) > timezone.now()
def to_list(self, l):
return ",".join(map(str, l))
def cull_invalid(self, list):
""" Removes all objects in the list that have expired. """
rlist = []
for obj in list:
obj.delete() if not self.is_valid(obj) else rlist.append(obj)
return rlist
def update_cache(self, obj_type, obj_id):
"""
Adds an object of type obj_type and id obj_id to the cache - querying ICAT - and returns the object.
E.g., obj_type = InstrumentCache, obj_id = "WISH".
"""
self.open_icat() # Open an ICAT session if we don't have one open.
if obj_type != ExperimentCache:
# Check func_list for the attributes that each model should have, and the corresponding ICATCommunication function to query for it; call it for each, building a dict, and then splice it into the constructor kwargs.
new_obj = obj_type(
**{
attr: (getattr(self.icat, func)(obj_id) if typ is None else
self.to_list(getattr(self.icat, func)(obj_id)))
for (func, model, attr, typ) in func_list
if model == obj_type
})
else:
# In this case, ICATCommunication returns all the ExperimentCache fields in one query, so we splice that into the constructor.
new_obj = obj_type(
**{
attr: str(val)
for attr, val in self.icat.get_experiment_details(
obj_id).iteritems() if attr is not "reference_number"
})
new_obj.id_name = obj_id
new_obj.save()
return new_obj
def check_cache(self, obj_type, obj_id):
"""
Checks the cache for an object of type obj_type and id obj_id - querying for a new one if there isn't a fresh copy - and returns it.
If ICAT is unavailable, use a local copy if it exists. If we can't use anything, return None.
"""
in_cache = obj_type.objects.filter(id_name=obj_id).order_by("-created")
ret_obj = None
if in_cache:
ret_obj = in_cache[0]
if not self.is_valid(ret_obj):
try:
ret_obj = self.update_cache(obj_type, obj_id)
self.cull_invalid(in_cache)
except ICATConnectionException:
pass
return ret_obj
def get_valid_experiments_for_instruments(self, user_number, instruments):
experiment_dict = {}
user_experiments = set(self.get_associated_experiments(user_number))
for instrument_name in instruments:
instrument_experiments = self.get_valid_experiments_for_instrument(
instrument_name)
experiment_dict[instrument_name] = list(
user_experiments.intersection(instrument_experiments))
return experiment_dict
def get_experiment_details(self, experiment_number):
experiment = self.check_cache(ExperimentCache, experiment_number)
return experiment.__dict__
