def retrieve_experiment(exp_id, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve an experiment in the local database based on its id.
Can call a sync if missing if the flag is true.
:param exp_id: Id of the experiment to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The experiment found
"""
if not exp_id: raise Exception("Trying to retrieve an experiment without providing an experiment ID")
from uuid import UUID
if isinstance(exp_id,UUID): exp_id = str(exp_id)
# If we dont force the update -> look first in the DB
exp = DataStore.get_experiment(exp_id) or DataStore.get_most_recent_experiment(exp_id)
if exp:
# If we have an experiment and we want to force the update -> delete it
if not force_update:
return exp
else:
DataStore.delete_experiment(exp)
if not sync_if_missing:
raise Exception('Experiment %s not found in the local database and sync disabled.' % exp_id)
logger.info('Experiment with id %s not found in local database, trying sync.' % exp_id)
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
exp = COMPS_experiment_to_local_db(exp_id, endpoint, verbose)
if exp: return exp
raise Exception("Experiment '%s' could not be retrieved." % exp_id)
def hard_delete(self):
"""
Delete data for experiment and marks the server entity for deletion.
"""
# Mark experiment for deletion in COMPS.
COMPS_login(self.endpoint)
self.comps_experiment.delete()
# Delete in the DB
from simtools.DataAccess.DataStore import DataStore
DataStore.delete_experiment(self.experiment)
def hard_delete(self):
"""
Delete experiment and output data.
"""
# Delete local simulation data.
exp_path = self.experiment.get_path()
if os.path.exists(exp_path):
try:
shutil.rmtree(exp_path)
except Exception as e:
print("Could not delete path: {}\nReason: {}".format(exp_path,e))
# Delete in the DB
from simtools.DataAccess.DataStore import DataStore
DataStore.delete_experiment(self.experiment)
def retrieve_object(obj_id, obj_type, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve an object (Experiment, Simulation) in the local database based on its id.
Can call a sync if missing if the flag is true
:param obj_id: Id of the object to retrieve
:param obj_type: Type of the object to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The experiment found
#TODO: SHould also support suites
"""
typename = obj_type.__name__
if not obj_id: raise ValueError("Trying to retrieve an object (%s) without providing an ID" % typename)
# Try a hit in the DB
if obj_type == Experiment:
obj = DataStore.get_experiment(obj_id)
elif obj_type == Simulation:
obj = DataStore.get_simulation(obj_id)
if obj:
# If we have an experiment and we want to force the update -> delete it
if not force_update:
return obj
else:
if obj_type == Experiment:
DataStore.delete_experiment(obj_id)
elif obj_type == Simulation:
DataStore.delete_simulation(obj_id)
if not sync_if_missing:
raise Exception('%s %s not found in the local database and sync disabled.' % (typename, obj_id))
logger.info('%s with id %s not found in local database, trying sync.' % (typename, obj_id))
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
if obj_type == Experiment:
obj = COMPS_experiment_to_local_db(obj_id, endpoint, verbose)
elif obj_type == Simulation:
obj =None
if obj: return obj
raise Exception("%s '%s' could not be retrieved." % (typename, obj_id))
def sync(args, unknownArgs):
"""
Sync COMPS db with local db
"""
# Create a default HPC setup parser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
user = sp.get('user')
COMPS_login(endpoint)
exp_to_save = list()
exp_deleted = 0
# Retrieve all the experiment id from COMPS for the current user
exp_ids = get_experiment_ids_for_user(user)
# Test the experiments present in the local DB to make sure they still exist in COMPS
for exp in DataStore.get_experiments():
if exp.location == "HPC":
if exp.exp_id not in exp_ids:
# The experiment doesnt exist on COMPS anymore -> delete from local
DataStore.delete_experiment(exp)
exp_deleted += 1
# Consider experiment id option
exp_id = args.exp_id if args.exp_id else None
exp_name = args.exp_name if args.exp_name else None
user = args.user if args.user else user
if exp_name:
experiments = get_experiments_by_name(exp_name, user)
for experiment_data in experiments:
experiment = COMPS_experiment_to_local_db(
exp_id=str(experiment_data.id),
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
if experiment:
exp_to_save.append(experiment)
elif exp_id:
# Create a new experiment
experiment = COMPS_experiment_to_local_db(exp_id=exp_id,
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
else:
# By default only get experiments created in the last month
# day_limit = args.days if args.days else day_limit_default
day_limit = 30
today = datetime.date.today()
limit_date = today - datetime.timedelta(days=int(day_limit))
# For each of them, check if they are in the db
for exp in get_experiments_per_user_and_date(user, limit_date):
# Create a new experiment
experiment = COMPS_experiment_to_local_db(
exp_id=str(exp.id),
endpoint=endpoint,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
# Save the experiments if any
if len(exp_to_save) > 0 and exp_deleted == 0:
DataStore.batch_save_experiments(exp_to_save)
logger.info(
"The following experiments have been added to the database:")
logger.info("\n".join(["- " + str(exp) for exp in exp_to_save]))
logger.info("%s experiments have been updated in the DB." %
len(exp_to_save))
logger.info("%s experiments have been deleted from the DB." %
exp_deleted)
else:
print("The database was already up to date.")
# Start overseer
BaseExperimentManager.check_overseer()