def process_batch(self):
self.save_batch()
# Now that the save is done, we have the ids ready -> create the simulations
for sim in self.created_simulations:
self.cache.append(
DataStore.create_simulation(
id=str(sim.id),
tags=sim.tags,
experiment_id=self.experiment.exp_id))
def COMPS_experiment_to_local_db(exp_id, endpoint, verbose=False, save_new_experiment=True):
"""
Return a DB object representing an experiment coming from COMPS.
This function saves the newly retrieved experiment in the DB by default but this behavior can be changed by switching
the save_new_experiment parameter allowing to return an experiment object and save later with a batch for example.
:param exp_id:
:param endpoint:
:param verbose:
:param save_new_experiment:
:return:
"""
# Make sure we are logged in
COMPS_login(endpoint)
#Ensure exp_id is a string
exp_id = str(exp_id)
# IF the experiment already exists and
experiment = DataStore.get_experiment(exp_id)
if experiment and experiment.is_done():
if verbose:
print("Experiment ('%s') already exists in local db." % exp_id)
# Do not bother with finished experiments
return None
from COMPS.Data import QueryCriteria
try:
query_criteria = QueryCriteria().select_children('tags')
exp_comps = get_experiment_by_id(exp_id, query_criteria) or get_experiments_by_name(exp_id, query_criteria)[-1]
except:
if verbose:
print("The experiment ('%s') doesn't exist in COMPS." % exp_id)
return None
# Case: experiment doesn't exist in local db
if not experiment:
# Cast the creation_date
experiment = DataStore.create_experiment(exp_id=str(exp_comps.id),
suite_id=str(exp_comps.suite_id) if exp_comps.suite_id else None,
exp_name=exp_comps.name,
tags=exp_comps.tags,
date_created=utc_to_local(exp_comps.date_created).replace(tzinfo=None),
location='HPC',
selected_block='HPC',
endpoint=endpoint)
# Note: experiment may be new or comes from local db
# Get associated simulations of the experiment
sims = exp_comps.get_simulations(QueryCriteria().select(['id', 'state', 'date_created']).select_children('tags'))
# Skip empty experiments or experiments that have the same number of sims
if len(sims) == 0 or len(sims) == len(experiment.simulations):
if verbose:
if len(sims) == 0:
print("Skip empty experiment ('%s')." % exp_id)
elif len(sims) == len(experiment.simulations):
print("Skip experiment ('%s') since local one has the same number of simulations." % exp_id)
return None
# Go through the sims and create them
for sim in sims:
# Cast the simulation tags
# Create the simulation
simulation = DataStore.create_simulation(id=str(sim.id),
status=sim.state, # this is already a SimulationState object
tags={tag:cast_number(val) for tag,val in sim.tags.items()},
date_created=utc_to_local(sim.date_created).replace(tzinfo=None))
# Add to the experiment
experiment.simulations.append(simulation)
# Save it to the DB
if save_new_experiment: DataStore.save_experiment(experiment, verbose=verbose)
return experiment
