def get_all_records(cls):
all_records = None
with session_scope(Session_logs()) as session:
all_records = session.query(LogRecord).all()
session.expunge_all()
return all_records
def remove_empty_batch(cls):
with session_scope() as session:
cnt = session.query(Batch).filter(
and_(~Batch.experiments.any(),
~Batch.simulations.any())).delete(
synchronize_session=False)
return cnt
def get_setting(cls, setting):
with session_scope() as session:
setting = session.query(Settings).filter(
Settings.key == setting).one_or_none()
session.expunge_all()
return setting
def get_batch_by_name(cls, batch_name):
with session_scope() as session:
batch = session.query(Batch).filter(Batch.name == batch_name) \
.options(joinedload('simulations').joinedload('experiment')) \
.one_or_none()
session.expunge_all()
return batch
def get_simulation(cls, sim_id):
with session_scope() as session:
simulation = session.query(Simulation).options(
joinedload('experiment')).filter(
Simulation.id == sim_id).one_or_none()
session.expunge_all()
return simulation
def clear_batch(cls, batches=None):
if batches is None:
return
with session_scope() as session:
for batch in batches:
batch.experiments = []
batch.simulations = []
session.merge(batch)
def get_all_modules(cls):
modules = None
with session_scope(Session_logs()) as session:
modules = [
module[0]
for module in session.query(distinct(LogRecord.module)).all()
]
session.expunge_all()
return modules
def get_batch_by_id(cls, batch_id):
if batch_id is None:
return None
with session_scope() as session:
batch = session.query(Batch).filter(
Batch.id == batch_id).one_or_none()
session.expunge_all()
return batch
def get_records(cls, level, modules, number):
records = None
with session_scope(Session_logs()) as session:
query = session.query(LogRecord)\
.filter(and_(LogRecord.module.in_(modules), LogRecord.log_level >= level))\
.order_by(LogRecord.created.desc()) \
.limit(number)
records = query.all()
session.expunge_all()
return reversed(records)
def get_most_recent_experiment(cls, id_or_name=None):
logger.debug("Get most recent experiment")
id_or_name = '' if not id_or_name else id_or_name
with session_scope() as session:
experiment = session.query(Experiment) \
.filter(or_(Experiment.exp_id.like('%%%s%%' % id_or_name), Experiment.exp_name.like('%%%s%%' % id_or_name))) \
.options(joinedload('simulations').joinedload('experiment')) \
.order_by(Experiment.date_created.desc()).first()
session.expunge_all()
return experiment
def record_time(cls, label, extra_info=None):
current_thread = str(threading.current_thread().ident)
elapsed = timeit.default_timer() - cls.start_times[current_thread +
label]
with session_scope(Session_logs()) as session:
session.add(
Timing(elapsed_time=elapsed,
label=label,
timing_id=cls.timing_id,
extra_info=extra_info))
return elapsed
def save_batch(cls, batch):
with session_scope() as session:
if not batch.id:
session.add(batch)
session.flush()
if not batch.name:
batch.name = 'batch_%s' % batch.id
session.merge(batch)
session.expunge_all()
else:
session.merge(batch)
def get_simulation_states(cls, simids):
logger.debug("Get simulation states")
states_ret = []
from simtools.DataAccess.DataStore import batch
for ids in batch(simids, 50):
with session_scope() as session:
states = session.query(Simulation.id,
Simulation.status).filter(
Simulation.id.in_(ids)).all()
session.expunge_all()
states_ret.extend(states)
return states_ret
def get_experiment(cls, exp_id, current_session=None):
logger.debug("Get Experiment")
with session_scope(current_session) as session:
# Get the experiment
# Also load the associated simulations eagerly
experiment = session.query(Experiment).options(
joinedload('simulations').joinedload('experiment')) \
.filter(Experiment.exp_id == exp_id).one_or_none()
# Detach the object from the session
session.expunge_all()
return experiment
def get_experiments_by_suite(cls, suite_ids):
"""
Get the experiments which are associated with suite_id
suite_ids: list of suite ids
"""
with session_scope() as session:
exp_ids = session.query(Experiment.exp_id).filter(
Experiment.suite_id.in_(suite_ids)).all()
# Retrieve the individual experiments
exp_list = [cls.get_experiment(exp[0], session) for exp in exp_ids]
session.expunge_all()
return exp_list
def save_experiment(cls, experiment, verbose=True, session=None):
logger.debug("Save experiment")
if verbose:
# Dont display the null values
logger.info('Saving meta-data for experiment:')
logger.info(
json.dumps(remove_null_values(experiment.toJSON()),
indent=3,
cls=GeneralEncoder,
sort_keys=True))
with session_scope(session) as sess:
sess.merge(experiment)
def get_active_experiments(cls, location=None):
logger.debug("Get active experiments")
with session_scope() as session:
experiments = session.query(Experiment).distinct(Experiment.exp_id) \
.join(Experiment.simulations) \
.options(joinedload('simulations').joinedload('experiment')) \
.filter(~Simulation.status_s.in_((SimulationState.Succeeded.name, SimulationState.Failed.name, SimulationState.Canceled.name)))
if location:
experiments = experiments.filter(
Experiment.location == location)
experiments = experiments.all()
session.expunge_all()
return experiments
def get_experiments(cls, id_or_name=None, current_dir=None):
logger.debug("Get experiments")
id_or_name = '' if not id_or_name else id_or_name
with session_scope() as session:
experiments = session.query(Experiment)\
.filter(or_(Experiment.exp_id.like('%%%s%%' % id_or_name), Experiment.exp_name.like('%%%s%%' % id_or_name))) \
.options(joinedload('simulations').joinedload('experiment'))
if current_dir:
experiments = experiments.filter(
Experiment.working_directory == current_dir)
experiments = experiments.all()
session.expunge_all()
return experiments
def delete_batch(cls, batch=None):
with session_scope() as session:
if batch:
session.query(BatchSimulation).filter(
BatchSimulation.batch_id == batch.id).delete(
synchronize_session=False)
session.query(BatchExperiment).filter(
BatchExperiment.batch_id == batch.id).delete(
synchronize_session=False)
session.delete(batch)
else:
session.query(Batch).delete(synchronize_session=False)
session.query(BatchSimulation).delete(
synchronize_session=False)
session.query(BatchExperiment).delete(
synchronize_session=False)
def get_batch_list(cls, id_or_name=None):
batches = None
with session_scope() as session:
try:
batch_id = int(id_or_name)
batches = session.query(Batch).filter(Batch.id == batch_id) \
.options(joinedload('simulations').joinedload('experiment')).all()
except:
pass
if not batches:
batches = session.query(Batch) \
.filter(Batch.name.like("%{}%".format(id_or_name or ""))) \
.options(joinedload('simulations').joinedload('experiment')).all()
session.expunge_all()
return batches
def get_most_recent_experiment(cls, id_or_name=None):
with session_scope() as session:
# Retrieve the ID of the most recent experiment first
query = session.query(Experiment)
if id_or_name:
query = query.filter(
or_(Experiment.exp_id.like('%%%s%%' % id_or_name),
Experiment.exp_name.like('%%%s%%' % id_or_name)))
e = query.order_by(Experiment.date_created.desc()).first()
if not e:
return None
eid = e.exp_id
experiment = cls.get_experiment(eid, session)
session.expunge_all()
return experiment
def batch_simulations_update(cls, simulation_batch):
"""
Takes a batch of simulations and update their status in the DB.
This function provides performance considerations when updating large number of simulations in the db.
The batch needs to be formatted as follow:
[
{'sid':'simid', "status": 'simstatus'},
{'sid':'simid', "status": 'simstatus'}
]
Args:
batch: Batch of simulations to save
"""
if len(simulation_batch) == 0: return
with session_scope() as session:
stmt = update(Simulation).where(and_(Simulation.id == bindparam("sid"),
not_(Simulation.status in (SimulationState.Succeeded, SimulationState.Failed, SimulationState.Canceled))))\
.values(status_s=bindparam("status"))
session.execute(stmt, simulation_batch)
def get_recent_experiment_by_filter(cls,
num=20,
is_all=False,
name=None,
location=None):
with session_scope() as session:
experiment = session.query(Experiment) \
.options(joinedload('simulations')) \
.order_by(Experiment.date_created.desc())
if name:
experiment = experiment.filter(
Experiment.exp_name.like('%%%s%%' % name))
if location:
experiment = experiment.filter(Experiment.location == location)
if is_all:
experiment = experiment.all()
else:
experiment = experiment.limit(num).all()
session.expunge_all()
return experiment
def batch_save_experiments(cls, batch):
logger.debug("Batch save experiments")
with session_scope() as session:
for exp in batch:
cls.save_experiment(exp, False, session)
def save_record(cls, record):
try:
with session_scope(Session_logs()) as session:
session.add(record)
except:
pass
def save_setting(cls, setting):
with session_scope() as session:
session.merge(setting)
def delete_experiment(cls, experiment):
logger.debug("Delete experiment %s" % experiment.id)
with session_scope() as session:
session.delete(experiment)
def bulk_insert_simulations(cls, simulations):
with session_scope() as session:
session.bulk_save_objects(simulations)
def delete_simulation(cls, simulation):
logger.debug("Delete simulation %s" % simulation.id)
with session_scope() as session:
session.query(Simulation).filter(
Simulation.id == simulation.id).delete(
synchronize_session=False)
def save_simulation(cls, simulation, session=None):
logger.debug("Save simulation")
with session_scope(session) as session:
session.merge(simulation)
