def get_simulation_csv(self, simulation):
user = self.getCurrentUser()
simulation_model = Simulation()
summary_stats = simulation_model.get_summary_stats(simulation, user)
# The values of summary stats will typically be nested dicts, now we flatten them
summary_stats = {
time: flatten_dict(data)
for time, data in summary_stats.items()
}
# move it to a list and sort by time
summary_stats = [(time, data) for time, data in summary_stats.items()]
summary_stats.sort(key=lambda x: float(x[0]))
# write a csv to memory
with io.StringIO() as sio:
csvwriter = csv.writer(sio, dialect='excel')
if len(summary_stats) > 0:
# header
csvwriter.writerow(
["time", *[label for label, value in summary_stats[0][1]]])
for time, data in summary_stats:
csvwriter.writerow(
[time, *[value for label, value in data]])
rest.setResponseHeader('Content-Type', 'text/csv')
return sio.getvalue()
def list_simulations(self,
limit,
offset,
sort,
includeArchived,
mine,
experiments,
creator=None,
config=None):
user = self.getCurrentUser()
simulation_model = Simulation()
if mine and user is None:
return []
if mine and creator and creator['_id'] != user['_id']:
return []
if mine:
creator = user
return simulation_model.list(
includeArchived=includeArchived,
user=user,
limit=limit,
offset=offset,
sort=sort,
creator=creator,
config=config,
in_experiment=experiments,
)
def execute_simulation(self, name, config, folder=None):
target_time = config.get('simulation', {}).get('run_time', 50)
user, token = self.getCurrentUser(returnToken=True)
folder_model = Folder()
job_model = Job()
if folder is None:
folder = folder_model.findOne(
{'parentId': user['_id'], 'name': 'Public', 'parentCollection': 'user'}
)
if folder is None:
raise RestException('Could not find the user\'s "public" folder.')
simulation_model = Simulation()
simulation = simulation_model.createSimulation(
folder,
name,
config,
user,
True,
)
girder_config = GirderConfig(
api=GIRDER_API, token=str(token['_id']), folder=str(folder['_id'])
)
simulation_config = SimulationConfig(NLI_CONFIG_FILE, config)
# TODO: This would be better stored as a dict, but it's easier once we change the
# config object format.
simulation_config_file = StringIO()
simulation_config.write(simulation_config_file)
job = job_model.createJob(
title='NLI Simulation',
type=NLI_JOB_TYPE,
kwargs={
'girder_config': attr.asdict(girder_config),
'simulation_config': simulation_config_file.getvalue(),
'config': config,
'simulation_id': simulation['_id'],
},
user=user,
)
simulation['nli']['job_id'] = job['_id']
simulation_model.save(simulation)
run_simulation.delay(
name=name,
girder_config=girder_config,
simulation_config=simulation_config,
target_time=target_time,
job=job,
simulation_id=simulation['_id'],
)
return job
def cancel_experiment(self, experiment):
simulation_model = Simulation()
for simulation in simulation_model.childFolders(
experiment['_id'], 'folder'):
# TODO: I added the 'folder' so that the signature matches. This was my best guess;
# 'folder', 'user', 'collection' are the available options.
# noinspection PyBroadException
try:
self._cancel_simulation(simulation)
except Exception:
logger.exception(
f'Failed to cancel simulation "{simulation["_id"]}"')
def update_status(event):
simulation_model = Simulation()
job = event.info['job']
if job['type'] != NLI_JOB_TYPE:
return
simulation_id = job['kwargs'].get('simulation_id')
simulation = simulation_model.load(simulation_id, force=True)
if simulation is None:
logger.error(f'Could not find simulation for job {job["_id"]}')
return
progress = job['progress']
simulation['nli']['progress'] = 100 * (progress['current'] /
progress['total'])
simulation['nli']['status'] = job['status']
simulation_model.save(simulation)
def simulation_runner(
*,
config,
parent_folder,
job_model: Job,
run_name,
target_time,
token,
user,
experiment=None,
):
simulation_model = Simulation()
simulation = simulation_model.createSimulation(
parentFolder=parent_folder,
name=run_name,
config=config,
creator=user,
version=nlisim_version,
public=True,
experiment=experiment,
)
# if this is to be part of an experiment, let the experiment know about it
if experiment is not None:
experiment['nli']['component_simulations'].append(simulation['_id'])
experiment['nli']['per_sim_progress'][str(simulation['_id'])] = 0.0
experiment['nli']['per_sim_status'][str(
simulation['_id'])] = JobStatus.INACTIVE
experiment_model = Experiment()
experiment_model.save(experiment)
girder_config = GirderConfig(api=GIRDER_API,
token=str(token['_id']),
folder=str(parent_folder['_id']))
simulation_config = SimulationConfig(NLI_CONFIG_FILE, config)
# TODO: This would be better stored as a dict, but it's easier once we change the
# config object format.
simulation_config_file = StringIO()
simulation_config.write(simulation_config_file)
job = job_model.createJob(
title='NLI Simulation',
type=NLI_JOB_TYPE,
kwargs={
'girder_config': attr.asdict(girder_config),
'simulation_config': simulation_config_file.getvalue(),
'config': config,
'simulation_id': simulation['_id'],
'in_experiment': (experiment is not None),
'experiment_id': None if experiment is None else experiment['_id'],
},
user=user,
)
simulation['nli']['job_id'] = job['_id']
simulation_model.save(simulation)
run_simulation.delay(
name=run_name,
girder_config=girder_config,
simulation_config=simulation_config,
target_time=target_time,
job=job,
simulation_id=simulation['_id'],
)
return job, simulation
def mark_simulation_archived(self, simulation, archived):
simulation['nli']['archived'] = archived
simulation_model = Simulation()
return simulation_model.save(simulation)
def mark_simulation_complete(self, simulation):
simulation_model = Simulation()
return simulation_model.setSimulationComplete(simulation)
def get_simulation_json(self, simulation):
user = self.getCurrentUser()
simulation_model = Simulation()
return simulation_model.get_summary_stats(simulation, user)
def update_status(event):
simulation_model = Simulation()
job = event.info['job']
if job['type'] != NLI_JOB_TYPE:
return
simulation_id = job['kwargs'].get('simulation_id')
simulation = simulation_model.load(simulation_id, force=True)
if simulation is None:
logger.error(f'Could not find simulation for job {job["_id"]}')
return
progress = job['progress']
simulation['nli']['progress'] = 100 * (progress['current'] / progress['total'])
simulation['nli']['status'] = job['status']
simulation_model.save(simulation)
# update the progress for the experiment, if this is part of one
if job['kwargs'].get('in_experiment'):
experiment_model = Experiment()
experiment = experiment_model.load(job['kwargs'].get('experiment_id'), force=True)
# update the individual progress
experiment['nli']['per_sim_progress'][str(simulation_id)] = simulation['nli']['progress']
per_sim_progress = experiment['nli']['per_sim_progress']
# update the total progress (defining this as the mean progress)
experiment['nli']['progress'] = sum(per_sim_progress.values()) / len(per_sim_progress)
# update job status
experiment['nli']['per_sim_status'][str(simulation_id)] = job['status']
# any errors or cancellations count as an error or cancellation of the experiment,
# experiment doesn't become active until all of the sims are active.
if any(
status == JobStatus.ERROR for status in experiment['nli']['per_sim_status'].values()
):
experiment['nli']['status'] = JobStatus.ERROR
elif any(
status == JobStatus.CANCELED for status in experiment['nli']['per_sim_status'].values()
):
experiment['nli']['status'] = JobStatus.CANCELED
elif any(
status == JobStatus.INACTIVE for status in experiment['nli']['per_sim_status'].values()
):
experiment['nli']['status'] = JobStatus.INACTIVE
else:
# in this case, all statuses must be QUEUED, RUNNING, or SUCCESS
# we take the "minimum" for the experiment's status.
if any(
status == JobStatus.QUEUED
for status in experiment['nli']['per_sim_status'].values()
):
experiment['nli']['status'] = JobStatus.QUEUED
elif any(
status == JobStatus.RUNNING
for status in experiment['nli']['per_sim_status'].values()
):
experiment['nli']['status'] = JobStatus.RUNNING
else:
experiment['nli']['status'] = JobStatus.SUCCESS
experiment_model.save(experiment)