def get_simulations_from_big_experiments(experiment_id):
e = get_experiment_by_id(experiment_id)
start_date = end_date = e.date_created
import pytz
limit_date = datetime.today().replace(tzinfo=pytz.utc)
interval = 60
stop_flag = False
results = {}
while start_date < limit_date:
start_date = end_date + timedelta(minutes=interval)
try:
batch = Simulation.get(query_criteria=QueryCriteria()
.select(['id', 'state', 'date_created']).select_children('tags')
.where(["experiment_id={}".format(experiment_id),
"date_created>={}".format(end_date.strftime('%Y-%m-%d %T')),
"date_created<={}".format(start_date.strftime('%Y-%m-%d %T'))])
)
except:
interval /= 2
continue
if not batch:
if stop_flag:
break
else:
interval = 120
stop_flag = True
else:
stop_flag = False
for s in batch:
results[s.id] = s
end_date = start_date
return results.values()
def query_simulation(sid, criteria=None, children=None):
from COMPS.Data import Simulation
from COMPS.Data import QueryCriteria
if children:
criteria = criteria or QueryCriteria()
criteria.select_children(children)
return Simulation.get(sid, query_criteria=criteria)
# EDIT
# Change this to be the congig update you actually want
# config['parameters']['AIDS_Duration_In_Months'] = 999
with open(filename, 'w') as fp
json.dump(config, fp)
return None
#### END EDIT #######################################################################
def copy_simulation(simulation, to_experiment)
simulation.refresh(query_criteria=QueryCriteria().select_children(['files', 'hpc_jobs', 'tags']))
new_simulation = Simulation(simulation.name, description=simulation.description)
new_simulation.experiment_id = to_experiment.id
tags = copy.copy(simulation.tags)
tags[CopiedFromSimulation] = simulation.id
new_simulation.set_tags(tags)
job = simulation.hpc_jobs[-1]
# override any fields here as necessary...
if job and job.configuration
new_simulation.configuration = Configuration(
environment_name=job.configuration.environment_name,
simulation_input_args=job.configuration.simulation_input_args,
working_directory_root=job.configuration.working_directory_root,
executable_path=job.configuration.executable_path,
def copy_simulation(simulation, to_experiment):
simulation.refresh(query_criteria=QueryCriteria().select_children(
['files', 'hpc_jobs', 'tags']))
new_simulation = Simulation(simulation.name,
description=simulation.description)
new_simulation.experiment_id = to_experiment.id
tags = copy.copy(simulation.tags)
tags["CopiedFromSimulation"] = simulation.id
new_simulation.set_tags(tags)
job = simulation.hpc_jobs[-1]
# override any fields here as necessary...
if job and job.configuration:
new_simulation.configuration = Configuration(
environment_name=job.configuration.environment_name,
simulation_input_args=job.configuration.simulation_input_args,
working_directory_root=job.configuration.working_directory_root,
executable_path=job.configuration.executable_path,
maximum_number_of_retries=SetupParser.get(parameter='num_retries'),
priority=SetupParser.get(parameter='priority'),
min_cores=job.configuration.min_cores,
max_cores=job.configuration.max_cores,
exclusive=job.configuration.exclusive,
node_group_name=SetupParser.get(parameter='node_group'),
asset_collection_id=job.configuration.asset_collection_id)
with tempfile.TemporaryDirectory() as dir:
files_to_add_last = {}
for f in simulation.files:
if f.file_name == 'config.json':
dest_file = os.path.join(dir, 'config.json')
with open(dest_file, 'wb') as fp:
fp.write(f.retrieve())
modify_config_json(dest_file)
# with open(dest_file, 'rb') as fp:
# data = fp.read()
filename = dest_file
# checksum = hashlib.md5(data).hexdigest()
sf = SimulationFile(file_name=filename,
file_type=f.file_type,
description=f.description)
files_to_add_last[filename] = sf
else:
filename = f.file_name
checksum = f.md5_checksum
sf = SimulationFile(file_name=filename,
file_type=f.file_type,
description=f.description,
md5_checksum=checksum)
new_simulation.add_file(sf)
new_simulation.save(return_missing_files=False)
if len(files_to_add_last) > 0:
for file_path, sf in files_to_add_last.items():
new_simulation.add_file(sf, file_path=file_path)
new_simulation.save(return_missing_files=False)
print('new sim = ' + str(new_simulation.id))
return new_simulation
exp_num, SIMULATIONS_PER_EXPERIMENT))
# Create the suite
em = ExperimentManagerFactory.init()
s = em.create_suite(SUITE_NAME)
print("Suite named {} created with the id: {}".format(SUITE_NAME, s))
# Create the experiments
experiments = []
for i in range(exp_num):
em.create_experiment("{}_{}".format(SUITE_NAME, i), suite_id=s)
experiments.append(em.comps_experiment)
print("Created the experiment: {}".format(em.experiment.exp_name))
exp_index = 0
sims_in_exp = 0
# For each simulations batch them
for s in sims:
s.experiment_id = experiments[exp_index].id
sims_in_exp += 1
if sims_in_exp == SIMULATIONS_PER_EXPERIMENT:
print("Experiment {} full...".format(experiments[exp_index].name))
Simulation.save_all(save_batch_callback=None,
save_semaphore=Simulation.get_save_semaphore())
exp_index += 1
sims_in_exp = 0
print("Processing done. Use the new suite id for analysis:")
print(s)
def get_semaphore():
return Simulation.get_save_semaphore()
def sims_from_experiment_id(exp_id):
return Simulation.get(query_criteria=QueryCriteria().select(
['id', 'state']).where('experiment_id=%s' % exp_id))
def get_simulation_by_id(sim_id, query_criteria=None):
return Simulation.get(id=sim_id, query_criteria=query_criteria)
def get_asset_collection_id_for_simulation_id(sim_id):
query_criteria = QueryCriteria().select_children('configuration')
simulation = Simulation.get(id=sim_id, query_criteria=query_criteria)
collection_id = simulation.configuration.asset_collection_id
return collection_id
def save_batch(self):
# Batch save after all sims in list have been added
with nostdout(stderr=True):
Simulation.save_all(save_semaphore=self.save_semaphore)
def create_simulation(self, cb):
name = cb.get_param('Config_Name') or self.experiment.exp_name
return Simulation(name=name, experiment_id=self.experiment.exp_id,
configuration=Configuration(asset_collection_id=cb.assets.collection_id))