def sims_from_suite_id(suite_id):
exps = Experiment.get(query_criteria=QueryCriteria().where('suite_id=%s' %
suite_id))
sims = []
for e in exps:
sims += sims_from_experiment(e)
return sims
def get_all_experiments_for_user(user):
# COMPS limits the retrieval to 1000 so to make sure we get all experiments for a given user, we need to be clever
# Also COMPS does not have an order_by so we have to go through all date ranges
interval = 365
results = {}
end_date = start_date = datetime.today()
limit_date = datetime.strptime("2014-03-31", '%Y-%m-%d') # Oldest simulation in COMPS
while start_date > limit_date:
start_date = end_date - timedelta(days=interval)
batch = Experiment.get(query_criteria=QueryCriteria().where(["owner={}".format(user),
"date_created<={}".format(
end_date.strftime('%Y-%m-%d')),
"date_created>={}".format(
start_date.strftime('%Y-%m-%d'))]))
if len(batch) == 1000:
# We hit a limit, reduce the interval and run again
interval = interval / 2
continue
if len(batch) == 0:
interval *= 2
else:
# Add the experiments to the dict
for e in batch:
results[e.id] = e
# Go from there
end_date = start_date
return results.values()
def query_experiment(eid=None, criteria=None, children=None):
from COMPS.Data import Experiment
from COMPS.Data import QueryCriteria
criteria = criteria or QueryCriteria()
children = children or ["tags"]
criteria.select_children(children)
exp = Experiment.get(eid, query_criteria=criteria)
return exp
def create_experiment(self,
experiment_name,
experiment_id=None,
suite_id=None):
# Also create the experiment in COMPS to get the ID
COMPS_login(SetupParser.get('server_endpoint'))
experiment_name = self.clean_experiment_name(experiment_name)
subdirectory = experiment_name[
0:self.MAX_SUBDIRECTORY_LENGTH] + '_' + timestamp()
config = Configuration(
environment_name=SetupParser.get('environment'),
simulation_input_args=self.commandline.Options,
working_directory_root=os.path.join(SetupParser.get('sim_root'),
subdirectory),
executable_path=self.commandline.Executable,
node_group_name=SetupParser.get('node_group'),
maximum_number_of_retries=int(SetupParser.get('num_retries')),
priority=Priority[SetupParser.get('priority')],
min_cores=self.config_builder.get_param('Num_Cores', 1)
if self.config_builder else 1,
max_cores=self.config_builder.get_param('Num_Cores', 1)
if self.config_builder else 1,
exclusive=self.config_builder.get_param('Exclusive', False)
if self.config_builder else False)
e = Experiment(name=experiment_name,
configuration=config,
suite_id=suite_id)
# Add tags if present
if self.experiment_tags: e.set_tags(self.experiment_tags)
e.save()
# Store in our object
self.comps_experiment = e
# Also add it to the cache
COMPSCache.add_experiment_to_cache(e)
# Create experiment in the base class
super(CompsExperimentManager,
self).create_experiment(experiment_name, str(e.id), suite_id)
# Set some extra stuff
self.experiment.endpoint = self.endpoint
md5_checksum=checksum)
new_simulation.add_file(sf)
new_simulation.save(return_missing_files=False)
if len(files_to_add_last) 0
for file_path, sf in files_to_add_last.items()
new_simulation.add_file(sf, file_path=file_path)
new_simulation.save(return_missing_files=False)
print('new sim = ' + str(new_simulation.id))
return new_simulation
def copy_experiment(experiment, to_suite)
new_experiment = Experiment(name=experiment.name, suite_id=to_suite.id)
new_experiment.set_tags({CopiedFromExperiment experiment.id})
new_experiment.save()
return new_experiment
if __name__ == __main__
from simtools.Utilities.COMPSUtilities import exps_for_suite_id, sims_from_experiment_id
from simtools.Utilities.Experiments import retrieve_experiment
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager
SetupParser.init()
# load the original stuff
original_suite = Suite.get(id=original_suite_id)
def copy_experiment(experiment, to_suite):
new_experiment = Experiment(name=experiment.name, suite_id=to_suite.id)
new_experiment.set_tags({"CopiedFromExperiment": experiment.id})
new_experiment.save()
return new_experiment
def get_experiments_by_name(name,user):
return Experiment.get(query_criteria=QueryCriteria().where(['name~%s' % name, 'owner=%s' % user]))
def display(users, top=15, save=False, refresh=False):
DiskSpaceUsage.OWNERS = users
DiskSpaceUsage.TOP_COUNT = top if top else 15
DiskSpaceUsage.FORCE_REFRESH = refresh
# Create/open the cache
current_folder = os.path.dirname(os.path.realpath(__file__))
cache_folder = os.path.join(current_folder, "cache")
cache = FanoutCache(shards=6, directory=cache_folder)
# All experiments
all_experiments = list(
itertools.chain(*(Experiment.get(query_criteria=QueryCriteria().
where(["owner={}".format(owner)]))
for owner in DiskSpaceUsage.OWNERS)))
all_experiments_len = len(all_experiments)
# Create the pool of worker
p = Pool(6)
r = p.starmap_async(DiskSpaceUsage.get_experiment_info,
itertools.product(all_experiments, (cache, )))
p.close()
print("Analyzing disk space for:")
print(" | {} experiments".format(all_experiments_len))
print(" | Users: {}".format(", ".join(DiskSpaceUsage.OWNERS)))
# Wait for completion and display progress
sys.stdout.write(
" | Experiment analyzed: 0/{}".format(all_experiments_len))
sys.stdout.flush()
# While we are analyzing, display the status
while not r.ready():
# Estimate how many remaining we have. This is just an estimations and needs to be bounded
remaining = max(
0, min(all_experiments_len, r._number_left * r._chunksize))
sys.stdout.write("\r {} Experiment analyzed: {}/{}".format(
next(animation), all_experiments_len - remaining,
all_experiments_len))
sys.stdout.flush()
time.sleep(.5)
sys.stdout.write("\r | Experiment analyzed: {}/{}".format(
all_experiments_len, all_experiments_len))
sys.stdout.flush()
# Get all the results
experiments_info = [
cache.get(e.id) for e in all_experiments if cache.get(e.id)
]
cache.close()
# Display
print("\n\n---------------------------")
DiskSpaceUsage.top_count_experiments(experiments_info)
print("\n---------------------------")
DiskSpaceUsage.total_size_per_user(experiments_info)
print("\n---------------------------")
DiskSpaceUsage.top_count_experiments_per_user(experiments_info)
# save to a csv file
if save:
DiskSpaceUsage.save_to_file(experiments_info)
def workdirs_from_experiment_id(exp_id, experiment=None):
e = experiment or Experiment.get(exp_id)
sims = sims_from_experiment(e)
return workdirs_from_simulations(sims)
def exps_for_suite_id(suite_id):
try:
return Experiment.get(
query_criteria=QueryCriteria().where('suite_id=%s' % suite_id))
except:
return None
def get_experiments_by_name(name, user=None):
filters = ["name~{}".format(name)]
if user: filters.append("owner={}".format(user))
return Experiment.get(query_criteria=QueryCriteria().where(filters))
def get_experiments_per_user_and_date(user, limit_date):
limit_date_str = limit_date.strftime("%Y-%m-%d")
return Experiment.get(
query_criteria=QueryCriteria().where('owner=%s,DateCreated>%s' %
(user, limit_date_str)))
def get_experiment_by_id(exp_id, query_criteria=None):
return Experiment.get(exp_id, query_criteria=query_criteria)
def get_experiment_ids_for_user(user):
exps = Experiment.get(query_criteria=QueryCriteria().select(['id']).where(
['owner={}'.format(user)]))
return [str(exp.id) for exp in exps]