def __setattr__(self, experiment_name, experiment):
assert isinstance(experiment, Experiment), "Your experiment must be an experiment!"
if experiment.name is None:
experiment.name = experiment_name
assert experiment_name not in get_global_experiment_library(), "Experiment %s is already in the library" % (experiment_name, )
self.__dict__[experiment_name] = experiment
get_global_experiment_library()[experiment_name] = experiment
def deprefix_experiment_ids(experiment_ids):
"""
Given a list of experiment ids, removed the common root experiments from the list.
:param experiment_ids: A list of experiment ids.
:return: A list of experiment ids with the root prefix removed.
"""
# First build dict mapping experiment_ids to their parents experiment_ids
glib = get_global_experiment_library()
exp_to_parent = {}
for eid in glib.keys():
ex = glib[eid]
for var in ex.get_variants():
exp_to_parent[var.get_id()] = ex.get_id()
@memoize
def get_experiment_tuple(exp_id):
if exp_id in exp_to_parent:
parent_id = exp_to_parent[exp_id]
parent_tuple = get_experiment_tuple(parent_id)
return parent_tuple + (exp_id[len(parent_id) + 1:], )
else:
return (exp_id, )
# Then for each experiment in the list,
tuples = [get_experiment_tuple(eid) for eid in experiment_ids]
de_prefixed_tuples = remove_common_prefix(tuples, keep_base=False)
start_with = '' if len(de_prefixed_tuples[0]) == len(tuples[0]) else '.'
new_strings = [
start_with + '.'.join(ex_tup) for ex_tup in de_prefixed_tuples
]
return new_strings
def _filter_experiment_record_list(user_range, experiment_record_ids):
if user_range=='all':
return [True]*len(experiment_record_ids)
elif user_range=='new':
return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True)
# return [n for n, is_old in izip_equal(get_record_ids(), old) if not old]
elif user_range=='old':
return [not x for x in _filter_records(user_range, 'new')]
elif user_range=='orphans':
orphans = []
global_lib = get_global_experiment_library()
for i, record_id in enumerate(experiment_record_ids):
info = load_experiment_record(record_id).info
if 'Module' in info:
try:
import_module(info['Module'])
if not record_id_to_experiment_id(record_id) in global_lib:
orphans.append(True)
else:
orphans.append(False)
except ImportError:
orphans.append(True)
else: # They must be old... lets kill them!
orphans.append(True)
return orphans
# elif user_range
else:
which_ones = interpret_numbers(user_range)
if which_ones is None:
raise Exception('Could not interpret user range: "{}"'.format(user_range))
filters = [False]*len(experiment_record_ids)
for i in which_ones:
filters[i] = True
return filters
def deprefix_experiment_ids(experiment_ids):
"""
Given a list of experiment ids, removed the common root experiments from the list.
:param experiment_ids: A list of experiment ids.
:return: A list of experiment ids with the root prefix removed.
"""
# First build dict mapping experiment_ids to their parents experiment_ids
glib = get_global_experiment_library()
exp_to_parent = {}
for eid in glib.keys():
ex = glib[eid]
for var in ex.get_variants():
exp_to_parent[var.get_id()] = ex.get_id()
@memoize
def get_experiment_tuple(exp_id):
if exp_id in exp_to_parent:
parent_id = exp_to_parent[exp_id]
parent_tuple = get_experiment_tuple(parent_id)
return parent_tuple + (exp_id[len(parent_id)+1:], )
else:
return (exp_id, )
# Then for each experiment in the list,
tuples = [get_experiment_tuple(eid) for eid in experiment_ids]
de_prefixed_tuples = remove_common_prefix(tuples, keep_base=False)
start_with = '' if len(de_prefixed_tuples[0])==len(tuples[0]) else '.'
new_strings = [start_with+'.'.join(ex_tup) for ex_tup in de_prefixed_tuples]
return new_strings
def _filter_experiment_record_list(user_range, experiment_record_ids):
if user_range=='all':
return [True]*len(experiment_record_ids)
elif user_range=='new':
return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True)
# return [n for n, is_old in izip_equal(get_record_ids(), old) if not old]
elif user_range=='old':
return [not x for x in _filter_records(user_range, 'new')]
elif user_range=='orphans':
orphans = []
global_lib = get_global_experiment_library()
for i, record_id in enumerate(experiment_record_ids):
info = load_experiment_record(record_id).info
if 'Module' in info:
try:
import_module(info['Module'])
if not record_id_to_experiment_id(record_id) in global_lib:
orphans.append(True)
else:
orphans.append(False)
except ImportError:
orphans.append(True)
else: # They must be old... lets kill them!
orphans.append(True)
return orphans
else:
which_ones = interpret_numbers(user_range)
if which_ones is None:
raise Exception('Could not interpret user range: "{}"'.format(user_range))
filters = [False]*len(experiment_record_ids)
for i in which_ones:
filters[i] = True
return filters
def reload_record_dict(self):
names = get_global_experiment_library().keys()
if self.root_experiment is not None:
# We could just go [ex.name for ex in self.root_experiment.get_all_variants(include_self=True)]
# but we want to preserve the order in which experiments were created
descendents_of_root = set(
ex.name for ex in self.root_experiment.get_all_variants(
include_self=True))
names = [name for name in names if name in descendents_of_root]
d = get_experient_to_record_dict(names)
return d
def find_experiment(*search_terms):
"""
Find an experiment. Invoke
:param search_term: A term that will be used to search for an experiment.
:return:
"""
global_lib = get_global_experiment_library()
found_experiments = OrderedDict((name, ex) for name, ex in global_lib.items() if all(re.search(term, name) for term in search_terms))
if len(found_experiments)==0:
raise Exception("None of the {} experiments matched the search: '{}'".format(len(global_lib), search_terms))
elif len(found_experiments)>1:
raise Exception("More than one experiment matched the search '{}', you need to be more specific. Found: {}".format(search_terms, found_experiments.keys()))
else:
return found_experiments.values()[0]
def find_experiment(*search_terms):
"""
Find an experiment. Invoke
:param search_term: A term that will be used to search for an experiment.
:return:
"""
global_lib = get_global_experiment_library()
found_experiments = OrderedDict((name, ex) for name, ex in global_lib.iteritems() if all(re.search(term, name) for term in search_terms))
if len(found_experiments)==0:
raise Exception("None of the {} experiments matched the search: '{}'".format(len(global_lib), search_terms))
elif len(found_experiments)>1:
raise Exception("More than one experiment matched the search '{}', you need to be more specific. Found: {}".format(search_terms, found_experiments.keys()))
else:
return found_experiments.values()[0]
def register_experiment(name, function, description = None, conclusion = None, versions = None, current_version = None, **kwargs):
"""
This is the old interface to experiments. We keep it, for now, for the sake of
backwards-compatibility.
In the future, use the @experiment_function decorator instead.
"""
info = OrderedDict()
if description is not None:
info['Description'] = description
if conclusion is not None:
info['Conclusion'] = conclusion
if versions is not None:
if current_version is None:
assert len(versions)==1
current_version = versions.keys()[0]
assert current_version is not None
function = partial(function, **versions[current_version])
assert name not in get_global_experiment_library(), 'An experiment with name "%s" has already been registered!' % (name, )
experiment = Experiment(name = name, function=function, info=info, **kwargs)
get_global_experiment_library()[name] = experiment
return experiment
def run_experiment(name, exp_dict='global', **experiment_record_kwargs):
"""
Run an experiment and save the results. Return a string which uniquely identifies the experiment.
You can run the experiment agin later by calling show_experiment(location_string):
:param name: The name for the experiment (must reference something in exp_dict)
:param exp_dict: A dict<str:func> where funcs is a function with no arguments that run the experiment.
:param experiment_record_kwargs: Passed to ExperimentRecord.
:return: A location_string, uniquely identifying the experiment.
"""
if exp_dict == 'global':
exp_dict = get_global_experiment_library()
experiment = exp_dict[name]
return experiment.run(**experiment_record_kwargs)
def reload_record_dict(self):
names = get_global_experiment_library().keys()
if self.root_experiment is not None:
# We could just go [ex.name for ex in self.root_experiment.get_all_variants(include_self=True)]
# but we want to preserve the order in which experiments were created
descendents_of_root = set(
ex.name for ex in self.root_experiment.get_all_variants(
include_self=True))
names = [name for name in names if name in descendents_of_root]
d = OrderedDict(
(name, experiment_id_to_record_ids(name)) for name in names)
if self.just_last_record:
for k in d.keys():
d[k] = [d[k][-1]] if len(d[k]) > 0 else []
return d
def get_experient_to_record_dict(experiment_ids=None):
"""
Given a list of experiment ids, return an OrderedDict whose keys are the experiment ids and whose values
are lists of experiment record ids.
:param experiment_ids: A list of experiment ids. (Defaults to all imported experiments)
:return: A dict<experiment_id -> list<experiment_record_id>
"""
if experiment_ids is None:
experiment_ids = get_global_experiment_library().keys()
record_ids = get_all_record_ids(experiment_ids)
exp_rec_dict = OrderedDict((exp_id, []) for exp_id in experiment_ids)
for rid in record_ids:
rec = load_experiment_record(rid)
exp_id = rec.get_experiment_id()
exp_rec_dict[exp_id].append(rid)
return exp_rec_dict
def run_experiment_by_name(name, exp_dict='global', slurm_job=False, experiment_path=None, **experiment_record_kwargs):
"""
Run an experiment and save the results. Return a string which uniquely identifies the experiment.
You can run the experiment again later by calling show_experiment(location_string):
:param name: The name for the experiment (must reference something in exp_dict)
:param exp_dict: A dict<str:func> where funcs is a function with no arguments that run the experiment.
:param slurm_job: It True, this function is interpreted as being run from within a SLURM call.
:param experiment_path: If not None, the 'experiment_directory' option in the 'experiments' section of the .artemisrc file will be temporarily set to this value
:param experiment_record_kwargs: Passed to ExperimentRecord.
:return: A location_string, uniquely identifying the experiment.
"""
if exp_dict == 'global':
exp_dict = get_global_experiment_library()
experiment = exp_dict[name]
return run_experiment(experiment,slurm_job, experiment_path, **experiment_record_kwargs)
def get_experient_to_record_dict(experiment_ids = None):
"""
Given a list of experiment ids, return an OrderedDict whose keys are the experiment ids and whose values
are lists of experiment record ids.
:param experiment_ids: A list of experiment ids. (Defaults to all imported experiments)
:return: A dict<experiment_id -> list<experiment_record_id>
"""
if experiment_ids is None:
experiment_ids = get_global_experiment_library().keys()
record_ids = get_all_record_ids(experiment_ids)
exp_rec_dict = OrderedDict((exp_id, []) for exp_id in experiment_ids)
for rid in record_ids:
rec = load_experiment_record(rid)
exp_id = rec.get_experiment_id()
exp_rec_dict[exp_id].append(rid)
return exp_rec_dict
def run_experiment_by_name(name, exp_dict='global', slurm_job=False, experiment_path=None, **experiment_record_kwargs):
"""
Run an experiment and save the results. Return a string which uniquely identifies the experiment.
You can run the experiment again later by calling show_experiment(location_string):
:param name: The name for the experiment (must reference something in exp_dict)
:param exp_dict: A dict<str:func> where funcs is a function with no arguments that run the experiment.
:param slurm_job: It True, this function is interpreted as being run from within a SLURM call.
:param experiment_path: If not None, the 'experiment_directory' option in the 'experiments' section of the .artemisrc file will be temporarily set to this value
:param experiment_record_kwargs: Passed to ExperimentRecord.
:return: A location_string, uniquely identifying the experiment.
"""
if exp_dict == 'global':
exp_dict = get_global_experiment_library()
experiment = exp_dict[name]
return run_experiment(experiment,slurm_job, experiment_path, **experiment_record_kwargs)
def __getattr__(self, name):
if name in get_global_experiment_library():
return get_global_experiment_library()[name]
else:
return _ExperimentConstructor(name)
def __call__(self, **kwargs):
if self.name in get_global_experiment_library():
raise Exception("You tried to run create experiment '%s', but it already exists in the library. Give it another name!" % (self.name, ))
return register_experiment(name = self.name, **kwargs)
def get_experiments(self):
return get_global_experiment_library()
