def __setattr__(self, experiment_name, experiment): assert isinstance(experiment, Experiment), "Your experiment must be an experiment!" if experiment.name is None: experiment.name = experiment_name assert experiment_name not in get_global_experiment_library(), "Experiment %s is already in the library" % (experiment_name, ) self.__dict__[experiment_name] = experiment get_global_experiment_library()[experiment_name] = experiment
def deprefix_experiment_ids(experiment_ids): """ Given a list of experiment ids, removed the common root experiments from the list. :param experiment_ids: A list of experiment ids. :return: A list of experiment ids with the root prefix removed. """ # First build dict mapping experiment_ids to their parents experiment_ids glib = get_global_experiment_library() exp_to_parent = {} for eid in glib.keys(): ex = glib[eid] for var in ex.get_variants(): exp_to_parent[var.get_id()] = ex.get_id() @memoize def get_experiment_tuple(exp_id): if exp_id in exp_to_parent: parent_id = exp_to_parent[exp_id] parent_tuple = get_experiment_tuple(parent_id) return parent_tuple + (exp_id[len(parent_id) + 1:], ) else: return (exp_id, ) # Then for each experiment in the list, tuples = [get_experiment_tuple(eid) for eid in experiment_ids] de_prefixed_tuples = remove_common_prefix(tuples, keep_base=False) start_with = '' if len(de_prefixed_tuples[0]) == len(tuples[0]) else '.' new_strings = [ start_with + '.'.join(ex_tup) for ex_tup in de_prefixed_tuples ] return new_strings
def _filter_experiment_record_list(user_range, experiment_record_ids): if user_range=='all': return [True]*len(experiment_record_ids) elif user_range=='new': return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True) # return [n for n, is_old in izip_equal(get_record_ids(), old) if not old] elif user_range=='old': return [not x for x in _filter_records(user_range, 'new')] elif user_range=='orphans': orphans = [] global_lib = get_global_experiment_library() for i, record_id in enumerate(experiment_record_ids): info = load_experiment_record(record_id).info if 'Module' in info: try: import_module(info['Module']) if not record_id_to_experiment_id(record_id) in global_lib: orphans.append(True) else: orphans.append(False) except ImportError: orphans.append(True) else: # They must be old... lets kill them! orphans.append(True) return orphans # elif user_range else: which_ones = interpret_numbers(user_range) if which_ones is None: raise Exception('Could not interpret user range: "{}"'.format(user_range)) filters = [False]*len(experiment_record_ids) for i in which_ones: filters[i] = True return filters
def deprefix_experiment_ids(experiment_ids): """ Given a list of experiment ids, removed the common root experiments from the list. :param experiment_ids: A list of experiment ids. :return: A list of experiment ids with the root prefix removed. """ # First build dict mapping experiment_ids to their parents experiment_ids glib = get_global_experiment_library() exp_to_parent = {} for eid in glib.keys(): ex = glib[eid] for var in ex.get_variants(): exp_to_parent[var.get_id()] = ex.get_id() @memoize def get_experiment_tuple(exp_id): if exp_id in exp_to_parent: parent_id = exp_to_parent[exp_id] parent_tuple = get_experiment_tuple(parent_id) return parent_tuple + (exp_id[len(parent_id)+1:], ) else: return (exp_id, ) # Then for each experiment in the list, tuples = [get_experiment_tuple(eid) for eid in experiment_ids] de_prefixed_tuples = remove_common_prefix(tuples, keep_base=False) start_with = '' if len(de_prefixed_tuples[0])==len(tuples[0]) else '.' new_strings = [start_with+'.'.join(ex_tup) for ex_tup in de_prefixed_tuples] return new_strings
def _filter_experiment_record_list(user_range, experiment_record_ids): if user_range=='all': return [True]*len(experiment_record_ids) elif user_range=='new': return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True) # return [n for n, is_old in izip_equal(get_record_ids(), old) if not old] elif user_range=='old': return [not x for x in _filter_records(user_range, 'new')] elif user_range=='orphans': orphans = [] global_lib = get_global_experiment_library() for i, record_id in enumerate(experiment_record_ids): info = load_experiment_record(record_id).info if 'Module' in info: try: import_module(info['Module']) if not record_id_to_experiment_id(record_id) in global_lib: orphans.append(True) else: orphans.append(False) except ImportError: orphans.append(True) else: # They must be old... lets kill them! orphans.append(True) return orphans else: which_ones = interpret_numbers(user_range) if which_ones is None: raise Exception('Could not interpret user range: "{}"'.format(user_range)) filters = [False]*len(experiment_record_ids) for i in which_ones: filters[i] = True return filters
def reload_record_dict(self): names = get_global_experiment_library().keys() if self.root_experiment is not None: # We could just go [ex.name for ex in self.root_experiment.get_all_variants(include_self=True)] # but we want to preserve the order in which experiments were created descendents_of_root = set( ex.name for ex in self.root_experiment.get_all_variants( include_self=True)) names = [name for name in names if name in descendents_of_root] d = get_experient_to_record_dict(names) return d
def find_experiment(*search_terms): """ Find an experiment. Invoke :param search_term: A term that will be used to search for an experiment. :return: """ global_lib = get_global_experiment_library() found_experiments = OrderedDict((name, ex) for name, ex in global_lib.items() if all(re.search(term, name) for term in search_terms)) if len(found_experiments)==0: raise Exception("None of the {} experiments matched the search: '{}'".format(len(global_lib), search_terms)) elif len(found_experiments)>1: raise Exception("More than one experiment matched the search '{}', you need to be more specific. Found: {}".format(search_terms, found_experiments.keys())) else: return found_experiments.values()[0]
def find_experiment(*search_terms): """ Find an experiment. Invoke :param search_term: A term that will be used to search for an experiment. :return: """ global_lib = get_global_experiment_library() found_experiments = OrderedDict((name, ex) for name, ex in global_lib.iteritems() if all(re.search(term, name) for term in search_terms)) if len(found_experiments)==0: raise Exception("None of the {} experiments matched the search: '{}'".format(len(global_lib), search_terms)) elif len(found_experiments)>1: raise Exception("More than one experiment matched the search '{}', you need to be more specific. Found: {}".format(search_terms, found_experiments.keys())) else: return found_experiments.values()[0]
def register_experiment(name, function, description = None, conclusion = None, versions = None, current_version = None, **kwargs): """ This is the old interface to experiments. We keep it, for now, for the sake of backwards-compatibility. In the future, use the @experiment_function decorator instead. """ info = OrderedDict() if description is not None: info['Description'] = description if conclusion is not None: info['Conclusion'] = conclusion if versions is not None: if current_version is None: assert len(versions)==1 current_version = versions.keys()[0] assert current_version is not None function = partial(function, **versions[current_version]) assert name not in get_global_experiment_library(), 'An experiment with name "%s" has already been registered!' % (name, ) experiment = Experiment(name = name, function=function, info=info, **kwargs) get_global_experiment_library()[name] = experiment return experiment
def run_experiment(name, exp_dict='global', **experiment_record_kwargs): """ Run an experiment and save the results. Return a string which uniquely identifies the experiment. You can run the experiment agin later by calling show_experiment(location_string): :param name: The name for the experiment (must reference something in exp_dict) :param exp_dict: A dict<str:func> where funcs is a function with no arguments that run the experiment. :param experiment_record_kwargs: Passed to ExperimentRecord. :return: A location_string, uniquely identifying the experiment. """ if exp_dict == 'global': exp_dict = get_global_experiment_library() experiment = exp_dict[name] return experiment.run(**experiment_record_kwargs)
def reload_record_dict(self): names = get_global_experiment_library().keys() if self.root_experiment is not None: # We could just go [ex.name for ex in self.root_experiment.get_all_variants(include_self=True)] # but we want to preserve the order in which experiments were created descendents_of_root = set( ex.name for ex in self.root_experiment.get_all_variants( include_self=True)) names = [name for name in names if name in descendents_of_root] d = OrderedDict( (name, experiment_id_to_record_ids(name)) for name in names) if self.just_last_record: for k in d.keys(): d[k] = [d[k][-1]] if len(d[k]) > 0 else [] return d
def get_experient_to_record_dict(experiment_ids=None): """ Given a list of experiment ids, return an OrderedDict whose keys are the experiment ids and whose values are lists of experiment record ids. :param experiment_ids: A list of experiment ids. (Defaults to all imported experiments) :return: A dict<experiment_id -> list<experiment_record_id> """ if experiment_ids is None: experiment_ids = get_global_experiment_library().keys() record_ids = get_all_record_ids(experiment_ids) exp_rec_dict = OrderedDict((exp_id, []) for exp_id in experiment_ids) for rid in record_ids: rec = load_experiment_record(rid) exp_id = rec.get_experiment_id() exp_rec_dict[exp_id].append(rid) return exp_rec_dict
def run_experiment_by_name(name, exp_dict='global', slurm_job=False, experiment_path=None, **experiment_record_kwargs): """ Run an experiment and save the results. Return a string which uniquely identifies the experiment. You can run the experiment again later by calling show_experiment(location_string): :param name: The name for the experiment (must reference something in exp_dict) :param exp_dict: A dict<str:func> where funcs is a function with no arguments that run the experiment. :param slurm_job: It True, this function is interpreted as being run from within a SLURM call. :param experiment_path: If not None, the 'experiment_directory' option in the 'experiments' section of the .artemisrc file will be temporarily set to this value :param experiment_record_kwargs: Passed to ExperimentRecord. :return: A location_string, uniquely identifying the experiment. """ if exp_dict == 'global': exp_dict = get_global_experiment_library() experiment = exp_dict[name] return run_experiment(experiment,slurm_job, experiment_path, **experiment_record_kwargs)
def get_experient_to_record_dict(experiment_ids = None): """ Given a list of experiment ids, return an OrderedDict whose keys are the experiment ids and whose values are lists of experiment record ids. :param experiment_ids: A list of experiment ids. (Defaults to all imported experiments) :return: A dict<experiment_id -> list<experiment_record_id> """ if experiment_ids is None: experiment_ids = get_global_experiment_library().keys() record_ids = get_all_record_ids(experiment_ids) exp_rec_dict = OrderedDict((exp_id, []) for exp_id in experiment_ids) for rid in record_ids: rec = load_experiment_record(rid) exp_id = rec.get_experiment_id() exp_rec_dict[exp_id].append(rid) return exp_rec_dict
def __getattr__(self, name): if name in get_global_experiment_library(): return get_global_experiment_library()[name] else: return _ExperimentConstructor(name)
def __call__(self, **kwargs): if self.name in get_global_experiment_library(): raise Exception("You tried to run create experiment '%s', but it already exists in the library. Give it another name!" % (self.name, )) return register_experiment(name = self.name, **kwargs)
def get_experiments(self): return get_global_experiment_library()