def run(config_updates, mongodb_url="localhost:27017"): """Run a single experiment with the given configuration Args: config_updates (dict): Configuration updates mongodb_url (str): MongoDB URL, or None if no Mongo observer should be used for this run """ # Dynamically bind experiment config and main function ex = Experiment() ex.config(base_config) ex.main(element_world_v4) # Attach MongoDB observer if necessary if mongodb_url is not None and not ex.observers: ex.observers.append(MongoObserver(url=mongodb_url)) # Suppress warnings about padded MPDs with warnings.catch_warnings(): warnings.filterwarnings(action="ignore", category=PaddedMDPWarning) # Run the experiment run = ex.run(config_updates=config_updates ) # , options={"--loglevel": "ERROR"}) # Return the result return run.result
def __init__(self, f_main, f_config, f_capture, cfg, mongo_url='127.0.0.1:27017', disable_logging=False): self.cfg = cfg curr_db_name = self.sacred_db_name() ex_name = self.sacred_ex_name() ex = Experiment(ex_name) ex.captured_out_filter = apply_backspaces_and_linefeeds if disable_logging == False: print(f'Connecting to MongoDB at {mongo_url}:{curr_db_name}') ex.observers.append(MongoObserver.create(url=mongo_url, db_name=curr_db_name)) # init the experiment configuration (params) ex.config(f_config) # init the experiment logging (capture) method f_ex_capture = ex.capture(f_capture) # init the experiment main @ex.main def ex_main(_run): return main_wrapper(f_main, ex, f_ex_capture, self.sacred_db_name(), _run) self.ex = ex
def sacred_hyperopt_objective(params): """ Objective to call with hyperopt that uses sacred to log the experiment results """ ex = Experiment('example') ex.config(expt_config) ex.main(difficult_optimization_objective) ex.observers.append( MongoObserver.create( url=f"{os.environ['MONGO_WRITE_IP']}:{os.environ['MONGO_PORT']}")) run = ex.run(config_updates=params, options={"--loglevel": 40}) return {"loss": run.result, "status": hyperopt.STATUS_OK}
def run(config, mongodb_url="localhost:27017"): """Run a single experiment with the given configuration""" # Dynamically bind experiment config and main function ex = Experiment() ex.config(base_config) ex.main(canonical_puddle_world) # Attach MongoDB observer if necessary if not ex.observers: ex.observers.append(MongoObserver(url=mongodb_url)) # Suppress warnings about padded MPDs with warnings.catch_warnings(): warnings.filterwarnings(action="ignore", category=PaddedMDPWarning) # Run the experiment run = ex.run(config_updates=config, options={"--loglevel": "ERROR"}) # Return the result return run.result
class SacredExperiment(object): def __init__( self, experiment_name, experiment_dir, observer_type="file_storage", mongo_url=None, db_name=None, ): """__init__ :param experiment_name: The name of the experiments. :param experiment_dir: The directory to store all the results of the experiments(This is for file_storage). :param observer_type: The observer to record the results: the `file_storage` or `mongo` :param mongo_url: The mongo url(for mongo observer) :param db_name: The mongo url(for mongo observer) """ self.experiment_name = experiment_name self.experiment = Experiment(self.experiment_name) self.experiment_dir = experiment_dir self.experiment.logger = get_module_logger("Sacred") self.observer_type = observer_type self.mongo_db_url = mongo_url self.mongo_db_name = db_name self._setup_experiment() def _setup_experiment(self): if self.observer_type == "file_storage": file_storage_observer = FileStorageObserver.create( basedir=self.experiment_dir) self.experiment.observers.append(file_storage_observer) elif self.observer_type == "mongo": mongo_observer = MongoObserver.create(url=self.mongo_db_url, db_name=self.mongo_db_name) self.experiment.observers.append(mongo_observer) else: raise NotImplementedError("Unsupported observer type: {}".format( self.observer_type)) def add_artifact(self, filename): self.experiment.add_artifact(filename) def add_info(self, key, value): self.experiment.info[key] = value def main_wrapper(self, func): return self.experiment.main(func) def config_wrapper(self, func): return self.experiment.config(func)
multiple_values = get_multiple_values(configurations) for index, configuration in enumerate(configurations): global_vars.set_config(configuration) if index+1 < global_vars.get('start_exp_idx'): continue if global_vars.get('exp_id'): exp_id = global_vars.get('exp_id') configuration['DEFAULT']['exp_id'] = exp_id if FIRST_RUN: FIRST_DATASET = global_vars.get('dataset') if global_vars.get('include_params_folder_name'): multiple_values.extend(global_vars.get('include_params_folder_name')) FIRST_RUN = False exp_name = f"{exp_id}_{index+1}_{experiment}" exp_name = add_params_to_name(exp_name, multiple_values) ex.config = {} ex.add_config({**configuration, **{'tags': [exp_id]}}) if len(ex.observers) == 0 and not args.debug_mode: ex.observers.append(MongoObserver.create(url=f'mongodb://{global_vars.get("mongodb_server")}' f'/{global_vars.get("mongodb_name")}', db_name=global_vars.get("mongodb_name"))) global_vars.set('sacred_ex', ex) try: run = ex.run(options={'--name': exp_name}) if not args.debug_mode: exp_line = add_exp(exp_name, index+1, all_exps, run) pd.DataFrame(all_exps).to_csv(f'reports/{exp_id}.csv', index=False) if global_vars.get('upload_exp_results'): upload_exp_results_to_gdrive(exp_line, 'University/Masters/Experiment Results/EEGNAS_results.xlsx') except Exception as e: print(f'failed experiment {exp_id}_{index+1}, continuing...')
'kwargs': { 'lr': training_settings['learning_rate'], 'betas': (0.9, 0.999), 'amsgrad': False, 'weight_decay': training_settings['weight_decay'], } } trainer = { 'class': 'dcase2020_workshop.trainers.PTLTrainer', 'kwargs': { 'max_epochs': training_settings['epochs'], 'checkpoint_callback': False, 'logger': False, 'early_stop_callback': False, 'gpus': [0], 'show_progress_bar': True, 'progress_bar_refresh_rate': 1000 } } ex = Experiment('dcase2020_workshop_BaselineExperiment') cfg = ex.config(configuration) @ex.automain def run(_config, _run): experiment = BaselineExperiment(_config, _run) return experiment.run()
import os import numpy as np import tensorflow as tf from sacred import Experiment from keras.optimizers import SGD from keras.models import load_model from callbacks import IncEpochsFileCallback, tb_logger from config import config ex = Experiment('jhu') # connect the experiment to the imported modules IncEpochsFileCallback.on_epoch_end = \ ex.capture(IncEpochsFileCallback.on_epoch_end) tb_logger = ex.capture(tb_logger) config = ex.config(config) @ex.capture def save(model, savefile): model.save(savefile) @ex.capture def train(model, data, labels, epochs_elapsed, epochs, batch_size, n_test): tb_log = tb_logger() p_test = n_test / len(data) model.fit( \ data, \
def __init__(self, f_main, f_config, f_capture, observer_type='file', mongo_url='mongodb://localhost:27017', verbose=False): """ :param f_main: function The main function for the experiment :param f_config: function or str The function where all the sacred parameters are init or a file with the config parameters :param f_capture: function The function that implements the metrics logging API with sacred (should be used with Lambda in keras but has problem right now. Thus it can be ignored) :param mongo_url: str The url for MongoDB :param verbose: bool If True logging is enabled """ self.sacred_db_name() ex = Experiment(self.sacred_ex_name()) ex.captured_out_filter = apply_backspaces_and_linefeeds if observer_type == 'mongodb': print('Connecting to MongoDB at {}:{}'.format( mongo_url, self.sacred_db_name())) ex.observers.append( MongoObserver.create(url=mongo_url, db_name=self.sacred_db_name())) elif observer_type == 'file': basedir = os.path.join(config['logs'], 'sacred') ex.observers.append(FileStorageObserver.create(basedir)) else: raise ValueError( '{} is not a valid type for a SACRED observer.'.format( observer_type)) if hasattr(f_config, '__call__'): # init the experiment configuration using a function ex.config(f_config) elif isinstance(f_config, str): # init the experiment configuration usinga file ex.add_config(f_config) elif isinstance(f_config, dict): # init the experiment configuration usinga file ex.add_config(f_config) else: raise ValueError( 'You should provide either a fucntion or a config file for setting up an experiemet.' 'The given paramter has type {} which is not valid'.format( type(f_config))) # init the experiment logging (capture) method f_ex_capture = ex.capture(f_capture) # init the experiment main @ex.main def ex_main(_run): if observer_type == 'mongodb': return main_wrapper(f_main, ex, f_ex_capture, self.sacred_db_name(), _run) else: f_main(ex, _run, f_ex_capture) self.ex = ex
def add_logging_config(experiment: sacred.Experiment, name: str) -> None: experiment.add_config( {"log_root": os.path.join(serialize.get_output_dir(), name)}) experiment.config(logging_config)