def run(config_updates, mongodb_url="localhost:27017"):
"""Run a single experiment with the given configuration
Args:
config_updates (dict): Configuration updates
mongodb_url (str): MongoDB URL, or None if no Mongo observer should be used for
this run
"""
# Dynamically bind experiment config and main function
ex = Experiment()
ex.config(base_config)
ex.main(element_world_v4)
# Attach MongoDB observer if necessary
if mongodb_url is not None and not ex.observers:
ex.observers.append(MongoObserver(url=mongodb_url))
# Suppress warnings about padded MPDs
with warnings.catch_warnings():
warnings.filterwarnings(action="ignore", category=PaddedMDPWarning)
# Run the experiment
run = ex.run(config_updates=config_updates
) # , options={"--loglevel": "ERROR"})
# Return the result
return run.result
def __init__(self, f_main, f_config, f_capture, cfg,
mongo_url='127.0.0.1:27017', disable_logging=False):
self.cfg = cfg
curr_db_name = self.sacred_db_name()
ex_name = self.sacred_ex_name()
ex = Experiment(ex_name)
ex.captured_out_filter = apply_backspaces_and_linefeeds
if disable_logging == False:
print(f'Connecting to MongoDB at {mongo_url}:{curr_db_name}')
ex.observers.append(MongoObserver.create(url=mongo_url, db_name=curr_db_name))
# init the experiment configuration (params)
ex.config(f_config)
# init the experiment logging (capture) method
f_ex_capture = ex.capture(f_capture)
# init the experiment main
@ex.main
def ex_main(_run):
return main_wrapper(f_main, ex, f_ex_capture, self.sacred_db_name(), _run)
self.ex = ex
def sacred_hyperopt_objective(params):
"""
Objective to call with hyperopt
that uses sacred to log the experiment results
"""
ex = Experiment('example')
ex.config(expt_config)
ex.main(difficult_optimization_objective)
ex.observers.append(
MongoObserver.create(
url=f"{os.environ['MONGO_WRITE_IP']}:{os.environ['MONGO_PORT']}"))
run = ex.run(config_updates=params, options={"--loglevel": 40})
return {"loss": run.result, "status": hyperopt.STATUS_OK}
def run(config, mongodb_url="localhost:27017"):
"""Run a single experiment with the given configuration"""
# Dynamically bind experiment config and main function
ex = Experiment()
ex.config(base_config)
ex.main(canonical_puddle_world)
# Attach MongoDB observer if necessary
if not ex.observers:
ex.observers.append(MongoObserver(url=mongodb_url))
# Suppress warnings about padded MPDs
with warnings.catch_warnings():
warnings.filterwarnings(action="ignore", category=PaddedMDPWarning)
# Run the experiment
run = ex.run(config_updates=config, options={"--loglevel": "ERROR"})
# Return the result
return run.result
class SacredExperiment(object):
def __init__(
self,
experiment_name,
experiment_dir,
observer_type="file_storage",
mongo_url=None,
db_name=None,
):
"""__init__
:param experiment_name: The name of the experiments.
:param experiment_dir: The directory to store all the results of the experiments(This is for file_storage).
:param observer_type: The observer to record the results: the `file_storage` or `mongo`
:param mongo_url: The mongo url(for mongo observer)
:param db_name: The mongo url(for mongo observer)
"""
self.experiment_name = experiment_name
self.experiment = Experiment(self.experiment_name)
self.experiment_dir = experiment_dir
self.experiment.logger = get_module_logger("Sacred")
self.observer_type = observer_type
self.mongo_db_url = mongo_url
self.mongo_db_name = db_name
self._setup_experiment()
def _setup_experiment(self):
if self.observer_type == "file_storage":
file_storage_observer = FileStorageObserver.create(
basedir=self.experiment_dir)
self.experiment.observers.append(file_storage_observer)
elif self.observer_type == "mongo":
mongo_observer = MongoObserver.create(url=self.mongo_db_url,
db_name=self.mongo_db_name)
self.experiment.observers.append(mongo_observer)
else:
raise NotImplementedError("Unsupported observer type: {}".format(
self.observer_type))
def add_artifact(self, filename):
self.experiment.add_artifact(filename)
def add_info(self, key, value):
self.experiment.info[key] = value
def main_wrapper(self, func):
return self.experiment.main(func)
def config_wrapper(self, func):
return self.experiment.config(func)
multiple_values = get_multiple_values(configurations)
for index, configuration in enumerate(configurations):
global_vars.set_config(configuration)
if index+1 < global_vars.get('start_exp_idx'):
continue
if global_vars.get('exp_id'):
exp_id = global_vars.get('exp_id')
configuration['DEFAULT']['exp_id'] = exp_id
if FIRST_RUN:
FIRST_DATASET = global_vars.get('dataset')
if global_vars.get('include_params_folder_name'):
multiple_values.extend(global_vars.get('include_params_folder_name'))
FIRST_RUN = False
exp_name = f"{exp_id}_{index+1}_{experiment}"
exp_name = add_params_to_name(exp_name, multiple_values)
ex.config = {}
ex.add_config({**configuration, **{'tags': [exp_id]}})
if len(ex.observers) == 0 and not args.debug_mode:
ex.observers.append(MongoObserver.create(url=f'mongodb://{global_vars.get("mongodb_server")}'
f'/{global_vars.get("mongodb_name")}',
db_name=global_vars.get("mongodb_name")))
global_vars.set('sacred_ex', ex)
try:
run = ex.run(options={'--name': exp_name})
if not args.debug_mode:
exp_line = add_exp(exp_name, index+1, all_exps, run)
pd.DataFrame(all_exps).to_csv(f'reports/{exp_id}.csv', index=False)
if global_vars.get('upload_exp_results'):
upload_exp_results_to_gdrive(exp_line, 'University/Masters/Experiment Results/EEGNAS_results.xlsx')
except Exception as e:
print(f'failed experiment {exp_id}_{index+1}, continuing...')
'kwargs': {
'lr': training_settings['learning_rate'],
'betas': (0.9, 0.999),
'amsgrad': False,
'weight_decay': training_settings['weight_decay'],
}
}
trainer = {
'class': 'dcase2020_workshop.trainers.PTLTrainer',
'kwargs': {
'max_epochs': training_settings['epochs'],
'checkpoint_callback': False,
'logger': False,
'early_stop_callback': False,
'gpus': [0],
'show_progress_bar': True,
'progress_bar_refresh_rate': 1000
}
}
ex = Experiment('dcase2020_workshop_BaselineExperiment')
cfg = ex.config(configuration)
@ex.automain
def run(_config, _run):
experiment = BaselineExperiment(_config, _run)
return experiment.run()
import os
import numpy as np
import tensorflow as tf
from sacred import Experiment
from keras.optimizers import SGD
from keras.models import load_model
from callbacks import IncEpochsFileCallback, tb_logger
from config import config
ex = Experiment('jhu')
# connect the experiment to the imported modules
IncEpochsFileCallback.on_epoch_end = \
ex.capture(IncEpochsFileCallback.on_epoch_end)
tb_logger = ex.capture(tb_logger)
config = ex.config(config)
@ex.capture
def save(model, savefile):
model.save(savefile)
@ex.capture
def train(model, data, labels, epochs_elapsed, epochs, batch_size, n_test):
tb_log = tb_logger()
p_test = n_test / len(data)
model.fit( \
data, \
def __init__(self,
f_main,
f_config,
f_capture,
observer_type='file',
mongo_url='mongodb://localhost:27017',
verbose=False):
"""
:param f_main: function
The main function for the experiment
:param f_config: function or str
The function where all the sacred parameters are init or
a file with the config parameters
:param f_capture: function
The function that implements the metrics logging API with sacred
(should be used with Lambda in keras but has problem right now. Thus it can be ignored)
:param mongo_url: str
The url for MongoDB
:param verbose: bool
If True logging is enabled
"""
self.sacred_db_name()
ex = Experiment(self.sacred_ex_name())
ex.captured_out_filter = apply_backspaces_and_linefeeds
if observer_type == 'mongodb':
print('Connecting to MongoDB at {}:{}'.format(
mongo_url, self.sacred_db_name()))
ex.observers.append(
MongoObserver.create(url=mongo_url,
db_name=self.sacred_db_name()))
elif observer_type == 'file':
basedir = os.path.join(config['logs'], 'sacred')
ex.observers.append(FileStorageObserver.create(basedir))
else:
raise ValueError(
'{} is not a valid type for a SACRED observer.'.format(
observer_type))
if hasattr(f_config, '__call__'):
# init the experiment configuration using a function
ex.config(f_config)
elif isinstance(f_config, str):
# init the experiment configuration usinga file
ex.add_config(f_config)
elif isinstance(f_config, dict):
# init the experiment configuration usinga file
ex.add_config(f_config)
else:
raise ValueError(
'You should provide either a fucntion or a config file for setting up an experiemet.'
'The given paramter has type {} which is not valid'.format(
type(f_config)))
# init the experiment logging (capture) method
f_ex_capture = ex.capture(f_capture)
# init the experiment main
@ex.main
def ex_main(_run):
if observer_type == 'mongodb':
return main_wrapper(f_main, ex, f_ex_capture,
self.sacred_db_name(), _run)
else:
f_main(ex, _run, f_ex_capture)
self.ex = ex
def add_logging_config(experiment: sacred.Experiment, name: str) -> None:
experiment.add_config(
{"log_root": os.path.join(serialize.get_output_dir(), name)})
experiment.config(logging_config)
