def register_result(self, name, value):
if os.path.exists(self.results_file):
results = Config.from_json(self.results_file).as_flat_dict()
else:
results = dict()
results[name] = value
Config.from_flat_dict(results).to_json(self.results_file)
def test_config_to_json(nested_dict_config, tmpdir):
filepath = tmpdir.join("nested_dict_config.json").strpath
config = Config.from_dict(nested_dict_config)
config.to_json(filepath)
recovered_config = Config.from_json(filepath)
assert config.to_dict() == recovered_config.to_dict()
def test_experiment_initialization(nested_dict_config, tmpdir):
experiments_dir = tmpdir.join("experiments").strpath
experiment = Experiment(nested_dict_config,
experiments_dir=experiments_dir)
config = Config.from_json(
os.path.join(experiments_dir, experiment.config.identifier,
"config.json"))
assert config.to_dict() == nested_dict_config
def test_config_from_json(nested_dict_config, tmpdir):
filepath = tmpdir.join("nested_dict_config.json").strpath
with open(filepath, "w") as f:
json.dump(nested_dict_config, f, indent=4)
config = Config.from_json(filepath)
assert config.a == 10
assert config._b == "a"
assert config.c.a == 10
assert config.c.b == [1, 2, 3]
def collect_results(directory, metrics):
experiments = os.listdir(directory)
if experiments:
fullpath = os.path.abspath(directory)
print(bold("\nResults for {directory}:".format(directory=fullpath)))
print()
all_results = defaultdict(list)
for experiment in experiments:
results_file = os.path.join(directory, experiment, "results.json")
if os.path.isfile(results_file):
result = Config.from_json(results_file)
result = result.as_flat_dict()
all_results["experiment"].append(experiment)
for metric in metrics:
all_results[metric].append(result.get(metric, ""))
return all_results
def __init__(self,
config=None,
resume_from=None,
logfile_name="log",
experiments_dir="./experiments",
implicit_resuming=False):
"""Create a new Experiment instance.
Args:
config: can be either a path to existing JSON file,
a dict, or an instance of mag.config.Config.
resume_from: an identifier (str) of an existing experiment or direct
path to an existing experiment. In the case when experiment
identifier is provided, experiment directory is assumed to be
located at experiments_dir / identifier.
logfile_name: str, naming for log file. This can be useful to
separate logs for different runs on the same experiment
experiments_dir: str, a path where experiment will be saved
implicit_resuming: bool, whether to allow resuming
even if experiment already exists
"""
self.experiments_dir = experiments_dir
self.logfile_name = logfile_name
if config is None and resume_from is None:
raise ValueError("If `config` argument was not passed explicitly, "
"indentifier of existing experiment "
"should be specified by `resume_from` argument.")
elif config is not None and resume_from is None:
if isinstance(config, str) and os.path.isfile(config):
self.config = Config.from_json(config)
elif isinstance(config, dict):
self.config = Config.from_dict(config)
elif isinstance(config, Config):
self.config = config
else:
raise ValueError(
"`config` should be either a path to JSON file, "
"a dictonary, or an instance of mag.config.Config")
if os.path.isdir(self.experiment_dir):
if not implicit_resuming:
raise ValueError(
"Experiment with identifier {identifier} "
"already exists. Set `resume_from` to the corresponding "
"identifier (directory name) {directory} or delete it "
"manually and then rerun the code.".format(
identifier=self.config.identifier,
directory=self.config.identifier))
else:
self._makedir()
self._save_config()
self._save_git_commit_hash()
self._save_command()
elif resume_from is not None and config is None:
if is_experiment(resume_from):
experiment_directory = resume_from
_candidate_experiments_dir = self._infer_experiments_dir(
experiment_directory)
self.experiments_dir = _candidate_experiments_dir
else:
experiment_directory = os.path.join(experiments_dir,
resume_from)
self.config = Config.from_json(
os.path.join(experiment_directory, "config.json"))
self._register_existing_directories()
elif config is not None and resume_from is not None:
raise ValueError(
"`config` and `resume_from` arguments are mutually "
"exclusive: either create new experiment (by passing only "
"`config`) or resume from the existing experiment "
"(by passing only `resume_from`)")
def results(self):
return Config.from_json(self.results_file)