def register_result(self, name, value):
if os.path.exists(self.results_file):
results = Config.from_json(self.results_file).as_flat_dict()
else:
results = dict()
results[name] = value
Config.from_flat_dict(results).to_json(self.results_file)
def test_config_with_boolean_field(nested_dict_config):
config = Config.from_dict(nested_dict_config)
config.d = Config.from_dict({"d": False})
# identifier should be sorted as follows:
# a, c.a, c.b, c.c
# _b is ignored as it starts from underscore
assert config.identifier == "10|10|1x2x3|a|no_d"
def test_config_with_boolean_field(nested_dict_config):
config = Config.from_dict(nested_dict_config)
config.d = Config.from_dict({"d": False})
# identifier should be sorted as follows:
# a, c.a, c.b, c.c
# _b is ignored as it starts from underscore
assert config.identifier == DEFAULT_SEPARATOR.join(
"10 10 1x2x3 a no_d".split())
def test_config_to_json(nested_dict_config, tmpdir):
filepath = tmpdir.join("nested_dict_config.json").strpath
config = Config.from_dict(nested_dict_config)
config.to_json(filepath)
recovered_config = Config.from_json(filepath)
assert config.to_dict() == recovered_config.to_dict()
def test_config_from_dict(simple_dict_config, nested_dict_config):
config = Config.from_dict(simple_dict_config)
assert config.a == 10
assert config.b == [1, 2, 3]
assert config.c == "a"
config = Config.from_dict(nested_dict_config)
assert config.a == 10
assert config._b == "a"
assert config.c.a == 10
assert config.c.b == [1, 2, 3]
def test_custom_separator(nested_dict_config):
use_custom_separator("-")
config = Config.from_dict(nested_dict_config)
config.d = Config.from_dict({"d": False})
assert config.identifier == "10-10-1x2x3-a-no_d"
use_default_separator()
config = Config.from_dict(nested_dict_config)
config.d = Config.from_dict({"d": False})
assert config.identifier == "10|10|1x2x3|a|no_d"
def test_custom_separator(nested_dict_config):
use_custom_separator("-")
config = Config.from_dict(nested_dict_config)
config.d = Config.from_dict({"d": False})
assert config.identifier == "10-10-1x2x3-a-no_d"
use_default_separator()
config = Config.from_dict(nested_dict_config)
config.d = Config.from_dict({"d": False})
assert config.identifier == DEFAULT_SEPARATOR.join(
"10 10 1x2x3 a no_d".split())
def test_config_indentifier(nested_dict_config):
config = Config.from_dict(nested_dict_config)
# identifier should be sorted as follows:
# a, c.a, c.b, c.c
# _b is ignored as it starts from underscore
assert config.identifier == DEFAULT_SEPARATOR.join("10 10 1x2x3 a".split())
def test_config_indentifier(nested_dict_config):
config = Config.from_dict(nested_dict_config)
# identifier should be sorted as follows:
# a, c.a, c.b, c.c
# _b is ignored as it starts from underscore
assert config.identifier == "10|10|1x2x3|a"
def test_config_from_flat_dict():
config = {"a.a": 10, "b": "b", "a.b": [1, 2, 3], "a.c.c": 1}
config = Config.from_flat_dict(config)
assert config.a.a == 10
assert config.b == "b"
assert config.a.b == [1, 2, 3]
assert config.a.c.c == 1
def test_experiment_initialization(nested_dict_config, tmpdir):
experiments_dir = tmpdir.join("experiments").strpath
experiment = Experiment(nested_dict_config,
experiments_dir=experiments_dir)
config = Config.from_json(
os.path.join(experiments_dir, experiment.config.identifier,
"config.json"))
assert config.to_dict() == nested_dict_config
def test_config_from_json(nested_dict_config, tmpdir):
filepath = tmpdir.join("nested_dict_config.json").strpath
with open(filepath, "w") as f:
json.dump(nested_dict_config, f, indent=4)
config = Config.from_json(filepath)
assert config.a == 10
assert config._b == "a"
assert config.c.a == 10
assert config.c.b == [1, 2, 3]
def collect_results(directory, metrics):
experiments = os.listdir(directory)
if experiments:
fullpath = os.path.abspath(directory)
print(bold("\nResults for {directory}:".format(directory=fullpath)))
print()
all_results = defaultdict(list)
for experiment in experiments:
results_file = os.path.join(directory, experiment, "results.json")
if os.path.isfile(results_file):
result = Config.from_json(results_file)
result = result.as_flat_dict()
all_results["experiment"].append(experiment)
for metric in metrics:
all_results[metric].append(result.get(metric, ""))
return all_results
styles = Config.from_dict(
dict(
reset="\033[0m",
bold="\033[01m",
disable="\033[02m",
underline="\033[04m",
reverse="\033[07m",
strikethrough="\033[09m",
invisible="\033[08m",
fg=dict(
black="\033[30m",
red="\033[31m",
green="\033[32m",
orange="\033[33m",
blue="\033[34m",
purple="\033[35m",
cyan="\033[36m",
lightgrey="\033[37m",
darkgrey="\033[90m",
lightred="\033[91m",
lightgreen="\033[92m",
yellow="\033[93m",
lightblue="\033[94m",
pink="\033[95m",
lightcyan="\033[96m"
),
bg=dict(
black="\033[40m",
red="\033[41m",
green="\033[42m",
orange="\033[43m",
blue="\033[44m",
purple="\033[45m",
cyan="\033[46m",
lightgrey="\033[47m"
)
)
)
def results(self):
return Config.from_json(self.results_file)
def test_config_as_flat_dict(nested_dict_config):
flat_dict = Config.from_dict(nested_dict_config).as_flat_dict()
assert flat_dict["c.a"] == 10
assert flat_dict["c.b"] == [1, 2, 3]
def test_config_to_dict(nested_dict_config):
config = Config.from_dict(nested_dict_config)
recovered_dict = config.to_dict()
assert nested_dict_config == recovered_dict
def __init__(self,
config=None,
resume_from=None,
logfile_name="log",
experiments_dir="./experiments",
implicit_resuming=False):
"""Create a new Experiment instance.
Args:
config: can be either a path to existing JSON file,
a dict, or an instance of mag.config.Config.
resume_from: an identifier (str) of an existing experiment or direct
path to an existing experiment. In the case when experiment
identifier is provided, experiment directory is assumed to be
located at experiments_dir / identifier.
logfile_name: str, naming for log file. This can be useful to
separate logs for different runs on the same experiment
experiments_dir: str, a path where experiment will be saved
implicit_resuming: bool, whether to allow resuming
even if experiment already exists
"""
self.experiments_dir = experiments_dir
self.logfile_name = logfile_name
if config is None and resume_from is None:
raise ValueError("If `config` argument was not passed explicitly, "
"indentifier of existing experiment "
"should be specified by `resume_from` argument.")
elif config is not None and resume_from is None:
if isinstance(config, str) and os.path.isfile(config):
self.config = Config.from_json(config)
elif isinstance(config, dict):
self.config = Config.from_dict(config)
elif isinstance(config, Config):
self.config = config
else:
raise ValueError(
"`config` should be either a path to JSON file, "
"a dictonary, or an instance of mag.config.Config")
if os.path.isdir(self.experiment_dir):
if not implicit_resuming:
raise ValueError(
"Experiment with identifier {identifier} "
"already exists. Set `resume_from` to the corresponding "
"identifier (directory name) {directory} or delete it "
"manually and then rerun the code.".format(
identifier=self.config.identifier,
directory=self.config.identifier))
else:
self._makedir()
self._save_config()
self._save_git_commit_hash()
self._save_command()
elif resume_from is not None and config is None:
if is_experiment(resume_from):
experiment_directory = resume_from
_candidate_experiments_dir = self._infer_experiments_dir(
experiment_directory)
self.experiments_dir = _candidate_experiments_dir
else:
experiment_directory = os.path.join(experiments_dir,
resume_from)
self.config = Config.from_json(
os.path.join(experiment_directory, "config.json"))
self._register_existing_directories()
elif config is not None and resume_from is not None:
raise ValueError(
"`config` and `resume_from` arguments are mutually "
"exclusive: either create new experiment (by passing only "
"`config`) or resume from the existing experiment "
"(by passing only `resume_from`)")