def register_result(self, name, value): if os.path.exists(self.results_file): results = Config.from_json(self.results_file).as_flat_dict() else: results = dict() results[name] = value Config.from_flat_dict(results).to_json(self.results_file)
def test_config_with_boolean_field(nested_dict_config): config = Config.from_dict(nested_dict_config) config.d = Config.from_dict({"d": False}) # identifier should be sorted as follows: # a, c.a, c.b, c.c # _b is ignored as it starts from underscore assert config.identifier == "10|10|1x2x3|a|no_d"
def test_config_with_boolean_field(nested_dict_config): config = Config.from_dict(nested_dict_config) config.d = Config.from_dict({"d": False}) # identifier should be sorted as follows: # a, c.a, c.b, c.c # _b is ignored as it starts from underscore assert config.identifier == DEFAULT_SEPARATOR.join( "10 10 1x2x3 a no_d".split())
def test_config_to_json(nested_dict_config, tmpdir): filepath = tmpdir.join("nested_dict_config.json").strpath config = Config.from_dict(nested_dict_config) config.to_json(filepath) recovered_config = Config.from_json(filepath) assert config.to_dict() == recovered_config.to_dict()
def test_config_from_dict(simple_dict_config, nested_dict_config): config = Config.from_dict(simple_dict_config) assert config.a == 10 assert config.b == [1, 2, 3] assert config.c == "a" config = Config.from_dict(nested_dict_config) assert config.a == 10 assert config._b == "a" assert config.c.a == 10 assert config.c.b == [1, 2, 3]
def test_custom_separator(nested_dict_config): use_custom_separator("-") config = Config.from_dict(nested_dict_config) config.d = Config.from_dict({"d": False}) assert config.identifier == "10-10-1x2x3-a-no_d" use_default_separator() config = Config.from_dict(nested_dict_config) config.d = Config.from_dict({"d": False}) assert config.identifier == "10|10|1x2x3|a|no_d"
def test_custom_separator(nested_dict_config): use_custom_separator("-") config = Config.from_dict(nested_dict_config) config.d = Config.from_dict({"d": False}) assert config.identifier == "10-10-1x2x3-a-no_d" use_default_separator() config = Config.from_dict(nested_dict_config) config.d = Config.from_dict({"d": False}) assert config.identifier == DEFAULT_SEPARATOR.join( "10 10 1x2x3 a no_d".split())
def test_config_indentifier(nested_dict_config): config = Config.from_dict(nested_dict_config) # identifier should be sorted as follows: # a, c.a, c.b, c.c # _b is ignored as it starts from underscore assert config.identifier == DEFAULT_SEPARATOR.join("10 10 1x2x3 a".split())
def test_config_indentifier(nested_dict_config): config = Config.from_dict(nested_dict_config) # identifier should be sorted as follows: # a, c.a, c.b, c.c # _b is ignored as it starts from underscore assert config.identifier == "10|10|1x2x3|a"
def test_config_from_flat_dict(): config = {"a.a": 10, "b": "b", "a.b": [1, 2, 3], "a.c.c": 1} config = Config.from_flat_dict(config) assert config.a.a == 10 assert config.b == "b" assert config.a.b == [1, 2, 3] assert config.a.c.c == 1
def test_experiment_initialization(nested_dict_config, tmpdir): experiments_dir = tmpdir.join("experiments").strpath experiment = Experiment(nested_dict_config, experiments_dir=experiments_dir) config = Config.from_json( os.path.join(experiments_dir, experiment.config.identifier, "config.json")) assert config.to_dict() == nested_dict_config
def test_config_from_json(nested_dict_config, tmpdir): filepath = tmpdir.join("nested_dict_config.json").strpath with open(filepath, "w") as f: json.dump(nested_dict_config, f, indent=4) config = Config.from_json(filepath) assert config.a == 10 assert config._b == "a" assert config.c.a == 10 assert config.c.b == [1, 2, 3]
def collect_results(directory, metrics): experiments = os.listdir(directory) if experiments: fullpath = os.path.abspath(directory) print(bold("\nResults for {directory}:".format(directory=fullpath))) print() all_results = defaultdict(list) for experiment in experiments: results_file = os.path.join(directory, experiment, "results.json") if os.path.isfile(results_file): result = Config.from_json(results_file) result = result.as_flat_dict() all_results["experiment"].append(experiment) for metric in metrics: all_results[metric].append(result.get(metric, "")) return all_results
styles = Config.from_dict( dict( reset="\033[0m", bold="\033[01m", disable="\033[02m", underline="\033[04m", reverse="\033[07m", strikethrough="\033[09m", invisible="\033[08m", fg=dict( black="\033[30m", red="\033[31m", green="\033[32m", orange="\033[33m", blue="\033[34m", purple="\033[35m", cyan="\033[36m", lightgrey="\033[37m", darkgrey="\033[90m", lightred="\033[91m", lightgreen="\033[92m", yellow="\033[93m", lightblue="\033[94m", pink="\033[95m", lightcyan="\033[96m" ), bg=dict( black="\033[40m", red="\033[41m", green="\033[42m", orange="\033[43m", blue="\033[44m", purple="\033[45m", cyan="\033[46m", lightgrey="\033[47m" ) ) )
def results(self): return Config.from_json(self.results_file)
def test_config_as_flat_dict(nested_dict_config): flat_dict = Config.from_dict(nested_dict_config).as_flat_dict() assert flat_dict["c.a"] == 10 assert flat_dict["c.b"] == [1, 2, 3]
def test_config_to_dict(nested_dict_config): config = Config.from_dict(nested_dict_config) recovered_dict = config.to_dict() assert nested_dict_config == recovered_dict
def __init__(self, config=None, resume_from=None, logfile_name="log", experiments_dir="./experiments", implicit_resuming=False): """Create a new Experiment instance. Args: config: can be either a path to existing JSON file, a dict, or an instance of mag.config.Config. resume_from: an identifier (str) of an existing experiment or direct path to an existing experiment. In the case when experiment identifier is provided, experiment directory is assumed to be located at experiments_dir / identifier. logfile_name: str, naming for log file. This can be useful to separate logs for different runs on the same experiment experiments_dir: str, a path where experiment will be saved implicit_resuming: bool, whether to allow resuming even if experiment already exists """ self.experiments_dir = experiments_dir self.logfile_name = logfile_name if config is None and resume_from is None: raise ValueError("If `config` argument was not passed explicitly, " "indentifier of existing experiment " "should be specified by `resume_from` argument.") elif config is not None and resume_from is None: if isinstance(config, str) and os.path.isfile(config): self.config = Config.from_json(config) elif isinstance(config, dict): self.config = Config.from_dict(config) elif isinstance(config, Config): self.config = config else: raise ValueError( "`config` should be either a path to JSON file, " "a dictonary, or an instance of mag.config.Config") if os.path.isdir(self.experiment_dir): if not implicit_resuming: raise ValueError( "Experiment with identifier {identifier} " "already exists. Set `resume_from` to the corresponding " "identifier (directory name) {directory} or delete it " "manually and then rerun the code.".format( identifier=self.config.identifier, directory=self.config.identifier)) else: self._makedir() self._save_config() self._save_git_commit_hash() self._save_command() elif resume_from is not None and config is None: if is_experiment(resume_from): experiment_directory = resume_from _candidate_experiments_dir = self._infer_experiments_dir( experiment_directory) self.experiments_dir = _candidate_experiments_dir else: experiment_directory = os.path.join(experiments_dir, resume_from) self.config = Config.from_json( os.path.join(experiment_directory, "config.json")) self._register_existing_directories() elif config is not None and resume_from is not None: raise ValueError( "`config` and `resume_from` arguments are mutually " "exclusive: either create new experiment (by passing only " "`config`) or resume from the existing experiment " "(by passing only `resume_from`)")