def load(configuration, extra_args=None, pretty=False): a = config.args() if extra_args: b = a a = b a.update(b) context = {} for name, obj in configuration.items(): # handle modules with registry if hasattr(obj, 'registry'): obj = obj.registry if isinstance(obj, util.Registry): registry = obj registered = registry.registered default = registry.default if default is None and name not in a: raise ValueError(f'Missing argument for `{name}`') if a.get(name, default) not in registered: raise ValueError('no `{}` found for a `{}`'.format( a.get(name, default), name)) cls = registered[a.get(name, default)] else: cls = obj spec = inspect.getargspec(cls.__init__) args = [] for arg in spec.args: # HACK if arg == 'dataset' and name == 'valid_pred': arg = 'valid_ds' if arg == 'dataset' and name == 'test_pred': arg = 'test_ds' if arg == 'self': continue elif arg in context: args.append(context[arg]) elif arg == 'random': args.append( np.random.RandomState(int(config.args()['random_seed']))) elif arg == 'logger': logger_name = (name + ':' + a.get(name, default)).rstrip(':') args.append(log.Logger(logger_name)) elif arg == 'config': args.append(config.apply_config(name, a, cls)) else: raise ValueError(f'cannot match argument `{arg}` for `{cls}`') context[name] = cls(*args) if pretty: _log_config(configuration.keys(), context) return context
def path_model(ranker): modelspace = config.args()['modelspace'] path = os.path.join(get_working(), 'models', modelspace, ranker.path_segment()) if not os.path.exists(path): os.makedirs(path) with open(os.path.join(path, 'config.json'), 'wt') as f: json.dump(ranker.config, f) with open(os.path.join(path, 'structure.txt'), 'wt') as f: f.write(repr(ranker)) return path
def path_log(): path = os.path.join(get_working(), 'logs') os.makedirs(path, exist_ok=True) path = os.path.join(path, config.args()['runid'] + '.log') return path
def path_modelspace(): modelspace = config.args()['modelspace'] path = os.path.join(get_working(), 'models', modelspace) os.makedirs(path, exist_ok=True) return path
def get_working(): base = config.args()['data_dir'] base = os.path.expanduser(base) os.makedirs(base, exist_ok=True) return base