def load(configuration, extra_args=None, pretty=False):
a = config.args()
if extra_args:
b = a
a = b
a.update(b)
context = {}
for name, obj in configuration.items():
# handle modules with registry
if hasattr(obj, 'registry'):
obj = obj.registry
if isinstance(obj, util.Registry):
registry = obj
registered = registry.registered
default = registry.default
if default is None and name not in a:
raise ValueError(f'Missing argument for `{name}`')
if a.get(name, default) not in registered:
raise ValueError('no `{}` found for a `{}`'.format(
a.get(name, default), name))
cls = registered[a.get(name, default)]
else:
cls = obj
spec = inspect.getargspec(cls.__init__)
args = []
for arg in spec.args:
# HACK
if arg == 'dataset' and name == 'valid_pred':
arg = 'valid_ds'
if arg == 'dataset' and name == 'test_pred':
arg = 'test_ds'
if arg == 'self':
continue
elif arg in context:
args.append(context[arg])
elif arg == 'random':
args.append(
np.random.RandomState(int(config.args()['random_seed'])))
elif arg == 'logger':
logger_name = (name + ':' + a.get(name, default)).rstrip(':')
args.append(log.Logger(logger_name))
elif arg == 'config':
args.append(config.apply_config(name, a, cls))
else:
raise ValueError(f'cannot match argument `{arg}` for `{cls}`')
context[name] = cls(*args)
if pretty:
_log_config(configuration.keys(), context)
return context
def path_model(ranker):
modelspace = config.args()['modelspace']
path = os.path.join(get_working(), 'models', modelspace, ranker.path_segment())
if not os.path.exists(path):
os.makedirs(path)
with open(os.path.join(path, 'config.json'), 'wt') as f:
json.dump(ranker.config, f)
with open(os.path.join(path, 'structure.txt'), 'wt') as f:
f.write(repr(ranker))
return path
def path_log():
path = os.path.join(get_working(), 'logs')
os.makedirs(path, exist_ok=True)
path = os.path.join(path, config.args()['runid'] + '.log')
return path
def path_modelspace():
modelspace = config.args()['modelspace']
path = os.path.join(get_working(), 'models', modelspace)
os.makedirs(path, exist_ok=True)
return path
def get_working():
base = config.args()['data_dir']
base = os.path.expanduser(base)
os.makedirs(base, exist_ok=True)
return base