def get_config_downloads(config_path):
dp_root_back = get_deeppavlov_root()
config = read_json(config_path)
set_deeppavlov_root(config)
downloads = set()
if 'metadata' in config and 'download' in config['metadata']:
for resource in config['metadata']['download']:
if isinstance(resource, str):
resource = {
'url': resource
}
url = resource['url']
dest = expand_path(resource.get('subdir', ''))
downloads.add((url, dest))
config_references = [expand_path(config_ref) for config_ref in get_all_elems_from_json(config, 'config_path')]
downloads |= {(url, dest) for config in config_references for url, dest in get_config_downloads(config)}
set_deeppavlov_root({'deeppavlov_root': dp_root_back})
return downloads
def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component:
"""Builds and returns the Component from corresponding dictionary of parameters."""
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError(
'Component with id "{id}" was referenced but not initialized'.
format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model_from_config
deeppavlov_root = get_deeppavlov_root()
refs = _refs.copy()
_refs.clear()
config = read_json(expand_path(config_params['config_path']))
model = build_model_from_config(config)
set_deeppavlov_root({'deeppavlov_root': deeppavlov_root})
_refs.clear()
_refs.update(refs)
return model
elif 'class' in config_params:
cls = cls_from_str(config_params.pop('class'))
else:
cls_name = config_params.pop('name', None)
if not cls_name:
e = ConfigError(
'Component config has no `name` nor `ref` or `class` fields')
log.exception(e)
raise e
cls = get_model(cls_name)
# find the submodels params recursively
config_params = {k: _init_param(v, mode) for k, v in config_params.items()}
try:
spec = inspect.getfullargspec(cls)
if 'mode' in spec.args + spec.kwonlyargs or spec.varkw is not None:
kwargs['mode'] = mode
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component
def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component:
"""Builds and returns the Component from corresponding dictionary of parameters."""
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError('Component with id "{id}" was referenced but not initialized'
.format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model_from_config
deeppavlov_root = get_deeppavlov_root()
refs = _refs.copy()
_refs.clear()
config = read_json(expand_path(config_params['config_path']))
model = build_model_from_config(config, as_component=True)
set_deeppavlov_root({'deeppavlov_root': deeppavlov_root})
_refs.clear()
_refs.update(refs)
return model
elif 'class' in config_params:
cls = cls_from_str(config_params.pop('class'))
else:
cls_name = config_params.pop('name', None)
if not cls_name:
e = ConfigError('Component config has no `name` nor `ref` or `class` fields')
log.exception(e)
raise e
cls = get_model(cls_name)
# find the submodels params recursively
config_params = {k: _init_param(v, mode) for k, v in config_params.items()}
try:
spec = inspect.getfullargspec(cls)
if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None:
kwargs['mode'] = mode
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component
def __init__(self, **kwargs):
""" Initialize the model and additional parent classes attributes
Args:
**kwargs: a dictionary containing parameters for model and parameters for training
it formed from json config file part that correspond to your model.
"""
# Parameters for parent classes
save_path = kwargs.get('save_path', None)
load_path = kwargs.get('load_path', None)
train_now = kwargs.get('train_now', None)
mode = kwargs.get('mode', None)
# Call parent constructors. Results in addition of attributes (save_path,
# load_path, train_now, mode to current instance) and creation of save_folder
# if it doesn't exist
super().__init__(save_path=save_path,
load_path=load_path,
train_now=train_now,
mode=mode)
nltk.download('punkt',
download_dir=str(get_deeppavlov_root().resolve()))
opt = deepcopy(kwargs)
self.train_now = opt['train_now']
self.opt = opt
self.interact_pred_num = opt['interact_pred_num']
self.vocabs = opt.get('vocabs', None)
dict_parameter_names = list(
inspect.signature(InsuranceDict.__init__).parameters)
dict_parameters = {
par: opt[par]
for par in dict_parameter_names if par in opt
}
network_parameter_names = list(
inspect.signature(RankingNetwork.__init__).parameters)
self.network_parameters = {
par: opt[par]
for par in network_parameter_names if par in opt
}
self.dict = InsuranceDict(**dict_parameters)
self.load()
train_parameters_names = list(
inspect.signature(self._net.train_on_batch).parameters)
self.train_parameters = {
par: opt[par]
for par in train_parameters_names if par in opt
}
def from_params(params: Dict, **kwargs) -> Component:
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError(
'Component with id "{id}" was referenced but not initialized'.
format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model_from_config
deeppavlov_root = get_deeppavlov_root()
config = read_json(expand_path(config_params['config_path']))
model = build_model_from_config(config, as_component=True)
set_deeppavlov_root({'deeppavlov_root': deeppavlov_root})
return model
elif 'class' in config_params:
c = config_params.pop('class')
try:
module_name, cls_name = c.split(':')
cls = getattr(importlib.import_module(module_name), cls_name)
except ValueError:
e = ConfigError(
'Expected class description in a `module.submodules:ClassName` form, but got `{}`'
.format(c))
log.exception(e)
raise e
else:
cls_name = config_params.pop('name', None)
if not cls_name:
e = ConfigError(
'Component config has no `name` nor `ref` or `class` fields')
log.exception(e)
raise e
try:
cls = REGISTRY[cls_name]
except KeyError:
e = ConfigError('Class {} is not registered.'.format(cls_name))
log.exception(e)
raise e
# find the submodels params recursively
for param_name, subcls_params in config_params.items():
if isinstance(subcls_params, dict):
if not {'ref', 'name', 'class', 'config_path'
}.intersection(subcls_params):
"This parameter is passed as dict to the class constructor."
" The user didn't intent it to be a component."
for k, v in subcls_params.items():
subcls_params[k] = _resolve(v)
continue
config_params[param_name] = from_params(subcls_params,
vocabs=kwargs['vocabs'],
mode=kwargs['mode'])
try:
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component
def from_params(params: Dict, mode='infer', **kwargs) -> Component:
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError('Component with id "{id}" was referenced but not initialized'
.format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model_from_config
deeppavlov_root = get_deeppavlov_root()
refs = _refs.copy()
_refs.clear()
config = read_json(expand_path(config_params['config_path']))
model = build_model_from_config(config, as_component=True)
set_deeppavlov_root({'deeppavlov_root': deeppavlov_root})
_refs.clear()
_refs.update(refs)
return model
elif 'class' in config_params:
c = config_params.pop('class')
try:
module_name, cls_name = c.split(':')
cls = getattr(importlib.import_module(module_name), cls_name)
except ValueError:
e = ConfigError('Expected class description in a `module.submodules:ClassName` form, but got `{}`'
.format(c))
log.exception(e)
raise e
else:
cls_name = config_params.pop('name', None)
if not cls_name:
e = ConfigError('Component config has no `name` nor `ref` or `class` fields')
log.exception(e)
raise e
try:
cls = REGISTRY[cls_name]
except KeyError:
e = ConfigError('Class {} is not registered.'.format(cls_name))
log.exception(e)
raise e
# find the submodels params recursively
config_params = {k: _init_param(v, mode) for k, v in config_params.items()}
try:
spec = inspect.getfullargspec(cls)
if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None:
kwargs['mode'] = mode
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component