def get_config_downloads(config_path): dp_root_back = get_deeppavlov_root() config = read_json(config_path) set_deeppavlov_root(config) downloads = set() if 'metadata' in config and 'download' in config['metadata']: for resource in config['metadata']['download']: if isinstance(resource, str): resource = { 'url': resource } url = resource['url'] dest = expand_path(resource.get('subdir', '')) downloads.add((url, dest)) config_references = [expand_path(config_ref) for config_ref in get_all_elems_from_json(config, 'config_path')] downloads |= {(url, dest) for config in config_references for url, dest in get_config_downloads(config)} set_deeppavlov_root({'deeppavlov_root': dp_root_back}) return downloads
def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component: """Builds and returns the Component from corresponding dictionary of parameters.""" # what is passed in json: config_params = {k: _resolve(v) for k, v in params.items()} # get component by reference (if any) if 'ref' in config_params: try: return _refs[config_params['ref']] except KeyError: e = ConfigError( 'Component with id "{id}" was referenced but not initialized'. format(id=config_params['ref'])) log.exception(e) raise e elif 'config_path' in config_params: from deeppavlov.core.commands.infer import build_model_from_config deeppavlov_root = get_deeppavlov_root() refs = _refs.copy() _refs.clear() config = read_json(expand_path(config_params['config_path'])) model = build_model_from_config(config) set_deeppavlov_root({'deeppavlov_root': deeppavlov_root}) _refs.clear() _refs.update(refs) return model elif 'class' in config_params: cls = cls_from_str(config_params.pop('class')) else: cls_name = config_params.pop('name', None) if not cls_name: e = ConfigError( 'Component config has no `name` nor `ref` or `class` fields') log.exception(e) raise e cls = get_model(cls_name) # find the submodels params recursively config_params = {k: _init_param(v, mode) for k, v in config_params.items()} try: spec = inspect.getfullargspec(cls) if 'mode' in spec.args + spec.kwonlyargs or spec.varkw is not None: kwargs['mode'] = mode component = cls(**dict(config_params, **kwargs)) try: _refs[config_params['id']] = component except KeyError: pass except Exception: log.exception("Exception in {}".format(cls)) raise return component
def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component: """Builds and returns the Component from corresponding dictionary of parameters.""" # what is passed in json: config_params = {k: _resolve(v) for k, v in params.items()} # get component by reference (if any) if 'ref' in config_params: try: return _refs[config_params['ref']] except KeyError: e = ConfigError('Component with id "{id}" was referenced but not initialized' .format(id=config_params['ref'])) log.exception(e) raise e elif 'config_path' in config_params: from deeppavlov.core.commands.infer import build_model_from_config deeppavlov_root = get_deeppavlov_root() refs = _refs.copy() _refs.clear() config = read_json(expand_path(config_params['config_path'])) model = build_model_from_config(config, as_component=True) set_deeppavlov_root({'deeppavlov_root': deeppavlov_root}) _refs.clear() _refs.update(refs) return model elif 'class' in config_params: cls = cls_from_str(config_params.pop('class')) else: cls_name = config_params.pop('name', None) if not cls_name: e = ConfigError('Component config has no `name` nor `ref` or `class` fields') log.exception(e) raise e cls = get_model(cls_name) # find the submodels params recursively config_params = {k: _init_param(v, mode) for k, v in config_params.items()} try: spec = inspect.getfullargspec(cls) if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None: kwargs['mode'] = mode component = cls(**dict(config_params, **kwargs)) try: _refs[config_params['id']] = component except KeyError: pass except Exception: log.exception("Exception in {}".format(cls)) raise return component
def __init__(self, **kwargs): """ Initialize the model and additional parent classes attributes Args: **kwargs: a dictionary containing parameters for model and parameters for training it formed from json config file part that correspond to your model. """ # Parameters for parent classes save_path = kwargs.get('save_path', None) load_path = kwargs.get('load_path', None) train_now = kwargs.get('train_now', None) mode = kwargs.get('mode', None) # Call parent constructors. Results in addition of attributes (save_path, # load_path, train_now, mode to current instance) and creation of save_folder # if it doesn't exist super().__init__(save_path=save_path, load_path=load_path, train_now=train_now, mode=mode) nltk.download('punkt', download_dir=str(get_deeppavlov_root().resolve())) opt = deepcopy(kwargs) self.train_now = opt['train_now'] self.opt = opt self.interact_pred_num = opt['interact_pred_num'] self.vocabs = opt.get('vocabs', None) dict_parameter_names = list( inspect.signature(InsuranceDict.__init__).parameters) dict_parameters = { par: opt[par] for par in dict_parameter_names if par in opt } network_parameter_names = list( inspect.signature(RankingNetwork.__init__).parameters) self.network_parameters = { par: opt[par] for par in network_parameter_names if par in opt } self.dict = InsuranceDict(**dict_parameters) self.load() train_parameters_names = list( inspect.signature(self._net.train_on_batch).parameters) self.train_parameters = { par: opt[par] for par in train_parameters_names if par in opt }
def from_params(params: Dict, **kwargs) -> Component: # what is passed in json: config_params = {k: _resolve(v) for k, v in params.items()} # get component by reference (if any) if 'ref' in config_params: try: return _refs[config_params['ref']] except KeyError: e = ConfigError( 'Component with id "{id}" was referenced but not initialized'. format(id=config_params['ref'])) log.exception(e) raise e elif 'config_path' in config_params: from deeppavlov.core.commands.infer import build_model_from_config deeppavlov_root = get_deeppavlov_root() config = read_json(expand_path(config_params['config_path'])) model = build_model_from_config(config, as_component=True) set_deeppavlov_root({'deeppavlov_root': deeppavlov_root}) return model elif 'class' in config_params: c = config_params.pop('class') try: module_name, cls_name = c.split(':') cls = getattr(importlib.import_module(module_name), cls_name) except ValueError: e = ConfigError( 'Expected class description in a `module.submodules:ClassName` form, but got `{}`' .format(c)) log.exception(e) raise e else: cls_name = config_params.pop('name', None) if not cls_name: e = ConfigError( 'Component config has no `name` nor `ref` or `class` fields') log.exception(e) raise e try: cls = REGISTRY[cls_name] except KeyError: e = ConfigError('Class {} is not registered.'.format(cls_name)) log.exception(e) raise e # find the submodels params recursively for param_name, subcls_params in config_params.items(): if isinstance(subcls_params, dict): if not {'ref', 'name', 'class', 'config_path' }.intersection(subcls_params): "This parameter is passed as dict to the class constructor." " The user didn't intent it to be a component." for k, v in subcls_params.items(): subcls_params[k] = _resolve(v) continue config_params[param_name] = from_params(subcls_params, vocabs=kwargs['vocabs'], mode=kwargs['mode']) try: component = cls(**dict(config_params, **kwargs)) try: _refs[config_params['id']] = component except KeyError: pass except Exception: log.exception("Exception in {}".format(cls)) raise return component
def from_params(params: Dict, mode='infer', **kwargs) -> Component: # what is passed in json: config_params = {k: _resolve(v) for k, v in params.items()} # get component by reference (if any) if 'ref' in config_params: try: return _refs[config_params['ref']] except KeyError: e = ConfigError('Component with id "{id}" was referenced but not initialized' .format(id=config_params['ref'])) log.exception(e) raise e elif 'config_path' in config_params: from deeppavlov.core.commands.infer import build_model_from_config deeppavlov_root = get_deeppavlov_root() refs = _refs.copy() _refs.clear() config = read_json(expand_path(config_params['config_path'])) model = build_model_from_config(config, as_component=True) set_deeppavlov_root({'deeppavlov_root': deeppavlov_root}) _refs.clear() _refs.update(refs) return model elif 'class' in config_params: c = config_params.pop('class') try: module_name, cls_name = c.split(':') cls = getattr(importlib.import_module(module_name), cls_name) except ValueError: e = ConfigError('Expected class description in a `module.submodules:ClassName` form, but got `{}`' .format(c)) log.exception(e) raise e else: cls_name = config_params.pop('name', None) if not cls_name: e = ConfigError('Component config has no `name` nor `ref` or `class` fields') log.exception(e) raise e try: cls = REGISTRY[cls_name] except KeyError: e = ConfigError('Class {} is not registered.'.format(cls_name)) log.exception(e) raise e # find the submodels params recursively config_params = {k: _init_param(v, mode) for k, v in config_params.items()} try: spec = inspect.getfullargspec(cls) if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None: kwargs['mode'] = mode component = cls(**dict(config_params, **kwargs)) try: _refs[config_params['id']] = component except KeyError: pass except Exception: log.exception("Exception in {}".format(cls)) raise return component