def run_population(population, evolution, gpus):
"""
Change save and load paths for obtained population, save config.json with model config,
run population via current python executor (with which evolve.py already run)
and on given devices (-1 means CPU, other integeres - visible for evolve.py GPUs)
Args:
population: list of dictionaries - configs of current population
evolution: ParamsEvolution
gpus: list of given devices (list of integers)
Returns:
None
"""
population_size = len(population)
for k in range(population_size // len(gpus) + 1):
procs = []
for j in range(len(gpus)):
i = k * len(gpus) + j
if i < population_size:
save_path = expand_path(
evolution.get_value_from_config(
parse_config(population[i]),
evolution.path_to_models_save_path))
save_path.mkdir(parents=True, exist_ok=True)
f_name = save_path / "config.json"
save_json(population[i], f_name)
with save_path.joinpath('out.txt').open('w', encoding='utf8') as outlog,\
save_path.joinpath('err.txt').open('w', encoding='utf8') as errlog:
env = dict(os.environ)
if len(gpus) > 1 or gpus[0] != -1:
env['CUDA_VISIBLE_DEVICES'] = str(gpus[j])
procs.append(
Popen("{} -m deeppavlov train {}".format(
sys.executable, str(f_name)),
shell=True,
stdout=outlog,
stderr=errlog,
env=env))
for j, proc in enumerate(procs):
i = k * len(gpus) + j
log.info(f'Waiting on {i}th proc')
if proc.wait() != 0:
save_path = expand_path(
evolution.get_value_from_config(
parse_config(population[i]),
evolution.path_to_models_save_path))
with save_path.joinpath('err.txt').open(
encoding='utf8') as errlog:
log.warning(
f'Population {i} returned an error code {proc.returncode} and an error log:\n'
+ errlog.read())
return None
def get_server_params(model_config: Union[str, Path]) -> Dict:
server_config = read_json(SERVER_CONFIG_PATH)
model_config = parse_config(model_config)
server_params = server_config['common_defaults']
if check_nested_dict_keys(model_config, ['metadata', 'server_utils']):
model_tag = model_config['metadata']['server_utils']
if check_nested_dict_keys(server_config,
['model_defaults', model_tag]):
model_defaults = server_config['model_defaults'][model_tag]
for param_name in model_defaults.keys():
if model_defaults[param_name]:
server_params[param_name] = model_defaults[param_name]
server_params['model_endpoint'] = server_params.get(
'model_endpoint', '/model')
arg_names = server_params['model_args_names'] or model_config['chainer'][
'in']
if isinstance(arg_names, str):
arg_names = [arg_names]
server_params['model_args_names'] = arg_names
return server_params
def save(self, path, copy_model=False):
if not os.path.isdir(path):
os.makedirs(path)
settings = {
'self_class': type(self).__name__,
'dist_class': type(self.dist).__name__,
'linearization_settings': self.dist.get_linearization_settings()
}
with open(os.path.join(path, 'settings.json'), 'w') as f:
json.dump(settings, f, indent=True)
model_settings = self.model_settings.copy()
if copy_model:
model_settings = parse_config(model_settings)
model_settings['metadata']['download'] = []
with open(os.path.join(path, 'model_settings.json'), 'w') as f:
json.dump(model_settings, f, indent=True)
Doc2Vec.save(self.doc2vec, os.path.join(path, 'doc2vec.model'))
with open(os.path.join(path, 'tree.model'), 'wb') as f:
pickle.dump(self.tree, f)
def get_config_downloads(
config: Union[str, Path, dict]) -> Set[Tuple[str, Path]]:
config = parse_config(config)
downloads = set()
if 'metadata' in config and 'download' in config['metadata']:
for resource in config['metadata']['download']:
if isinstance(resource, str):
resource = {'url': resource}
url = resource['url']
dest = expand_path(resource.get('subdir', ''))
downloads.add((url, dest))
config_references = [
expand_path(config_ref)
for config_ref in get_all_elems_from_json(config, 'config_path')
]
downloads |= {(url, dest)
for config in config_references
for url, dest in get_config_downloads(config)}
return downloads
def download_config(config_path):
src_file = src_dir / config_path
if not src_file.is_file():
src_file = test_src_dir / config_path
if not src_file.is_file():
raise RuntimeError('No config file {}'.format(config_path))
with src_file.open(encoding='utf8') as fin:
config: dict = json.load(fin)
# Download referenced config files
config_references = get_all_elems_from_json(parse_config(config), 'config_path')
for config_ref in config_references:
m_name = config_ref.split('/')[-2]
config_ref = '/'.join(config_ref.split('/')[-2:])
test_configs_path.joinpath(m_name).mkdir(exist_ok=True)
if not test_configs_path.joinpath(config_ref).exists():
download_config(config_ref)
# Update config for testing
config.setdefault('train', {}).setdefault('pytest_epochs', 1)
_override_with_test_values(config)
config_path = test_configs_path / config_path
config_path.parent.mkdir(exist_ok=True, parents=True)
with config_path.open("w", encoding='utf8') as fout:
json.dump(config, fout)
def download_config(config_path):
src_file = src_dir / config_path
if not src_file.is_file():
src_file = test_src_dir / config_path
if not src_file.is_file():
raise RuntimeError('No config file {}'.format(config_path))
with src_file.open(encoding='utf8') as fin:
config: dict = json.load(fin)
# Download referenced config files
config_references = get_all_elems_from_json(parse_config(config),
'config_path')
for config_ref in config_references:
m_name = config_ref.split('/')[-2]
config_ref = '/'.join(config_ref.split('/')[-2:])
test_configs_path.joinpath(m_name).mkdir(exist_ok=True)
if not test_configs_path.joinpath(config_ref).exists():
download_config(config_ref)
# Update config for testing
config.setdefault('train', {}).setdefault('pytest_epochs', 1)
config['train'].setdefault('pytest_max_batches', 2)
config['train'].setdefault('pytest_max_test_batches', 2)
_override_with_test_values(config)
config_path = test_configs_path / config_path
config_path.parent.mkdir(exist_ok=True, parents=True)
with config_path.open("w", encoding='utf8') as fout:
json.dump(config, fout)
def run_population(population, evolution, gpus):
"""
Change save and load paths for obtained population, save config.json with model config,
run population via current python executor (with which evolve.py already run)
and on given devices (-1 means CPU, other integeres - visible for evolve.py GPUs)
Args:
population: list of dictionaries - configs of current population
evolution: ParamsEvolution
gpus: list of given devices (list of integers)
Returns:
None
"""
population_size = len(population)
for k in range(population_size // len(gpus) + 1):
procs = []
for j in range(len(gpus)):
i = k * len(gpus) + j
if i < population_size:
save_path = expand_path(
evolution.get_value_from_config(parse_config(population[i]),
evolution.path_to_models_save_path))
save_path.mkdir(parents=True, exist_ok=True)
f_name = save_path / "config.json"
save_json(population[i], f_name)
with save_path.joinpath('out.txt').open('w', encoding='utf8') as outlog,\
save_path.joinpath('err.txt').open('w', encoding='utf8') as errlog:
env = dict(os.environ)
if len(gpus) > 1 or gpus[0] != -1:
env['CUDA_VISIBLE_DEVICES'] = str(gpus[j])
procs.append(Popen("{} -m deeppavlov train {}".format(sys.executable, str(f_name)),
shell=True, stdout=outlog, stderr=errlog, env=env))
for j, proc in enumerate(procs):
i = k * len(gpus) + j
log.info(f'Waiting on {i}th proc')
if proc.wait() != 0:
save_path = expand_path(
evolution.get_value_from_config(parse_config(population[i]),
evolution.path_to_models_save_path))
with save_path.joinpath('err.txt').open(encoding='utf8') as errlog:
log.warning(f'Population {i} returned an error code {proc.returncode} and an error log:\n' +
errlog.read())
return None
def from_params(params: Dict, mode: str = 'infer', serialized: Any = None, **kwargs) -> Component:
"""Builds and returns the Component from corresponding dictionary of parameters."""
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
component = _refs[config_params['ref']]
if serialized is not None:
component.deserialize(serialized)
return component
except KeyError:
e = ConfigError('Component with id "{id}" was referenced but not initialized'
.format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model
refs = _refs.copy()
_refs.clear()
config = parse_config(expand_path(config_params['config_path']))
model = build_model(config, serialized=serialized)
_refs.clear()
_refs.update(refs)
return model
cls_name = config_params.pop('class_name', None)
if not cls_name:
e = ConfigError('Component config has no `class_name` nor `ref` fields')
log.exception(e)
raise e
cls = get_model(cls_name)
# find the submodels params recursively
config_params = {k: _init_param(v, mode) for k, v in config_params.items()}
try:
spec = inspect.getfullargspec(cls)
if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None:
kwargs['mode'] = mode
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
if serialized is not None:
component.deserialize(serialized)
return component
def build_model(config: Union[str, Path, dict],
mode: str = 'infer',
load_trained: bool = False,
download: bool = False,
serialized: Optional[bytes] = None) -> Chainer:
"""Build and return the model described in corresponding configuration file."""
config = parse_config(config)
if serialized:
serialized: list = pickle.loads(serialized)
if download:
deep_download(config)
import_packages(config.get('metadata', {}).get('imports', []))
model_config = config['chainer']
model = Chainer(model_config['in'], model_config['out'],
model_config.get('in_y'))
for component_config in model_config['pipe']:
if load_trained and ('fit_on' in component_config
or 'in_y' in component_config):
try:
component_config['load_path'] = component_config['save_path']
except KeyError:
log.warning(
'No "save_path" parameter for the {} component, so "load_path" will not be renewed'
.format(
component_config.get(
'class_name',
component_config.get('ref', 'UNKNOWN'))))
if serialized and 'in' in component_config:
component_serialized = serialized.pop(0)
else:
component_serialized = None
component = from_params(component_config,
mode=mode,
serialized=component_serialized)
if 'id' in component_config:
model._components_dict[component_config['id']] = component
if 'in' in component_config:
c_in = component_config['in']
c_out = component_config['out']
in_y = component_config.get('in_y', None)
main = component_config.get('main', False)
model.append(component, c_in, c_out, in_y, main)
return model
def get_config_requirements(config: [str, Path, dict]):
config = parse_config(config)
requirements = set()
for req in config.get('metadata', {}).get('requirements', []):
requirements.add(req)
config_references = [expand_path(config_ref) for config_ref in get_all_elems_from_json(config, 'config_path')]
requirements |= {req for config in config_references for req in get_config_requirements(config)}
return requirements
def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component:
"""Builds and returns the Component from corresponding dictionary of parameters."""
# what is passed in json:
config_params = {k: _resolve(v) for k, v in params.items()}
# get component by reference (if any)
if 'ref' in config_params:
try:
return _refs[config_params['ref']]
except KeyError:
e = ConfigError(
'Component with id "{id}" was referenced but not initialized'.
format(id=config_params['ref']))
log.exception(e)
raise e
elif 'config_path' in config_params:
from deeppavlov.core.commands.infer import build_model
refs = _refs.copy()
_refs.clear()
config = parse_config(expand_path(config_params['config_path']))
model = build_model(config)
_refs.clear()
_refs.update(refs)
return model
cls_name = config_params.pop('class_name', None)
if not cls_name:
e = ConfigError(
'Component config has no `class_name` nor `ref` fields')
log.exception(e)
raise e
cls = get_model(cls_name)
# find the submodels params recursively
config_params = {k: _init_param(v, mode) for k, v in config_params.items()}
try:
spec = inspect.getfullargspec(cls)
if 'mode' in spec.args + spec.kwonlyargs or spec.varkw is not None:
kwargs['mode'] = mode
component = cls(**dict(config_params, **kwargs))
try:
_refs[config_params['id']] = component
except KeyError:
pass
except Exception:
log.exception("Exception in {}".format(cls))
raise
return component
def save(self, path, copy_model: bool = False):
BaseSearchEngine.save(self, path)
model_settings = self.model_settings.copy()
if copy_model:
model_settings = parse_config(model_settings)
self._copy_model_files(model_settings, path)
model_settings['metadata']['download'] = []
with open(os.path.join(path, 'model_settings.json'), 'w') as f:
json.dump(model_settings, f, indent=True)
Doc2Vec.save(self.doc2vec, os.path.join(path, 'doc2vec.model'))
def get_server_params(server_config_path, model_config):
server_config = read_json(server_config_path)
model_config = parse_config(model_config)
server_params = server_config['common_defaults']
if check_nested_dict_keys(model_config, ['metadata', 'labels', 'server_utils']):
model_tag = model_config['metadata']['labels']['server_utils']
if model_tag in server_config['model_defaults']:
model_defaults = server_config['model_defaults'][model_tag]
for param_name in model_defaults.keys():
if model_defaults[param_name]:
server_params[param_name] = model_defaults[param_name]
return server_params
def get_server_params(server_config_path, model_config):
server_config = read_json(server_config_path)
model_config = parse_config(model_config)
server_params = server_config['common_defaults']
if check_nested_dict_keys(model_config,
['metadata', 'labels', 'server_utils']):
model_tag = model_config['metadata']['labels']['server_utils']
if model_tag in server_config['model_defaults']:
model_defaults = server_config['model_defaults'][model_tag]
for param_name in model_defaults.keys():
if model_defaults[param_name]:
server_params[param_name] = model_defaults[param_name]
return server_params
def predict_with_model(config_path: [Path, str]) -> List[Optional[List[str]]]:
"""Returns predictions of morphotagging model given in config :config_path:.
Args:
config_path: a path to config
Returns:
a list of morphological analyses for each sentence. Each analysis is either a list of tags
or a list of full CONLL-U descriptions.
"""
config = parse_config(config_path)
reader_config = config['dataset_reader']
reader = get_model(reader_config['class_name'])()
data_path = expand_path(reader_config.get('data_path', ''))
read_params = {
k: v
for k, v in reader_config.items()
if k not in ['class_name', 'data_path']
}
data: Dict = reader.read(data_path, **read_params)
iterator_config = config['dataset_iterator']
iterator: MorphoTaggerDatasetIterator = from_params(iterator_config,
data=data)
model = build_model(config, load_trained=True)
answers = [None] * len(iterator.test)
batch_size = config['predict'].get("batch_size", -1)
for indexes, (x, _) in iterator.gen_batches(batch_size=batch_size,
data_type="test",
shuffle=False,
return_indexes=True):
y = model(x)
for i, elem in zip(indexes, y):
answers[i] = elem
outfile = config['predict'].get("outfile")
if outfile is not None:
outfile = Path(outfile)
if not outfile.exists():
outfile.parent.mkdir(parents=True, exist_ok=True)
with open(outfile, "w", encoding="utf8") as fout:
for elem in answers:
fout.write(elem + "\n")
return answers
def get_server_params(server_config_path, model_config):
server_config = read_json(server_config_path)
model_config = parse_config(model_config)
server_params = server_config['common_defaults']
if check_nested_dict_keys(model_config, ['metadata', 'labels', 'server_utils']):
model_tag = model_config['metadata']['labels']['server_utils']
if model_tag in server_config['model_defaults']:
model_defaults = server_config['model_defaults'][model_tag]
for param_name in model_defaults.keys():
if model_defaults[param_name]:
server_params[param_name] = model_defaults[param_name]
server_params['model_endpoint'] = server_params.get('model_endpoint', '/model')
server_params['model_args_names'] = server_params['model_args_names'] or model_config['chainer']['in']
return server_params
def install_from_config(config: [str, Path, dict]):
config = parse_config(config)
requirements_files = config.get('metadata', {}).get('requirements', [])
if not requirements_files:
log.warn('No requirements found in config')
return
requirements = []
for rf in requirements_files:
with expand_path(rf).open(encoding='utf8') as f:
for line in f:
line = re.sub(r'\s', '', line.strip())
if line and not line.startswith(
'#') and line not in requirements:
requirements.append(line)
for r in requirements:
install(r)
def build_model(config: Union[str, Path, dict], mode: str = 'infer',
load_trained: bool = False, download: bool = False,
serialized: Optional[bytes] = None) -> Chainer:
"""Build and return the model described in corresponding configuration file."""
config = parse_config(config)
if serialized:
serialized: list = pickle.loads(serialized)
if download:
deep_download(config)
import_packages(config.get('metadata', {}).get('imports', []))
model_config = config['chainer']
model = Chainer(model_config['in'], model_config['out'], model_config.get('in_y'))
for component_config in model_config['pipe']:
if load_trained and ('fit_on' in component_config or 'in_y' in component_config):
try:
component_config['load_path'] = component_config['save_path']
except KeyError:
log.warning('No "save_path" parameter for the {} component, so "load_path" will not be renewed'
.format(component_config.get('class_name', component_config.get('ref', 'UNKNOWN'))))
if serialized and 'in' in component_config:
component_serialized = serialized.pop(0)
else:
component_serialized = None
component = from_params(component_config, mode=mode, serialized=component_serialized)
if 'in' in component_config:
c_in = component_config['in']
c_out = component_config['out']
in_y = component_config.get('in_y', None)
main = component_config.get('main', False)
model.append(component, c_in, c_out, in_y, main)
return model
def predict_with_model(
config_path: [Path, str],
infile: Optional[Union[Path, str]] = None,
input_format: str = "ud",
batch_size: [int] = 16,
output_format: str = "basic") -> List[Optional[List[str]]]:
"""Returns predictions of morphotagging model given in config :config_path:.
Args:
config_path: a path to config
Returns:
a list of morphological analyses for each sentence. Each analysis is either a list of tags
or a list of full CONLL-U descriptions.
"""
config = parse_config(config_path)
if infile is None:
if sys.stdin.isatty():
raise RuntimeError(
'To process data from terminal please use interact mode')
infile = sys.stdin
else:
infile = expand_path(infile)
if input_format in ["ud", "conllu", "vertical"]:
from_words = (input_format == "vertical")
data: List[tuple] = read_infile(infile, from_words=from_words)
# keeping only sentences
data = [elem[0] for elem in data]
else:
if infile is not sys.stdin:
with open(infile, "r", encoding="utf8") as fin:
data = fin.readlines()
else:
data = sys.stdin.readlines()
model = build_model(config, load_trained=True)
model.pipe[-1][-1].set_format_mode(output_format)
answers = model.batched_call(data, batch_size=batch_size)
for elem in answers:
print(elem)
return answers
def get_config_downloads(config: Union[str, Path, dict]) -> Set[Tuple[str, Path]]:
config = parse_config(config)
downloads = set()
if 'metadata' in config and 'download' in config['metadata']:
for resource in config['metadata']['download']:
if isinstance(resource, str):
resource = {
'url': resource
}
url = resource['url']
dest = expand_path(resource.get('subdir', ''))
downloads.add((url, dest))
config_references = [expand_path(config_ref) for config_ref in get_all_elems_from_json(config, 'config_path')]
downloads |= {(url, dest) for config in config_references for url, dest in get_config_downloads(config)}
return downloads
def predict_with_model(config_path: [Path, str]) -> List[Optional[List[str]]]:
"""Returns predictions of morphotagging model given in config :config_path:.
Args:
config_path: a path to config
Returns:
a list of morphological analyses for each sentence. Each analysis is either a list of tags
or a list of full CONLL-U descriptions.
"""
config = parse_config(config_path)
reader_config = config['dataset_reader']
reader = get_model(reader_config['class_name'])()
data_path = expand_path(reader_config.get('data_path', ''))
read_params = {k: v for k, v in reader_config.items() if k not in ['class_name', 'data_path']}
data: Dict = reader.read(data_path, **read_params)
iterator_config = config['dataset_iterator']
iterator: MorphoTaggerDatasetIterator = from_params(iterator_config, data=data)
model = build_model(config, load_trained=True)
answers = [None] * len(iterator.test)
batch_size = config['predict'].get("batch_size", -1)
for indexes, (x, _) in iterator.gen_batches(
batch_size=batch_size, data_type="test", shuffle=False, return_indexes=True):
y = model(x)
for i, elem in zip(indexes, y):
answers[i] = elem
outfile = config['predict'].get("outfile")
if outfile is not None:
outfile = Path(outfile)
if not outfile.exists():
outfile.parent.mkdir(parents=True, exist_ok=True)
with open(outfile, "w", encoding="utf8") as fout:
for elem in answers:
fout.write(elem + "\n")
return answers
def upload(config_in_file):
config_in = parse_config(config_in_file)
config_in_file = find_config(config_in_file)
model_path = Path(
config_in['metadata']['variables']['MODEL_PATH']).expanduser()
model_name, class_name = config_in_file.stem, config_in_file.parent.name
tmp_dir = f'/tmp/{class_name}'
tmp_tar = f'/tmp/{class_name}/{model_name}.tar.gz'
shutil.rmtree(tmp_dir, ignore_errors=True)
os.mkdir(tmp_dir)
with tarfile.open(tmp_tar, "w:gz") as tar:
tar.add(model_path, arcname=model_name)
main(tmp_tar)
command = f'scp -r {tmp_dir} share.ipavlov.mipt.ru:/home/export/v1/'
donwload_url = f'http://files.deeppavlov.ai/v1/{class_name}/{model_name}.tar.gz'
print(command, donwload_url, sep='\n')
def download_config(conf_file):
src_file = src_dir / conf_file
if not src_file.is_file():
src_file = test_src_dir / conf_file
if not src_file.is_file():
raise RuntimeError('No config file {}'.format(conf_file))
with src_file.open(encoding='utf8') as fin:
config: dict = json.load(fin)
# Download referenced config files
config_references = get_all_elems_from_json(parse_config(config),
'config_path')
for config_ref in config_references:
m_name = config_ref.split('/')[-2]
config_ref = '/'.join(config_ref.split('/')[-2:])
test_configs_path.joinpath(m_name).mkdir(exist_ok=True)
if not test_configs_path.joinpath(config_ref).exists():
download_config(config_ref)
# Update config for testing
if config.get("train"):
config["train"]["epochs"] = 1
for pytest_key in [
k for k in config["train"] if k.startswith('pytest_')
]:
config["train"][pytest_key[len('pytest_'):]] = config["train"].pop(
pytest_key)
config_vars = config.setdefault('metadata', {}).setdefault('variables', {})
config_vars['ROOT_PATH'] = str(download_path)
config_vars['CONFIGS_PATH'] = str(test_configs_path)
conf_file = test_configs_path / conf_file
conf_file.parent.mkdir(exist_ok=True, parents=True)
with conf_file.open("w", encoding='utf8') as fout:
json.dump(config, fout)
def upload(config_in_file):
print(config_in_file)
config_in = parse_config(config_in_file)
config_in_file = find_config(config_in_file)
model_path = Path(
config_in['metadata']['variables']['MODEL_PATH']).expanduser()
models_path = Path(
config_in['metadata']['variables']['MODELS_PATH']).expanduser()
model_name, class_name = config_in_file.stem, config_in_file.parent.name
if str(model_name) not in str(model_path):
raise (f'{model_name} is not the path of the {model_path}')
arcname = str(model_path).split("models/")[1]
tar_path = models_path / model_name
tmp_folder = f'/tmp/'
tmp_tar = tmp_folder + f'{model_name}.tar.gz'
print("model_path", model_path)
print("class_name", class_name)
print("model_name", model_name)
print("Start tarring")
archive = tarfile.open(tmp_tar, "w|gz")
archive.add(model_path, arcname=arcname)
archive.close()
print("Stop tarring")
print("Calculating hash")
main(tmp_tar)
print("tmp_tar", tmp_tar)
command = f'scp -r {tmp_folder}{model_name}* share.ipavlov.mipt.ru:/home/export/v1/{class_name}'
donwload_url = f'http://files.deeppavlov.ai/v1/{class_name}/{model_name}.tar.gz'
print(command, donwload_url, sep='\n')
def calc_cv_score(config, data=None, n_folds=5, is_loo=False):
config = parse_config(config)
if data is None:
data = read_data_by_config(config)
config, dirs_for_saved_models = change_savepath_for_model(config)
cv_score = OrderedDict()
for data_i in generate_train_valid(data, n_folds=n_folds, is_loo=is_loo):
iterator = get_iterator_from_config(config, data_i)
create_dirs_to_save_models(dirs_for_saved_models)
score = train_evaluate_model_from_config(config, iterator=iterator)
delete_dir_for_saved_models(dirs_for_saved_models)
for key, value in score['valid'].items():
if key not in cv_score:
cv_score[key] = []
cv_score[key].append(value)
for key, value in cv_score.items():
cv_score[key] = np.mean(value)
log.info('Cross-Validation \"{}\" is: {}'.format(key, cv_score[key]))
return cv_score
def calc_cv_score(config, data=None, n_folds=5, is_loo=False):
config = parse_config(config)
if data is None:
data = read_data_by_config(config)
config, dirs_for_saved_models = change_savepath_for_model(config)
cv_score = OrderedDict()
for data_i in generate_train_valid(data, n_folds=n_folds, is_loo=is_loo):
iterator = get_iterator_from_config(config, data_i)
create_dirs_to_save_models(dirs_for_saved_models)
score = train_evaluate_model_from_config(config, iterator=iterator)
delete_dir_for_saved_models(dirs_for_saved_models)
for key, value in score['valid'].items():
if key not in cv_score:
cv_score[key] = []
cv_score[key].append(value)
for key, value in cv_score.items():
cv_score[key] = np.mean(value)
log.info('Cross-Validation \"{}\" is: {}'.format(key, cv_score[key]))
return cv_score
def load_elmo(elmo_output_names=("word_emb",)):
config = parse_config(getattr(configs.elmo_embedder, "elmo_ru-news"))
elmo_config = config["chainer"]["pipe"][-1]
elmo_config['elmo_output_names'] = elmo_output_names
embedder = from_params(elmo_config)
return embedder
def train_evaluate_model_from_config(config: [str, Path, dict], iterator=None, *,
to_train=True, to_validate=True, download=False,
start_epoch_num=0, recursive=False) -> Dict[str, Dict[str, float]]:
"""Make training and evaluation of the model described in corresponding configuration file."""
config = parse_config(config)
if download:
deep_download(config)
if to_train and recursive:
for subconfig in get_all_elems_from_json(config['chainer'], 'config_path'):
log.info(f'Training "{subconfig}"')
train_evaluate_model_from_config(subconfig, download=False, recursive=True)
import_packages(config.get('metadata', {}).get('imports', []))
if iterator is None:
try:
data = read_data_by_config(config)
except ConfigError as e:
to_train = False
log.warning(f'Skipping training. {e.message}')
else:
iterator = get_iterator_from_config(config, data)
train_config = {
'metrics': ['accuracy'],
'validate_best': to_validate,
'test_best': True,
'show_examples': False
}
try:
train_config.update(config['train'])
except KeyError:
log.warning('Train config is missing. Populating with default values')
in_y = config['chainer'].get('in_y', ['y'])
if isinstance(in_y, str):
in_y = [in_y]
if isinstance(config['chainer']['out'], str):
config['chainer']['out'] = [config['chainer']['out']]
metrics_functions = _parse_metrics(train_config['metrics'], in_y, config['chainer']['out'])
if to_train:
model = fit_chainer(config, iterator)
if callable(getattr(model, 'train_on_batch', None)):
_train_batches(model, iterator, train_config, metrics_functions, start_epoch_num=start_epoch_num)
model.destroy()
res = {}
if iterator is not None and (train_config['validate_best'] or train_config['test_best']):
model = build_model(config, load_trained=to_train)
log.info('Testing the best saved model')
if train_config['validate_best']:
report = {
'valid': _test_model(model, metrics_functions, iterator,
train_config.get('batch_size', -1), 'valid',
show_examples=train_config['show_examples'])
}
res['valid'] = report['valid']['metrics']
print(json.dumps(report, ensure_ascii=False))
if train_config['test_best']:
report = {
'test': _test_model(model, metrics_functions, iterator,
train_config.get('batch_size', -1), 'test',
show_examples=train_config['show_examples'])
}
res['test'] = report['test']['metrics']
print(json.dumps(report, ensure_ascii=False))
model.destroy()
return res
from deeppavlov import configs, build_model, train_model from deeppavlov.core.commands.utils import parse_config config_dict = parse_config(configs.ner.ner_ontonotes_bert_mult) reader = config_dict['dataset_reader'] print(config_dict['dataset_reader']['data_path']) ner_model = train_model(configs.ner.ner_ontonotes_bert_mult, download=False)
def create_settings(paraphrases: Optional[List[Tuple[Tuple[str, str], int]]],
name: str,
train_size=0.8,
fasttext_embed_path: str = None,
root_path: str = '~/.deeppavlov',
model_settings: Optional[Dict] = None,
max_sequence_length=30,
nn_class_name='mpm_nn',
hidden_dim=200,
aggregation_dim=200):
if model_settings is None:
model_settings = {
'max_sequence_length': max_sequence_length,
'class_name': 'mpm_nn',
'hidden_dim': 200,
'aggregation_dim': 200
}
if nn_class_name != 'mpm_nn':
warnings.warn(
'Parameter nn_class_name deprecated: in 0.6.12 will be removed in 1.0'
)
model_settings['class_name'] = nn_class_name
if hidden_dim != 200:
warnings.warn(
'Parameter hidden_dim deprecated: in 0.6.12 will be removed in 1.0'
)
model_settings['hidden_dim'] = hidden_dim
if aggregation_dim != 200:
warnings.warn(
'Parameter aggregation_dim deprecated: in 0.6.12 will be removed in 1.0'
)
model_settings['aggregation_dim'] = aggregation_dim
if max_sequence_length != 30:
warnings.warn(
'Parameter max_sequence_length deprecated: in 0.6.12 will be removed in 1.0'
)
model_settings['max_sequence_length'] = max_sequence_length
downloads = []
if fasttext_embed_path is None:
fasttext_embed_path = '{DOWNLOADS_PATH}/embeddings/lenta_lower_100.bin'
downloads.append({
'url': 'http://files.deeppavlov.ai/embeddings/lenta_lower_100.bin',
'subdir': '{DOWNLOADS_PATH}/embeddings'
})
preproc = {
'id': 'preproc',
'class_name': 'siamese_preprocessor',
'use_matrix': False,
'max_sequence_length': max_sequence_length,
'fit_on': ['x'],
'in': ['x'],
'out': ['x_proc'],
'sent_vocab': {
'id': 'siam_sent_vocab',
'class_name': 'simple_vocab',
'save_path': '{MODELS_PATH}/%s/sent.dict' % name,
'load_path': '{MODELS_PATH}/%s/sent.dict' % name
},
'tokenizer': {
'class_name': 'nltk_tokenizer'
},
'vocab': {
'id': 'siam_vocab',
'class_name': 'simple_vocab',
'save_path': '{MODELS_PATH}/%s/tok.dict' % name,
'load_path': '{MODELS_PATH}/%s/tok.dict' % name
}
}
embedding = {
'id': 'embeddings',
'class_name': 'emb_mat_assembler',
'embedder': '#siam_embedder',
'vocab': '#siam_vocab'
}
nn = {
'id': 'model',
'in': ['x_proc'],
'in_y': ['y'],
'out': ['y_predicted'],
'len_vocab': '#siam_vocab.len',
'use_matrix': False,
'attention': True,
'emb_matrix': '#embeddings.emb_mat',
'embedding_dim': '#siam_embedder.dim',
'max_sequence_length': '#preproc.max_sequence_length',
'seed': 243,
'learning_rate': 1e-3,
'triplet_loss': False,
'batch_size': 256,
'save_path': '{MODELS_PATH}/%s/model_weights.h5' % name,
'load_path': '{MODELS_PATH}/%s/model_weights.h5' % name,
'preprocess': '#preproc.__call__'
}
nn.update(model_settings)
preproc['embedder'] = {
'id': 'siam_embedder',
'class_name': 'fasttext',
'load_path': fasttext_embed_path
}
pipe = [preproc, embedding, nn]
res = {
'dataset_reader': {
'class_name': 'simple_reader',
'data_path': '{MODELS_PATH}/%s/dataset.json' % name
},
'dataset_iterator': {
'class_name': 'siamese_iterator',
'seed': 243
},
'chainer': {
'in': ['x'],
'in_y': ['y'],
'pipe': pipe,
'out': ['y_predicted']
},
'train': {
'epochs': 10,
'batch_size': 256,
'pytest_max_batches': 2,
'train_metrics': ['f1', 'acc', 'log_loss'],
'metrics': ['f1', 'acc', 'log_loss'],
'validation_patience': 10,
'val_every_n_epochs': 1,
'log_every_n_batches': 1,
'class_name': 'nn_trainer',
'evaluation_targets': ['test']
},
'metadata': {
'variables': {
'ROOT_PATH': root_path,
'DOWNLOADS_PATH': '{ROOT_PATH}/downloads',
'MODELS_PATH': '{ROOT_PATH}/models'
},
'requirements': [],
'download': downloads
}
}
if paraphrases:
SimpleDataReader().save(
paraphrases,
parse_config(res)['dataset_reader']['data_path'], train_size)
return res
def results_to_table(population, evolution, considered_metrics, result_file,
result_table_columns):
population_size = len(population)
train_config = evolution.basic_config.get('train', {})
if 'evaluation_targets' in train_config:
evaluation_targets = train_config['evaluation_targets']
else:
evaluation_targets = []
if train_config.get('validate_best', True):
evaluation_targets.append('valid')
elif train_config.get('test_best', True):
evaluation_targets.append('test')
if 'valid' in evaluation_targets:
target = 'valid'
elif 'test' in evaluation_targets:
target = 'test'
elif 'train' in evaluation_targets:
target = 'train'
else:
raise ConfigError('evaluation_targets are empty. Can not evolve')
if target != 'valid':
log.info(f"Tuning parameters on {target}")
population_metrics = {}
for m in considered_metrics:
population_metrics[m] = []
for i in range(population_size):
log_path = expand_path(
evolution.get_value_from_config(
parse_config(population[i]),
evolution.path_to_models_save_path)) / "out.txt"
report = {}
with log_path.open(encoding='utf8') as f:
for line in f:
try:
report.update(json.loads(line))
except:
pass
result_table_dict = defaultdict(list)
for m in considered_metrics:
for data_type in evaluation_targets:
result_table_dict[f'{m}_{data_type}'].append(
report[data_type]['metrics'][m])
if data_type == target:
population_metrics[m].append(
report[data_type]['metrics'][m])
result_table_dict[result_table_columns[-1]] = [
json.dumps(population[i])
]
result_table = pd.DataFrame(result_table_dict)
result_table.loc[:, result_table_columns].to_csv(result_file,
index=False,
sep='\t',
mode='a',
header=None)
return population_metrics
def results_to_table(population, evolution, considered_metrics, result_file,
result_table_columns):
population_size = len(population)
validate_best = evolution.get_value_from_config(
evolution.basic_config,
list(evolution.find_model_path(
evolution.basic_config, "validate_best"))[0] + ["validate_best"])
test_best = evolution.get_value_from_config(
evolution.basic_config,
list(evolution.find_model_path(evolution.basic_config, "test_best"))[0]
+ ["test_best"])
if (not validate_best) and test_best:
log.info("Validate_best is set to False. Tuning parameters on test")
elif (not validate_best) and (not test_best):
raise ConfigError(
"Validate_best and test_best are set to False. Can not evolve.")
population_metrics = {}
for m in considered_metrics:
population_metrics[m] = []
for i in range(population_size):
logpath = expand_path(
evolution.get_value_from_config(
parse_config(population[i]),
evolution.path_to_models_save_path)) / "out.txt"
reports_data = logpath.read_text(encoding='utf8').splitlines()[-2:]
reports = []
for j in range(2):
try:
reports.append(json.loads(reports_data[j]))
except:
pass
val_results = {}
test_results = {}
for m in considered_metrics:
val_results[m] = None
test_results[m] = None
if len(reports) == 2 and "valid" in reports[0].keys(
) and "test" in reports[1].keys():
val_results = reports[0]["valid"]["metrics"]
test_results = reports[1]["test"]["metrics"]
elif len(reports) == 2 and "valid" in reports[0].keys(
) and "valid" in reports[1].keys():
val_results = reports[1]["valid"]["metrics"]
elif len(reports) == 2 and "test" in reports[0].keys(
) and "test" in reports[1].keys():
val_results = reports[1]["test"]["metrics"]
elif len(reports) == 2 and "train" in reports[0].keys(
) and "valid" in reports[1].keys():
val_results = reports[1]["valid"]["metrics"]
elif len(reports) == 2 and "train" in reports[0].keys(
) and "test" in reports[1].keys():
val_results = reports[1]["test"]["metrics"]
elif len(reports) == 2 and "train" in reports[0].keys(
) and "train" in reports[1].keys():
val_results = reports[1]["train"]["metrics"]
elif len(reports) == 1 and "valid" in reports[0].keys():
val_results = reports[0]["valid"]["metrics"]
elif len(reports) == 1 and "test" in reports[0].keys():
test_results = reports[0]["test"]["metrics"]
else:
raise ConfigError(
"Can not proceed output files: didn't find valid and/or test results"
)
result_table_dict = {}
for el in result_table_columns:
result_table_dict[el] = []
for m in considered_metrics:
result_table_dict[m + "_valid"].append(val_results[m])
result_table_dict[m + "_test"].append(test_results[m])
if validate_best:
population_metrics[m].append(val_results[m])
elif test_best:
population_metrics[m].append(test_results[m])
result_table_dict[result_table_columns[-1]] = [
json.dumps(population[i])
]
result_table = pd.DataFrame(result_table_dict)
result_table.loc[:, result_table_columns].to_csv(result_file,
index=False,
sep='\t',
mode='a',
header=None)
return population_metrics
def results_to_table(population, evolution, considered_metrics, result_file, result_table_columns):
population_size = len(population)
validate_best = evolution.get_value_from_config(evolution.basic_config,
list(evolution.find_model_path(
evolution.basic_config, "validate_best"))[0]
+ ["validate_best"])
test_best = evolution.get_value_from_config(evolution.basic_config,
list(evolution.find_model_path(
evolution.basic_config, "test_best"))[0]
+ ["test_best"])
if (not validate_best) and test_best:
log.info("Validate_best is set to False. Tuning parameters on test")
elif (not validate_best) and (not test_best):
raise ConfigError("Validate_best and test_best are set to False. Can not evolve.")
population_metrics = {}
for m in considered_metrics:
population_metrics[m] = []
for i in range(population_size):
logpath = expand_path(evolution.get_value_from_config(parse_config(population[i]),
evolution.path_to_models_save_path)
) / "out.txt"
reports_data = logpath.read_text(encoding='utf8').splitlines()[-2:]
reports = []
for j in range(2):
try:
reports.append(json.loads(reports_data[j]))
except:
pass
val_results = {}
test_results = {}
for m in considered_metrics:
val_results[m] = None
test_results[m] = None
if len(reports) == 2 and "valid" in reports[0].keys() and "test" in reports[1].keys():
val_results = reports[0]["valid"]["metrics"]
test_results = reports[1]["test"]["metrics"]
elif len(reports) == 2 and "valid" in reports[0].keys() and "valid" in reports[1].keys():
val_results = reports[1]["valid"]["metrics"]
elif len(reports) == 2 and "test" in reports[0].keys() and "test" in reports[1].keys():
val_results = reports[1]["test"]["metrics"]
elif len(reports) == 2 and "train" in reports[0].keys() and "valid" in reports[1].keys():
val_results = reports[1]["valid"]["metrics"]
elif len(reports) == 2 and "train" in reports[0].keys() and "test" in reports[1].keys():
val_results = reports[1]["test"]["metrics"]
elif len(reports) == 2 and "train" in reports[0].keys() and "train" in reports[1].keys():
val_results = reports[1]["train"]["metrics"]
elif len(reports) == 1 and "valid" in reports[0].keys():
val_results = reports[0]["valid"]["metrics"]
elif len(reports) == 1 and "test" in reports[0].keys():
test_results = reports[0]["test"]["metrics"]
else:
raise ConfigError("Can not proceed output files: didn't find valid and/or test results")
result_table_dict = {}
for el in result_table_columns:
result_table_dict[el] = []
for m in considered_metrics:
result_table_dict[m + "_valid"].append(val_results[m])
result_table_dict[m + "_test"].append(test_results[m])
if validate_best:
population_metrics[m].append(val_results[m])
elif test_best:
population_metrics[m].append(test_results[m])
result_table_dict[result_table_columns[-1]] = [json.dumps(population[i])]
result_table = pd.DataFrame(result_table_dict)
result_table.loc[:, result_table_columns].to_csv(result_file, index=False, sep='\t', mode='a', header=None)
return population_metrics
def train_evaluate_model_from_config(
config: Union[str, Path, dict],
iterator: Union[DataLearningIterator, DataFittingIterator] = None,
*,
to_train: bool = True,
evaluation_targets: Optional[Iterable[str]] = None,
to_validate: Optional[bool] = None,
download: bool = False,
start_epoch_num: Optional[int] = None,
recursive: bool = False) -> Dict[str, Dict[str, float]]:
"""Make training and evaluation of the model described in corresponding configuration file."""
config = parse_config(config)
if download:
deep_download(config)
if to_train and recursive:
for subconfig in get_all_elems_from_json(config['chainer'],
'config_path'):
log.info(f'Training "{subconfig}"')
train_evaluate_model_from_config(subconfig,
download=False,
recursive=True)
import_packages(config.get('metadata', {}).get('imports', []))
if iterator is None:
try:
data = read_data_by_config(config)
except ConfigError as e:
to_train = False
log.warning(f'Skipping training. {e.message}')
else:
iterator = get_iterator_from_config(config, data)
if 'train' not in config:
log.warning('Train config is missing. Populating with default values')
train_config = config.get('train')
if start_epoch_num is not None:
train_config['start_epoch_num'] = start_epoch_num
if 'evaluation_targets' not in train_config and (
'validate_best' in train_config or 'test_best' in train_config):
log.warning(
'"validate_best" and "test_best" parameters are deprecated.'
' Please, use "evaluation_targets" list instead')
train_config['evaluation_targets'] = []
if train_config.pop('validate_best', True):
train_config['evaluation_targets'].append('valid')
if train_config.pop('test_best', True):
train_config['evaluation_targets'].append('test')
trainer_class = get_model(train_config.pop('class_name', 'nn_trainer'))
trainer = trainer_class(config['chainer'], **train_config)
if to_train:
trainer.train(iterator)
res = {}
if iterator is not None:
if to_validate is not None:
if evaluation_targets is None:
log.warning(
'"to_validate" parameter is deprecated and will be removed in future versions.'
' Please, use "evaluation_targets" list instead')
evaluation_targets = ['test']
if to_validate:
evaluation_targets.append('valid')
else:
log.warn(
'Both "evaluation_targets" and "to_validate" parameters are specified.'
' "to_validate" is deprecated and will be ignored')
res = trainer.evaluate(iterator,
evaluation_targets,
print_reports=True)
trainer.get_chainer().destroy()
res = {k: v['metrics'] for k, v in res.items()}
return res
INTENT_PHRASES_PATH = os.environ.get("INTENT_PHRASES_PATH", "intent_phrases.json")
CONFIG_NAME = os.environ.get("CONFIG_NAME", None)
if CONFIG_NAME is None:
raise NotImplementedError("No config file name is given.")
try:
intents_model = build_model(CONFIG_NAME, download=True)
logger.info("Model loaded")
regexp = get_regexp(INTENT_PHRASES_PATH)
logger.info("Regexp model loaded")
except Exception as e:
sentry_sdk.capture_exception(e)
logger.exception(e)
raise e
parsed = parse_config(CONFIG_NAME)
with open(expand_path(parsed["metadata"]["variables"]["MODEL_PATH"]).joinpath("classes.dict"), "r") as f:
intents = f.read().strip().splitlines()
intents = [el.strip().split("\t")[0] for el in intents]
logger.info(f"Considered intents: {intents}")
def get_classifier_predictions(batch_texts: List[List[str]], intents, intents_model, thresholds):
if thresholds is None:
# if we do not given thresholds, use 0.5 as default
thresholds = [0.5] * len(intents)
thresholds = np.array(thresholds)
# make a 1d-list of texts for classifier
sentences = list(chain.from_iterable(batch_texts))
sentences_text_ids = []
for text_id, text in enumerate(batch_texts):
def main():
params_helper = ParamsSearch()
args = parser.parse_args()
is_loo = False
n_folds = None
if args.folds == 'loo':
is_loo = True
elif args.folds is None:
n_folds = None
elif args.folds.isdigit():
n_folds = int(args.folds)
else:
raise NotImplementedError('Not implemented this type of CV')
# read config
pipeline_config_path = find_config(args.config_path)
config_init = read_json(pipeline_config_path)
config = parse_config(config_init)
data = read_data_by_config(config)
target_metric = parse_config(config_init)['train']['metrics'][0]
if isinstance(target_metric, dict):
target_metric = target_metric['name']
# get all params for search
param_paths = list(params_helper.find_model_path(config, 'search_choice'))
param_values = []
param_names = []
for path in param_paths:
value = params_helper.get_value_from_config(config, path)
param_name = path[-1]
param_value_search = value['search_choice']
param_names.append(param_name)
param_values.append(param_value_search)
# find optimal params
if args.search_type == 'grid':
# generate params combnations for grid search
combinations = list(product(*param_values))
# calculate cv scores
scores = []
for comb in combinations:
config = deepcopy(config_init)
for param_path, param_value in zip(param_paths, comb):
params_helper.insert_value_or_dict_into_config(
config, param_path, param_value)
config = parse_config(config)
if (n_folds is not None) | is_loo:
# CV for model evaluation
score_dict = calc_cv_score(config,
data=data,
n_folds=n_folds,
is_loo=is_loo)
score = score_dict[next(iter(score_dict))]
else:
# train/valid for model evaluation
data_to_evaluate = data.copy()
if len(data_to_evaluate['valid']) == 0:
data_to_evaluate['train'], data_to_evaluate[
'valid'] = train_test_split(data_to_evaluate['train'],
test_size=0.2)
iterator = get_iterator_from_config(config, data_to_evaluate)
score = train_evaluate_model_from_config(
config, iterator=iterator)['valid'][target_metric]
scores.append(score)
# get model with best score
best_params_dict = get_best_params(combinations, scores, param_names,
target_metric)
log.info('Best model params: {}'.format(best_params_dict))
else:
raise NotImplementedError('Not implemented this type of search')
# save config
best_config = config_init
for i, param_name in enumerate(best_params_dict.keys()):
if param_name != target_metric:
params_helper.insert_value_or_dict_into_config(
best_config, param_paths[i], best_params_dict[param_name])
best_model_filename = pipeline_config_path.with_suffix('.cvbest.json')
save_json(best_config, best_model_filename)
log.info('Best model saved in json-file: {}'.format(best_model_filename))