def _collate_best_model(meta, output_path, components):
bests = {}
for component in components:
bests[component] = _find_best(output_path, component)
best_dest = output_path / "model-best"
shutil.copytree(path2str(output_path / "model-final"), path2str(best_dest))
for component, best_component_src in bests.items():
shutil.rmtree(path2str(best_dest / component))
shutil.copytree(path2str(best_component_src / component),
path2str(best_dest / component))
accs = srsly.read_json(best_component_src / "accuracy.json")
for metric in _get_metrics(component):
meta["accuracy"][metric] = accs[metric]
srsly.write_json(best_dest / "meta.json", meta)
return best_dest
def create_dirs(package_path, force):
if package_path.exists():
if force:
shutil.rmtree(path2str(package_path))
else:
prints(package_path, Messages.M045, title=Messages.M044, exits=1)
Path.mkdir(package_path, parents=True)
def test_serialize_doc_roundtrip_disk_str_path(en_vocab):
doc = Doc(en_vocab, words=["hello", "world"])
with make_tempdir() as d:
file_path = d / "doc"
file_path = path2str(file_path)
doc.to_disk(file_path)
doc_d = Doc(en_vocab).from_disk(file_path)
assert doc.to_bytes() == doc_d.to_bytes()
def test_serialize_doc_roundtrip_disk_str_path(en_vocab):
doc = Doc(en_vocab, words=["hello", "world"])
with make_tempdir() as d:
file_path = d / "doc"
file_path = path2str(file_path)
doc.to_disk(file_path)
doc_d = Doc(en_vocab).from_disk(file_path)
assert doc.to_bytes() == doc_d.to_bytes()
def symlink_setup_target(request, symlink_target, symlink):
if not symlink_target.exists():
os.mkdir(path2str(symlink_target))
# yield -- need to cleanup even if assertion fails
# https://github.com/pytest-dev/pytest/issues/2508#issuecomment-309934240
def cleanup():
symlink_remove(symlink)
os.rmdir(path2str(symlink_target))
request.addfinalizer(cleanup)
def symlink_setup_target(request, symlink_target, symlink):
if not symlink_target.exists():
os.mkdir(path2str(symlink_target))
# yield -- need to cleanup even if assertion fails
# https://github.com/pytest-dev/pytest/issues/2508#issuecomment-309934240
def cleanup():
# Remove symlink only if it was created
if symlink.exists():
symlink_remove(symlink)
os.rmdir(path2str(symlink_target))
request.addfinalizer(cleanup)
def cleanup():
# Remove symlink only if it was created
if symlink.exists():
symlink_remove(symlink)
os.rmdir(path2str(symlink_target))
def make_tempdir():
d = Path(tempfile.mkdtemp())
yield d
shutil.rmtree(path2str(d))
def cleanup():
symlink_remove(symlink)
os.rmdir(path2str(symlink_target))
def package(input_dir,
output_dir,
meta_path=None,
create_meta=False,
force=False):
"""
Generate Python package for model data, including meta and required
installation files. A new directory will be created in the specified
output directory, and model data will be copied over.
"""
input_path = util.ensure_path(input_dir)
output_path = util.ensure_path(output_dir)
meta_path = util.ensure_path(meta_path)
if not input_path or not input_path.exists():
prints(input_path, title=Messages.M008, exits=1)
if not output_path or not output_path.exists():
prints(output_path, title=Messages.M040, exits=1)
if meta_path and not meta_path.exists():
prints(meta_path, title=Messages.M020, exits=1)
meta_path = meta_path or input_path / 'meta.json'
if meta_path.is_file():
meta = util.read_json(meta_path)
if not create_meta: # only print this if user doesn't want to overwrite
prints(meta_path, title=Messages.M041)
else:
meta = generate_meta(input_dir, meta)
meta = validate_meta(meta, ['lang', 'name', 'version'])
model_name = meta['lang'] + '_' + meta['name']
model_name_v = model_name + '-' + meta['version']
main_path = output_path / model_name_v
package_path = main_path / model_name
bin_path = main_path / 'bin'
include_path = main_path / 'include'
orig_nc_path = Path(__file__).parent.parent
nc_path = package_path / 'neuralcoref'
create_dirs(package_path, force)
create_dirs(bin_path, force)
create_dirs(nc_path, force)
shutil.copytree(path2str(input_path),
path2str(package_path / model_name_v))
orig_include_path = path2str(Path(__file__).parent / 'include')
shutil.copytree(path2str(orig_include_path), path2str(include_path))
nc1_path = path2str(orig_nc_path / 'neuralcoref.pyx')
nc2_path = path2str(orig_nc_path / 'neuralcoref.pxd')
shutil.copyfile(path2str(nc1_path), path2str(nc_path / 'neuralcoref.pyx'))
shutil.copyfile(path2str(nc2_path), path2str(nc_path / 'neuralcoref.pxd'))
create_file(nc_path / '__init__.py', TEMPLATE_INIT_NC)
create_file(nc_path / '__init__.pxd', TEMPLATE_INIT_PXD)
orig_bin_path = path2str(
Path(__file__).parent.parent.parent / 'bin' / 'cythonize.py')
shutil.copyfile(path2str(orig_bin_path),
path2str(bin_path / 'cythonize.py'))
create_file(main_path / 'meta.json', json_dumps(meta))
create_file(main_path / 'setup.py', TEMPLATE_SETUP)
create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST)
create_file(package_path / '__init__.py', TEMPLATE_INIT.format(model_name))
create_file(package_path / '__init__.pxd', TEMPLATE_INIT_PXD)
prints(main_path,
Messages.M043,
title=Messages.M042.format(name=model_name_v))
def cleanup():
# Remove symlink only if it was created
if symlink.exists():
symlink_remove(symlink)
os.rmdir(path2str(symlink_target))
def make_tempdir():
d = Path(tempfile.mkdtemp())
yield d
shutil.rmtree(path2str(d))
