def _collate_best_model(meta, output_path, components): bests = {} for component in components: bests[component] = _find_best(output_path, component) best_dest = output_path / "model-best" shutil.copytree(path2str(output_path / "model-final"), path2str(best_dest)) for component, best_component_src in bests.items(): shutil.rmtree(path2str(best_dest / component)) shutil.copytree(path2str(best_component_src / component), path2str(best_dest / component)) accs = srsly.read_json(best_component_src / "accuracy.json") for metric in _get_metrics(component): meta["accuracy"][metric] = accs[metric] srsly.write_json(best_dest / "meta.json", meta) return best_dest
def create_dirs(package_path, force): if package_path.exists(): if force: shutil.rmtree(path2str(package_path)) else: prints(package_path, Messages.M045, title=Messages.M044, exits=1) Path.mkdir(package_path, parents=True)
def test_serialize_doc_roundtrip_disk_str_path(en_vocab): doc = Doc(en_vocab, words=["hello", "world"]) with make_tempdir() as d: file_path = d / "doc" file_path = path2str(file_path) doc.to_disk(file_path) doc_d = Doc(en_vocab).from_disk(file_path) assert doc.to_bytes() == doc_d.to_bytes()
def test_serialize_doc_roundtrip_disk_str_path(en_vocab): doc = Doc(en_vocab, words=["hello", "world"]) with make_tempdir() as d: file_path = d / "doc" file_path = path2str(file_path) doc.to_disk(file_path) doc_d = Doc(en_vocab).from_disk(file_path) assert doc.to_bytes() == doc_d.to_bytes()
def symlink_setup_target(request, symlink_target, symlink): if not symlink_target.exists(): os.mkdir(path2str(symlink_target)) # yield -- need to cleanup even if assertion fails # https://github.com/pytest-dev/pytest/issues/2508#issuecomment-309934240 def cleanup(): symlink_remove(symlink) os.rmdir(path2str(symlink_target)) request.addfinalizer(cleanup)
def symlink_setup_target(request, symlink_target, symlink): if not symlink_target.exists(): os.mkdir(path2str(symlink_target)) # yield -- need to cleanup even if assertion fails # https://github.com/pytest-dev/pytest/issues/2508#issuecomment-309934240 def cleanup(): # Remove symlink only if it was created if symlink.exists(): symlink_remove(symlink) os.rmdir(path2str(symlink_target)) request.addfinalizer(cleanup)
def cleanup(): # Remove symlink only if it was created if symlink.exists(): symlink_remove(symlink) os.rmdir(path2str(symlink_target))
def make_tempdir(): d = Path(tempfile.mkdtemp()) yield d shutil.rmtree(path2str(d))
def cleanup(): symlink_remove(symlink) os.rmdir(path2str(symlink_target))
def package(input_dir, output_dir, meta_path=None, create_meta=False, force=False): """ Generate Python package for model data, including meta and required installation files. A new directory will be created in the specified output directory, and model data will be copied over. """ input_path = util.ensure_path(input_dir) output_path = util.ensure_path(output_dir) meta_path = util.ensure_path(meta_path) if not input_path or not input_path.exists(): prints(input_path, title=Messages.M008, exits=1) if not output_path or not output_path.exists(): prints(output_path, title=Messages.M040, exits=1) if meta_path and not meta_path.exists(): prints(meta_path, title=Messages.M020, exits=1) meta_path = meta_path or input_path / 'meta.json' if meta_path.is_file(): meta = util.read_json(meta_path) if not create_meta: # only print this if user doesn't want to overwrite prints(meta_path, title=Messages.M041) else: meta = generate_meta(input_dir, meta) meta = validate_meta(meta, ['lang', 'name', 'version']) model_name = meta['lang'] + '_' + meta['name'] model_name_v = model_name + '-' + meta['version'] main_path = output_path / model_name_v package_path = main_path / model_name bin_path = main_path / 'bin' include_path = main_path / 'include' orig_nc_path = Path(__file__).parent.parent nc_path = package_path / 'neuralcoref' create_dirs(package_path, force) create_dirs(bin_path, force) create_dirs(nc_path, force) shutil.copytree(path2str(input_path), path2str(package_path / model_name_v)) orig_include_path = path2str(Path(__file__).parent / 'include') shutil.copytree(path2str(orig_include_path), path2str(include_path)) nc1_path = path2str(orig_nc_path / 'neuralcoref.pyx') nc2_path = path2str(orig_nc_path / 'neuralcoref.pxd') shutil.copyfile(path2str(nc1_path), path2str(nc_path / 'neuralcoref.pyx')) shutil.copyfile(path2str(nc2_path), path2str(nc_path / 'neuralcoref.pxd')) create_file(nc_path / '__init__.py', TEMPLATE_INIT_NC) create_file(nc_path / '__init__.pxd', TEMPLATE_INIT_PXD) orig_bin_path = path2str( Path(__file__).parent.parent.parent / 'bin' / 'cythonize.py') shutil.copyfile(path2str(orig_bin_path), path2str(bin_path / 'cythonize.py')) create_file(main_path / 'meta.json', json_dumps(meta)) create_file(main_path / 'setup.py', TEMPLATE_SETUP) create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST) create_file(package_path / '__init__.py', TEMPLATE_INIT.format(model_name)) create_file(package_path / '__init__.pxd', TEMPLATE_INIT_PXD) prints(main_path, Messages.M043, title=Messages.M042.format(name=model_name_v))
def cleanup(): # Remove symlink only if it was created if symlink.exists(): symlink_remove(symlink) os.rmdir(path2str(symlink_target))
def make_tempdir(): d = Path(tempfile.mkdtemp()) yield d shutil.rmtree(path2str(d))