def load_from_meta_file(save_dir, meta_filename='meta.json', **kwargs) -> Component: load_path = save_dir save_dir = get_resource(save_dir) metapath = os.path.join(save_dir, meta_filename) if not os.path.isfile(metapath): tips = '' if save_dir.isupper(): from difflib import SequenceMatcher similar_keys = sorted(pretrained.ALL.keys(), key=lambda k: SequenceMatcher( None, save_dir, metapath).ratio(), reverse=True)[:5] tips = f'Check its spelling based on the available keys:\n' + \ f'{sorted(pretrained.ALL.keys())}\n' + \ f'Tips: it might be one of {similar_keys}' raise FileNotFoundError( f'The identifier {save_dir} resolves to a non-exist meta file {metapath}. {tips}' ) meta: dict = load_json(metapath) cls = meta.get('class_path', None) assert cls, f'{meta_filename} doesn\'t contain class_path field' obj: Component = object_from_class_path(cls, **kwargs) if hasattr(obj, 'load') and os.path.isfile( os.path.join(save_dir, 'config.json')): obj.load(save_dir) obj.meta['load_path'] = load_path return obj
def load_from_meta_file(save_dir, meta_filename='meta.json', transform_only=False, load_kwargs=None, **kwargs) -> Component: identifier = save_dir load_path = save_dir save_dir = get_resource(save_dir) metapath = os.path.join(save_dir, meta_filename) if not os.path.isfile(metapath): tips = '' if save_dir.isupper(): from difflib import SequenceMatcher similar_keys = sorted(pretrained.ALL.keys(), key=lambda k: SequenceMatcher( None, save_dir, metapath).ratio(), reverse=True)[:5] tips = f'Check its spelling based on the available keys:\n' + \ f'{sorted(pretrained.ALL.keys())}\n' + \ f'Tips: it might be one of {similar_keys}' raise FileNotFoundError( f'The identifier {save_dir} resolves to a non-exist meta file {metapath}. {tips}' ) meta: dict = load_json(metapath) cls = meta.get('class_path', None) assert cls, f'{meta_filename} doesn\'t contain class_path field' try: obj: Component = object_from_class_path(cls, **kwargs) if hasattr(obj, 'load') and os.path.isfile( os.path.join(save_dir, 'config.json')): if transform_only: # noinspection PyUnresolvedReferences obj.load_transform(save_dir) else: if load_kwargs is None: load_kwargs = {} obj.load(save_dir, **load_kwargs) obj.meta['load_path'] = load_path return obj except Exception as e: eprint(f'Failed to load {identifier}. See stack trace below') traceback.print_exc() model_version = meta.get("hanlp_version", "unknown") cur_version = version.__version__ if model_version != cur_version: eprint( f'{identifier} was created with hanlp-{model_version}, while you are running {cur_version}. ' f'Try to upgrade hanlp with\n' f'pip install --upgrade hanlp') exit(1)
def load_from_meta_file(save_dir: str, meta_filename='meta.json', transform_only=False, load_kwargs=None, **kwargs) -> Component: identifier = save_dir load_path = save_dir save_dir = get_resource(save_dir) if save_dir.endswith('.json'): meta_filename = os.path.basename(save_dir) save_dir = os.path.dirname(save_dir) metapath = os.path.join(save_dir, meta_filename) if not os.path.isfile(metapath): tips = '' if save_dir.isupper(): from difflib import SequenceMatcher similar_keys = sorted(pretrained.ALL.keys(), key=lambda k: SequenceMatcher( None, save_dir, metapath).ratio(), reverse=True)[:5] tips = f'Check its spelling based on the available keys:\n' + \ f'{sorted(pretrained.ALL.keys())}\n' + \ f'Tips: it might be one of {similar_keys}' raise FileNotFoundError( f'The identifier {save_dir} resolves to a non-exist meta file {metapath}. {tips}' ) meta: dict = load_json(metapath) cls = meta.get('class_path', None) assert cls, f'{meta_filename} doesn\'t contain class_path field' try: obj: Component = object_from_class_path(cls, **kwargs) if hasattr(obj, 'load'): if transform_only: # noinspection PyUnresolvedReferences obj.load_transform(save_dir) else: if load_kwargs is None: load_kwargs = {} if os.path.isfile(os.path.join(save_dir, 'config.json')): obj.load(save_dir, **load_kwargs) else: obj.load(metapath, **load_kwargs) obj.meta['load_path'] = load_path return obj except Exception as e: eprint(f'Failed to load {identifier}. See traceback below:') eprint(f'{"ERROR LOG BEGINS":=^80}') traceback.print_exc() eprint(f'{"ERROR LOG ENDS":=^80}') from pkg_resources import parse_version model_version = meta.get("hanlp_version", "unknown") installed_version = version.__version__ try: latest_version = get_latest_info_from_pypi() except: latest_version = None if parse_version(model_version) > parse_version(installed_version): eprint( f'{identifier} was created with hanlp-{model_version}, ' f'while you are running a lower version: {installed_version}. ' ) if installed_version != latest_version: eprint( f'Please upgrade hanlp with:\n' f'pip install --upgrade hanlp\n' f'If the problem still persists, please submit an issue to https://github.com/hankcs/HanLP/issues .' ) eprint( 'When reporting an issue, make sure to paste the FULL ERROR LOG above.' ) exit(1)
def load_meta(self, save_dir, filename='meta.json'): save_dir = get_resource(save_dir) metapath = os.path.join(save_dir, filename) if os.path.isfile(metapath): self.meta.update(load_json(metapath))
def from_meta(meta: Union[dict, str], **kwargs): if isinstance(meta, str): meta = load_json(meta) return Pipeline(*[load_from_meta(pipe) for pipe in meta['pipes']])
def load(self, filepath): meta = load_json(filepath) self.clear() self.extend(Pipeline.from_meta(meta))
def load_json(self, path): item = load_json(path) return self.copy_from(item)
def load_meta(self, save_dir, filename='meta.json'): save_dir = get_resource(save_dir) self.meta.update(load_json(os.path.join(save_dir, filename)))