def load_from_meta_file(save_dir,
meta_filename='meta.json',
**kwargs) -> Component:
load_path = save_dir
save_dir = get_resource(save_dir)
metapath = os.path.join(save_dir, meta_filename)
if not os.path.isfile(metapath):
tips = ''
if save_dir.isupper():
from difflib import SequenceMatcher
similar_keys = sorted(pretrained.ALL.keys(),
key=lambda k: SequenceMatcher(
None, save_dir, metapath).ratio(),
reverse=True)[:5]
tips = f'Check its spelling based on the available keys:\n' + \
f'{sorted(pretrained.ALL.keys())}\n' + \
f'Tips: it might be one of {similar_keys}'
raise FileNotFoundError(
f'The identifier {save_dir} resolves to a non-exist meta file {metapath}. {tips}'
)
meta: dict = load_json(metapath)
cls = meta.get('class_path', None)
assert cls, f'{meta_filename} doesn\'t contain class_path field'
obj: Component = object_from_class_path(cls, **kwargs)
if hasattr(obj, 'load') and os.path.isfile(
os.path.join(save_dir, 'config.json')):
obj.load(save_dir)
obj.meta['load_path'] = load_path
return obj
def load_from_meta_file(save_dir,
meta_filename='meta.json',
transform_only=False,
load_kwargs=None,
**kwargs) -> Component:
identifier = save_dir
load_path = save_dir
save_dir = get_resource(save_dir)
metapath = os.path.join(save_dir, meta_filename)
if not os.path.isfile(metapath):
tips = ''
if save_dir.isupper():
from difflib import SequenceMatcher
similar_keys = sorted(pretrained.ALL.keys(),
key=lambda k: SequenceMatcher(
None, save_dir, metapath).ratio(),
reverse=True)[:5]
tips = f'Check its spelling based on the available keys:\n' + \
f'{sorted(pretrained.ALL.keys())}\n' + \
f'Tips: it might be one of {similar_keys}'
raise FileNotFoundError(
f'The identifier {save_dir} resolves to a non-exist meta file {metapath}. {tips}'
)
meta: dict = load_json(metapath)
cls = meta.get('class_path', None)
assert cls, f'{meta_filename} doesn\'t contain class_path field'
try:
obj: Component = object_from_class_path(cls, **kwargs)
if hasattr(obj, 'load') and os.path.isfile(
os.path.join(save_dir, 'config.json')):
if transform_only:
# noinspection PyUnresolvedReferences
obj.load_transform(save_dir)
else:
if load_kwargs is None:
load_kwargs = {}
obj.load(save_dir, **load_kwargs)
obj.meta['load_path'] = load_path
return obj
except Exception as e:
eprint(f'Failed to load {identifier}. See stack trace below')
traceback.print_exc()
model_version = meta.get("hanlp_version", "unknown")
cur_version = version.__version__
if model_version != cur_version:
eprint(
f'{identifier} was created with hanlp-{model_version}, while you are running {cur_version}. '
f'Try to upgrade hanlp with\n'
f'pip install --upgrade hanlp')
exit(1)
def load_from_meta_file(save_dir: str,
meta_filename='meta.json',
transform_only=False,
load_kwargs=None,
**kwargs) -> Component:
identifier = save_dir
load_path = save_dir
save_dir = get_resource(save_dir)
if save_dir.endswith('.json'):
meta_filename = os.path.basename(save_dir)
save_dir = os.path.dirname(save_dir)
metapath = os.path.join(save_dir, meta_filename)
if not os.path.isfile(metapath):
tips = ''
if save_dir.isupper():
from difflib import SequenceMatcher
similar_keys = sorted(pretrained.ALL.keys(),
key=lambda k: SequenceMatcher(
None, save_dir, metapath).ratio(),
reverse=True)[:5]
tips = f'Check its spelling based on the available keys:\n' + \
f'{sorted(pretrained.ALL.keys())}\n' + \
f'Tips: it might be one of {similar_keys}'
raise FileNotFoundError(
f'The identifier {save_dir} resolves to a non-exist meta file {metapath}. {tips}'
)
meta: dict = load_json(metapath)
cls = meta.get('class_path', None)
assert cls, f'{meta_filename} doesn\'t contain class_path field'
try:
obj: Component = object_from_class_path(cls, **kwargs)
if hasattr(obj, 'load'):
if transform_only:
# noinspection PyUnresolvedReferences
obj.load_transform(save_dir)
else:
if load_kwargs is None:
load_kwargs = {}
if os.path.isfile(os.path.join(save_dir, 'config.json')):
obj.load(save_dir, **load_kwargs)
else:
obj.load(metapath, **load_kwargs)
obj.meta['load_path'] = load_path
return obj
except Exception as e:
eprint(f'Failed to load {identifier}. See traceback below:')
eprint(f'{"ERROR LOG BEGINS":=^80}')
traceback.print_exc()
eprint(f'{"ERROR LOG ENDS":=^80}')
from pkg_resources import parse_version
model_version = meta.get("hanlp_version", "unknown")
installed_version = version.__version__
try:
latest_version = get_latest_info_from_pypi()
except:
latest_version = None
if parse_version(model_version) > parse_version(installed_version):
eprint(
f'{identifier} was created with hanlp-{model_version}, '
f'while you are running a lower version: {installed_version}. '
)
if installed_version != latest_version:
eprint(
f'Please upgrade hanlp with:\n'
f'pip install --upgrade hanlp\n'
f'If the problem still persists, please submit an issue to https://github.com/hankcs/HanLP/issues .'
)
eprint(
'When reporting an issue, make sure to paste the FULL ERROR LOG above.'
)
exit(1)
def load_meta(self, save_dir, filename='meta.json'):
save_dir = get_resource(save_dir)
metapath = os.path.join(save_dir, filename)
if os.path.isfile(metapath):
self.meta.update(load_json(metapath))
def from_meta(meta: Union[dict, str], **kwargs):
if isinstance(meta, str):
meta = load_json(meta)
return Pipeline(*[load_from_meta(pipe) for pipe in meta['pipes']])
def load(self, filepath):
meta = load_json(filepath)
self.clear()
self.extend(Pipeline.from_meta(meta))
def load_json(self, path):
item = load_json(path)
return self.copy_from(item)
def load_meta(self, save_dir, filename='meta.json'):
save_dir = get_resource(save_dir)
self.meta.update(load_json(os.path.join(save_dir, filename)))
