def to(self,
devices: Union[int, float, List[int],
Dict[str, Union[int, torch.device]]] = None,
logger: logging.Logger = None,
verbose=HANLP_VERBOSE):
"""Move this component to devices.
Args:
devices: Target devices.
logger: Logger for printing progress report, as copying a model from CPU to GPU can takes several seconds.
verbose: ``True`` to print progress when logger is None.
"""
if devices is None:
# if getattr(torch, 'has_mps', None): # mac M1 chips
# devices = torch.device('mps:0')
# else:
devices = cuda_devices(devices)
elif devices == -1 or devices == [-1]:
devices = []
elif isinstance(devices, (int, float)):
devices = cuda_devices(devices)
if devices:
if logger:
logger.info(
f'Using GPUs: [on_blue][cyan][bold]{devices}[/bold][/cyan][/on_blue]'
)
if isinstance(devices, list):
if verbose:
flash(
f'Moving model to GPUs {devices} [blink][yellow]...[/yellow][/blink]'
)
self.model = self.model.to(devices[0])
if len(devices) > 1 and not isdebugging() and not isinstance(
self.model, nn.DataParallel):
self.model = self.parallelize(devices)
elif isinstance(devices, dict):
for name, module in self.model.named_modules():
for regex, device in devices.items():
try:
on_device: torch.device = next(
module.parameters()).device
except StopIteration:
continue
if on_device == device:
continue
if isinstance(device, int):
if on_device.index == device:
continue
if re.match(regex, name):
if not name:
name = '*'
flash(
f'Moving module [yellow]{name}[/yellow] to [on_yellow][magenta][bold]{device}'
f'[/bold][/magenta][/on_yellow]: [red]{regex}[/red]\n'
)
module.to(device)
elif isinstance(devices, torch.device):
if verbose:
flash(
f'Moving model to {devices} [blink][yellow]...[/yellow][/blink]'
)
self.model = self.model.to(devices)
else:
raise ValueError(f'Unrecognized devices {devices}')
if verbose:
flash('')
else:
if logger:
logger.info('Using [red]CPU[/red]')
def __init__(self,
dataset,
batch_size=32,
shuffle=False,
sampler=None,
batch_sampler=None,
num_workers=0,
collate_fn=None,
pin_memory=False,
drop_last=False,
timeout=0,
worker_init_fn=None,
multiprocessing_context=None,
pad: dict = None,
vocabs: VocabDict = None,
device=None,
**kwargs):
""" A dataloader commonly used for NLP tasks. It offers the following convenience.
- Bachify each field of samples into a :class:`~torch.Tensor` if the field name satisfies the following criterion.
- Name ends with _id, _ids, _count, _offset, _span, mask
- Name is in `pad` dict.
- Pad each field according to field name, the vocabs and pad dict.
- Move :class:`~torch.Tensor` onto device.
Args:
dataset: A :class:`~torch.utils.data.Dataset` to be bachified.
batch_size: Max size of each batch.
shuffle: ``True`` to shuffle batches.
sampler: A :class:`~torch.utils.data.Sampler` to sample samples from data.
batch_sampler: A :class:`~torch.utils.data.Sampler` to sample batches form all batches.
num_workers: Number of workers for multi-thread loading. Note that multi-thread loading aren't always
faster.
collate_fn: A function to perform batchifying. It must be set to ``None`` in order to make use of the
features this class offers.
pin_memory: If samples are loaded in the Dataset on CPU and would like to be pushed to
the GPU, enabling pin_memory can speed up the transfer. It's not useful since most data field are
not in Tensor type.
drop_last: Drop the last batch since it could be half-empty.
timeout: For multi-worker loading, set a timeout to wait for a worker.
worker_init_fn: Init function for multi-worker.
multiprocessing_context: Context for multiprocessing.
pad: A dict holding field names and their padding values.
vocabs: A dict of vocabs so padding value can be fetched from it.
device: The device tensors will be moved onto.
**kwargs: Other arguments will be passed to :meth:`torch.utils.data.Dataset.__init__`
"""
if device == -1:
device = None
if collate_fn is None:
collate_fn = self.collate_fn
if num_workers is None:
if isdebugging():
num_workers = 0
else:
num_workers = 2
if batch_sampler is None:
assert batch_size, 'batch_size has to be specified when batch_sampler is None'
else:
batch_size = 1
shuffle = None
drop_last = None
# noinspection PyArgumentList
super(PadSequenceDataLoader,
self).__init__(dataset=dataset,
batch_size=batch_size,
shuffle=shuffle,
sampler=sampler,
batch_sampler=batch_sampler,
num_workers=num_workers,
collate_fn=collate_fn,
pin_memory=pin_memory,
drop_last=drop_last,
timeout=timeout,
worker_init_fn=worker_init_fn,
multiprocessing_context=multiprocessing_context,
**kwargs)
self.vocabs = vocabs
if isinstance(dataset, TransformableDataset) and dataset.transform:
transform = dataset.transform
if not isinstance(transform, TransformList):
transform = []
for each in transform:
if isinstance(each, EmbeddingNamedTransform):
if pad is None:
pad = {}
if each.dst not in pad:
pad[each.dst] = 0
self.pad = pad
self.device = device
def fit(self,
trn_data,
dev_data,
save_dir,
batch_size,
epochs,
devices=None,
logger=None,
seed=None,
finetune: Union[bool, str] = False,
eval_trn=True,
_device_placeholder=False,
**kwargs):
"""Fit to data, triggers the training procedure. For training set and dev set, they shall be local or remote
files.
Args:
trn_data: Training set.
dev_data: Development set.
save_dir: The directory to save trained component.
batch_size: The number of samples in a batch.
epochs: Number of epochs.
devices: Devices this component will live on.
logger: Any :class:`logging.Logger` instance.
seed: Random seed to reproduce this training.
finetune: ``True`` to load from ``save_dir`` instead of creating a randomly initialized component. ``str``
to specify a different ``save_dir`` to load from.
eval_trn: Evaluate training set after each update. This can slow down the training but provides a quick
diagnostic for debugging.
_device_placeholder: ``True`` to create a placeholder tensor which triggers PyTorch to occupy devices so
other components won't take these devices as first choices.
**kwargs: Hyperparameters used by sub-classes.
Returns:
Any results sub-classes would like to return. Usually the best metrics on training set.
"""
# Common initialization steps
config = self._capture_config(locals())
if not logger:
logger = self.build_logger('train', save_dir)
if seed is None:
self.config.seed = 233 if isdebugging() else int(time.time())
set_seed(self.config.seed)
logger.info(self._savable_config.to_json(sort=True))
if isinstance(devices, list) or devices is None or isinstance(
devices, float):
flash('[yellow]Querying CUDA devices [blink]...[/blink][/yellow]')
devices = -1 if isdebugging() else cuda_devices(devices)
flash('')
# flash(f'Available GPUs: {devices}')
if isinstance(devices, list):
first_device = (devices[0] if devices else -1)
elif isinstance(devices, dict):
first_device = next(iter(devices.values()))
elif isinstance(devices, int):
first_device = devices
else:
first_device = -1
if _device_placeholder and first_device >= 0:
_dummy_placeholder = self._create_dummy_placeholder_on(
first_device)
if finetune:
if isinstance(finetune, str):
self.load(finetune, devices=devices)
else:
self.load(save_dir, devices=devices)
logger.info(
f'Finetune model loaded with {sum(p.numel() for p in self.model.parameters() if p.requires_grad)}'
f'/{sum(p.numel() for p in self.model.parameters())} trainable/total parameters.'
)
self.on_config_ready(**self.config, save_dir=save_dir)
trn = self.build_dataloader(**merge_dict(config,
data=trn_data,
batch_size=batch_size,
shuffle=True,
training=True,
device=first_device,
logger=logger,
vocabs=self.vocabs,
overwrite=True))
dev = self.build_dataloader(
**merge_dict(config,
data=dev_data,
batch_size=batch_size,
shuffle=False,
training=None,
device=first_device,
logger=logger,
vocabs=self.vocabs,
overwrite=True)) if dev_data else None
if not finetune:
flash('[yellow]Building model [blink]...[/blink][/yellow]')
self.model = self.build_model(**merge_dict(config, training=True))
flash('')
logger.info(
f'Model built with {sum(p.numel() for p in self.model.parameters() if p.requires_grad)}'
f'/{sum(p.numel() for p in self.model.parameters())} trainable/total parameters.'
)
assert self.model, 'build_model is not properly implemented.'
_description = repr(self.model)
if len(_description.split('\n')) < 10:
logger.info(_description)
self.save_config(save_dir)
self.save_vocabs(save_dir)
self.to(devices, logger)
if _device_placeholder and first_device >= 0:
del _dummy_placeholder
criterion = self.build_criterion(**merge_dict(config, trn=trn))
optimizer = self.build_optimizer(
**merge_dict(config, trn=trn, criterion=criterion))
metric = self.build_metric(**self.config)
if hasattr(trn, 'dataset') and dev and hasattr(dev, 'dataset'):
if trn.dataset and dev.dataset:
logger.info(
f'{len(trn.dataset)}/{len(dev.dataset)} samples in trn/dev set.'
)
if hasattr(trn, '__len__') and dev and hasattr(dev, '__len__'):
trn_size = len(trn) // self.config.get('gradient_accumulation', 1)
ratio_width = len(f'{trn_size}/{trn_size}')
else:
ratio_width = None
return self.execute_training_loop(**merge_dict(config,
trn=trn,
dev=dev,
epochs=epochs,
criterion=criterion,
optimizer=optimizer,
metric=metric,
logger=logger,
save_dir=save_dir,
devices=devices,
ratio_width=ratio_width,
trn_data=trn_data,
dev_data=dev_data,
eval_trn=eval_trn,
overwrite=True))
def load_from_meta_file(save_dir: str, meta_filename='meta.json', transform_only=False, verbose=HANLP_VERBOSE,
**kwargs) -> Component:
"""
Load a component from a ``meta.json`` (legacy TensorFlow component) or a ``config.json`` file.
Args:
save_dir: The identifier.
meta_filename (str): The meta file of that saved component, which stores the classpath and version.
transform_only: Load and return only the transform.
**kwargs: Extra parameters passed to ``component.load()``.
Returns:
A component.
"""
identifier = save_dir
load_path = save_dir
save_dir = get_resource(save_dir)
if save_dir.endswith('.json'):
meta_filename = os.path.basename(save_dir)
save_dir = os.path.dirname(save_dir)
metapath = os.path.join(save_dir, meta_filename)
if not os.path.isfile(metapath):
tf_model = False
metapath = os.path.join(save_dir, 'config.json')
else:
tf_model = True
cls = None
if not os.path.isfile(metapath):
tips = ''
if save_dir.isupper():
from difflib import SequenceMatcher
similar_keys = sorted(pretrained.ALL.keys(),
key=lambda k: SequenceMatcher(None, k, identifier).ratio(),
reverse=True)[:5]
tips = f'Check its spelling based on the available keys:\n' + \
f'{sorted(pretrained.ALL.keys())}\n' + \
f'Tips: it might be one of {similar_keys}'
# These components are not intended to be loaded in this way, but I'm tired of explaining it again and again
if identifier in pretrained.word2vec.ALL.values():
save_dir = os.path.dirname(save_dir)
metapath = os.path.join(save_dir, 'config.json')
save_json({'classpath': 'hanlp.layers.embeddings.word2vec.Word2VecEmbeddingComponent',
'embed': {'classpath': 'hanlp.layers.embeddings.word2vec.Word2VecEmbedding',
'embed': identifier, 'field': 'token', 'normalize': 'l2'},
'hanlp_version': version.__version__}, metapath)
elif identifier in pretrained.fasttext.ALL.values():
save_dir = os.path.dirname(save_dir)
metapath = os.path.join(save_dir, 'config.json')
save_json({'classpath': 'hanlp.layers.embeddings.fast_text.FastTextEmbeddingComponent',
'embed': {'classpath': 'hanlp.layers.embeddings.fast_text.FastTextEmbedding',
'filepath': identifier, 'src': 'token'},
'hanlp_version': version.__version__}, metapath)
else:
raise FileNotFoundError(f'The identifier {save_dir} resolves to a nonexistent meta file {metapath}. {tips}')
meta: dict = load_json(metapath)
cls = meta.get('classpath', cls)
if not cls:
cls = meta.get('class_path', None) # For older version
if tf_model:
# tf models are trained with version < 2.1. To migrate them to 2.1, map their classpath to new locations
upgrade = {
'hanlp.components.tok_tf.TransformerTokenizerTF': 'hanlp.components.tokenizers.tok_tf.TransformerTokenizerTF',
'hanlp.components.pos_tf.RNNPartOfSpeechTaggerTF': 'hanlp.components.taggers.pos_tf.RNNPartOfSpeechTaggerTF',
'hanlp.components.pos_tf.CNNPartOfSpeechTaggerTF': 'hanlp.components.taggers.pos_tf.CNNPartOfSpeechTaggerTF',
'hanlp.components.ner_tf.TransformerNamedEntityRecognizerTF': 'hanlp.components.ner.ner_tf.TransformerNamedEntityRecognizerTF',
'hanlp.components.parsers.biaffine_parser.BiaffineDependencyParser': 'hanlp.components.parsers.biaffine_parser_tf.BiaffineDependencyParserTF',
'hanlp.components.parsers.biaffine_parser.BiaffineSemanticDependencyParser': 'hanlp.components.parsers.biaffine_parser_tf.BiaffineSemanticDependencyParserTF',
'hanlp.components.tok_tf.NgramConvTokenizerTF': 'hanlp.components.tokenizers.tok_tf.NgramConvTokenizerTF',
'hanlp.components.classifiers.transformer_classifier.TransformerClassifier': 'hanlp.components.classifiers.transformer_classifier_tf.TransformerClassifierTF',
'hanlp.components.taggers.transformers.transformer_tagger.TransformerTagger': 'hanlp.components.taggers.transformers.transformer_tagger_tf.TransformerTaggerTF',
'hanlp.components.tok.NgramConvTokenizer': 'hanlp.components.tokenizers.tok_tf.NgramConvTokenizerTF',
}
cls = upgrade.get(cls, cls)
assert cls, f'{meta_filename} doesn\'t contain classpath field'
try:
obj: Component = object_from_classpath(cls)
if hasattr(obj, 'load'):
if transform_only:
# noinspection PyUnresolvedReferences
obj.load_transform(save_dir)
else:
if os.path.isfile(os.path.join(save_dir, 'config.json')):
obj.load(save_dir, verbose=verbose, **kwargs)
else:
obj.load(metapath, **kwargs)
obj.config['load_path'] = load_path
return obj
except ModuleNotFoundError as e:
if isdebugging():
raise e from None
else:
raise ModuleNotFoundError(
'Some modules required by this model are missing. Please install the full version:'
'\n\n\tpip install hanlp[full]') from None
except ValueError as e:
if e.args and isinstance(e.args[0], str) and 'Internet connection' in e.args[0]:
raise ConnectionError(
'Hugging Face 🤗 Transformers failed to download because your Internet connection is either off or bad.\n'
'See https://hanlp.hankcs.com/docs/install.html#server-without-internet for solutions.') \
from None
raise e from None
except Exception as e:
# Some users often install an incompatible tf and put the blame on HanLP. Teach them the basics.
try:
you_installed_wrong_versions, extras = check_version_conflicts(extras=('full',) if tf_model else None)
except:
you_installed_wrong_versions, extras = None, None
if you_installed_wrong_versions:
raise version.NotCompatible(you_installed_wrong_versions + '\nPlease reinstall HanLP in the right way:' +
'\n\n\tpip install --upgrade hanlp' + (
f'[{",".join(extras)}]' if extras else '')) from None
eprint(f'Failed to load {identifier}.')
from pkg_resources import parse_version
model_version = meta.get("hanlp_version", '2.0.0-alpha.0')
if model_version == '2.0.0': # Quick fix: the first version used a wrong string
model_version = '2.0.0-alpha.0'
model_version = parse_version(model_version)
installed_version = parse_version(version.__version__)
try:
latest_version = get_latest_info_from_pypi()
except:
latest_version = None
if model_version > installed_version:
eprint(f'{identifier} was created with hanlp-{model_version}, '
f'while you are running a lower version: {installed_version}. ')
if installed_version != latest_version:
eprint(
f'Please upgrade HanLP with:\n'
f'\n\tpip install --upgrade hanlp\n')
eprint(
'If the problem still persists, please submit an issue to https://github.com/hankcs/HanLP/issues\n'
'When reporting an issue, make sure to paste the FULL ERROR LOG below.')
eprint(f'{"ERROR LOG BEGINS":=^80}')
import platform
eprint(f'OS: {platform.platform()}')
eprint(f'Python: {platform.python_version()}')
import torch
eprint(f'PyTorch: {torch.__version__}')
if tf_model:
try:
import tensorflow
tf_version = tensorflow.__version__
except ModuleNotFoundError:
tf_version = 'not installed'
eprint(f'TensorFlow: {tf_version}')
eprint(f'HanLP: {version.__version__}')
import sys
sys.stderr.flush()
try:
if e.args and isinstance(e.args, tuple) and isinstance(e.args[0], str):
e.args = (e.args[0] + f'\n{"ERROR LOG ENDS":=^80}',) + e.args[1:]
except:
pass
raise e from None