def train(cls):
parser = argparse.ArgumentParser(description='Train a pos tagger')
parser.add_argument('--config_path',
type=str,
help='config file in json format')
parser.add_argument('--train_path',
type=str,
help='file path to the training set')
parser.add_argument('--dev_path',
type=str,
help='file path to the development set')
parser.add_argument('--model_path',
type=str,
help='file path where the model will be saved')
parser.add_argument('--word_embeddings',
type=str,
default=':'.join(
('fasttext', 'crawl-300d-2M-subword')),
help='word embeddings to use')
parser.add_argument('--flair_embeddings',
dest='flair',
action='store_false',
help='use Flair embeddings')
parser.add_argument('--learning_rate', type=float, default=0.1)
parser.add_argument('--mini_batch_size', type=int, default=32)
parser.add_argument('--max_epochs', type=int, default=100)
parser.add_argument('--anneal_factor', type=float, default=0.5)
parser.add_argument('--embeddings_in_memory',
action='store_true',
help='store embeddings in GPU memory')
args = None
try:
args = parser.parse_args(sys.argv[3:])
args = merge_args_with_config(args)
args['pretrained_embeddings'] = args['word_embeddings'].split(':')
for k in ['train_path', 'dev_path', 'model_path']:
if not args[k]:
eprint('--{} is required'.format(k))
exit(1)
args['trn_docs'] = conll_to_documents(args['train_path'],
headers={
0: 'text',
1: 'pos'
})
args['dev_docs'] = conll_to_documents(args['dev_path'],
headers={
0: 'text',
1: 'pos'
})
except SystemExit:
parser.print_help()
exit(1)
tagger = POSFlairTagger()
tagger.train(**args)
def evaluate(cls):
parser = argparse.ArgumentParser(description='Evaluate a pos tagger')
parser.add_argument('--model_path',
type=str,
default=ELIT_POS_FLAIR_EN_MIXED,
help='file path to the saved model')
parser.add_argument('--test_path',
type=str,
required=True,
help='gold file in tsv format')
args = None
try:
args = parser.parse_args(sys.argv[3:])
except SystemExit:
parser.print_help()
exit(1)
tagger = POSFlairTagger()
tagger.load(args.model_path)
tagger.evaluate(
conll_to_documents(args.test_path, headers={
0: 'text',
1: 'pos'
}))
**kwargs):
"""
Load model
:param model_path: path to stored model
:param model_root: the root for model_path
:param kwargs: not used
:return: self
"""
super().load(model_path, model_root=model_root, **kwargs)
return self
if __name__ == '__main__':
tagger = NERFlairTagger(mx.gpu(3))
model_path = 'data/model/ner/jumbo'
# tagger.train(conll_to_documents('data/conll-03/debug/eng.trn'), conll_to_documents('data/conll-03/debug/eng.dev'),
# model_path, pretrained_embeddings='data/embedding/glove/glove.6B.100d.debug.txt',
# forward_language_model='data/model/lm-news-forward',
# backward_language_model='data/model/lm-news-backward',
# max_epochs=1)
tagger.load(model_path)
test = conll_to_documents('data/dat/en-ner.tst',
headers={
0: 'text',
1: 'pos',
2: 'ner'
})
sent = tagger.decode(test)[0][SENS][3]
print(sent[NER])
print(tagger.evaluate(test))
# -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-10-03 16:27
from types import SimpleNamespace
from elit.component.embedding.fasttext import FastText
from elit.component.tagger.corpus import conll_to_documents, label_map_from_conll
from elit.component.token_tagger.cnn import CNNTokenTagger
from elit.util.mx import mxnet_prefer_gpu
label_map = label_map_from_conll('data/ptb/pos/dev.tsv')
print(label_map)
tagger = CNNTokenTagger(ctx=mxnet_prefer_gpu(), key='pos',
embs=[FastText('https://elit-models.s3-us-west-2.amazonaws.com/cc.en.300.bin.zip')],
input_config=SimpleNamespace(row=100, col=5, dropout=0.5),
output_config=SimpleNamespace(num_class=len(label_map), flatten=True),
label_map=label_map
)
# 94.38
save_path = 'data/model/cnntagger'
tagger.train(conll_to_documents('data/ptb/pos/train.tsv', headers={0: 'text', 1: 'pos'}, gold=True),
conll_to_documents('data/ptb/pos/dev.tsv', headers={0: 'text', 1: 'pos'}, gold=True),
save_path)
tagger.load(save_path)
# Parameter 'dense1.weight' is missing in file 'data/model/cnntagger.params', which contains parameters: 'dense0.weight', 'dense0.bias'. Set allow_missing=True to ignore missing parameters.
tagger.evaluate(conll_to_documents('data/ptb/pos/dev.tsv', headers={0: 'text', 1: 'pos'}))
# -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-06-16 11:37
from elit.component import POSFlairTagger
from elit.component.tagger.corpus import conll_to_documents
from elit.resources.pre_trained_models import ELIT_POS_FLAIR_EN_MIXED
tagger = POSFlairTagger()
model_path = ELIT_POS_FLAIR_EN_MIXED
tagger.load(model_path)
print(tagger.evaluate(conll_to_documents('data/dat/en-ddr.tst', headers={1: 'text', 3: 'pos'}),
output_dir='data/dat')) # 97.96%
"""
Load model
:param model_path: path to stored model
:param model_root: the root for model_path
:param kwargs: not used
:return: self
"""
super().load(model_path, model_root=model_root, **kwargs)
return self
if __name__ == '__main__':
tagger = POSFlairTagger(context=mx.gpu(3))
model_path = 'data/model/pos/wsj'
tagger.load(model_path)
# tagger.train(conll_to_documents('data/dat/en-pos.dev', headers={0: 'text', 1: 'pos'}),
# conll_to_documents('data/dat/en-pos.dev', headers={0: 'text', 1: 'pos'}),
# model_path, pretrained_embeddings='data/embedding/glove/glove.6B.100d.debug.txt',
# forward_language_model='data/model/lm-news-forward',
# backward_language_model='data/model/lm-news-backward',
# max_epochs=1,
# embeddings_in_memory=False)
test = conll_to_documents('data/wsj-pos/test.tsv',
headers={
0: 'text',
1: 'pos'
})
sent = tagger.decode(test)[0][SENS][3]
print(sent[POS])
print(tagger.evaluate(test))