def eval_sdp_and_ensemble(parser,
devfile,
dep_dev_output,
save_dir,
lang,
logger,
do_eval=True):
long_sent: dict = load_json(devfile.replace('.short.conllu', '.long.json'))
long_sent = dict((int(x), y) for x, y in long_sent.items())
sdp_dev_output = f'{save_dir}/{os.path.basename(devfile.replace(".conllu", ".sdp.pred.conllu"))}'
sdp_dev_output = sdp_dev_output.replace('.short', '')
if not os.path.isfile(sdp_dev_output) or do_eval:
if not parser.model:
parser.load(save_dir)
scores = parser.evaluate(devfile,
save_dir,
warm_up=False,
ret_scores=True,
logger=logger,
batch_size=256 if lang == 'cs' else None)[-1]
sdp_to_dag(parser, scores, sdp_dev_output, long_sent)
score = evaluate(devfile.replace('.short', ''), sdp_dev_output)
final_sdp_dev_output = sdp_dev_output.replace('.conllu', '.fixed.conllu')
sdp_elas = score["ELAS"].f1
sdp_clas = score["CLAS"].f1
logger.info(f'SDP score for {lang}:')
logger.info(f'ELAS: {sdp_elas * 100:.2f} - CLAS:{sdp_clas * 100:.2f}')
print(f'Model saved in {save_dir}')
ensemble_output = f'{save_dir}/{os.path.basename(devfile.replace(".conllu", ".ensemble.pred.conllu"))}'
if not os.path.isfile(sdp_dev_output) or do_eval:
sdp_to_dag(parser, scores, ensemble_output, long_sent, dep_dev_output)
score = evaluate(devfile.replace('.short', ''), ensemble_output)
final_ensemble_output = ensemble_output.replace('.conllu', '.fixed.conllu')
logger.info(f'Ensemble score for {lang}:')
ensemble_elas = score["ELAS"].f1
logger.info(
f'ELAS: {ensemble_elas * 100:.2f} - CLAS:{score["CLAS"].f1 * 100:.2f}')
return (sdp_elas, final_sdp_dev_output), (ensemble_elas,
final_ensemble_output)
from edparser.metrics.parsing.iwpt20_eval import evaluate
from edparser.components.parsers.biaffine_parser import BiaffineTransformerDependencyParser
from iwpt2020 import cdroot
cdroot()
save_dir = 'data/model/iwpt2020/en_bert_large_dep'
parser = BiaffineTransformerDependencyParser()
dataset = 'data/iwpt2020/train-dev/'
trnfile = f'{dataset}UD_English-EWT/en_ewt-ud-train.enhanced_collapse_empty_nodes.conllu'
devfile = f'{dataset}UD_English-EWT/en_ewt-ud-dev.enhanced_collapse_empty_nodes.conllu'
testfile = devfile
# parser.fit(trnfile,
# devfile,
# save_dir, 'bert-large-uncased-whole-word-masking',
# batch_size=128,
# warmup_steps_ratio=.1,
# samples_per_batch=150,
# # max_samples_per_batch=32,
# transformer_dropout=.33,
# learning_rate=2e-3,
# learning_rate_transformer=1e-5,
# epochs=1
# )
# parser.load(save_dir, tree='tarjan')
output = f'{testfile.replace(".conllu", ".pred.conllu")}'
# parser.evaluate(devfile, save_dir, warm_up=False, output=output)
score = evaluate(testfile, output)
print(
f'ELAS: {score["ELAS"].f1 * 100:.2f} - CLAS:{score["CLAS"].f1 * 100:.2f}')
print(f'Model saved in {save_dir}')
try:
cache = load_pickle('cache.pkl')
except FileNotFoundError:
cache = {}
for lang in ['mbert', 'bert']:
for model, color in zip(['dep', 'sdp', 'ens'], 'rgb'):
key = f'{lang}-{model}'
if key in cache:
xs, ys = cache[key]
else:
pred_file = template.replace('bert', lang).replace('dep', model)
xs = np.arange(5, 50, 5)
ys = [
evaluate(limit_len(gold_file, l),
limit_len(pred_file, l),
do_copy_cols=False,
do_enhanced_collapse_empty_nodes=True)['ELAS'].f1
for l in xs
]
cache[key] = (xs, ys)
plt.plot(xs,
ys,
label=key.replace('mbert', 'multilingual').replace(
'bert',
'language-specific').replace('dep', 'DTP').replace(
'sdp', 'DGP').replace('ens', 'ENS'),
color=color,
linestyle='-' if lang == 'mbert' else '--')
print(key)
save_pickle(cache, 'cache.pkl')
plt.xlabel('sentence length')
def run(lang, do_train=True, do_eval=True, mbert=True):
"""
Run training and decoding
:param lang: Language code, 2 letters.
:param do_train: Train model or not.
:param do_eval: Evaluate performance (generating output) or not.
:param mbert: Use mbert or language specific transformers.
"""
dataset = f'data/iwpt2020/train-dev-combined/{lang}'
trnfile = f'{dataset}/train.short.conllu'
# for idx, sent in enumerate(read_conll(trnfile)):
# print(f'\r{idx}', end='')
devfile = f'{dataset}/dev.short.conllu'
testfile = f'data/iwpt2020/test-udpipe/{lang}.fixed.short.conllu'
prefix = 'mbert'
transformer = 'bert-base-multilingual-cased'
if not mbert:
prefix = 'bert'
if lang == 'sv':
transformer = "KB/bert-base-swedish-cased"
if lang == 'ar':
transformer = "asafaya/bert-base-arabic"
elif lang == 'en':
transformer = 'albert-xxlarge-v2'
elif lang == 'ru':
transformer = "DeepPavlov/rubert-base-cased"
elif lang == 'fi':
transformer = "TurkuNLP/bert-base-finnish-cased-v1"
elif lang == 'it':
transformer = "dbmdz/bert-base-italian-cased"
elif lang == 'nl':
transformer = "wietsedv/bert-base-dutch-cased"
elif lang == 'et':
transformer = get_resource(
'http://dl.turkunlp.org/estonian-bert/etwiki-bert/pytorch/etwiki-bert-base-cased.tar.gz'
)
elif lang == 'fr':
transformer = 'camembert-base'
elif lang == 'pl':
transformer = "dkleczek/bert-base-polish-uncased-v1"
elif lang == 'sk' or lang == 'bg' or lang == 'cs':
transformer = get_resource(
'http://files.deeppavlov.ai/deeppavlov_data/bert/bg_cs_pl_ru_cased_L-12_H-768_A-12_pt.tar.gz'
)
else:
prefix = 'mbert'
save_dir = f'data/model/iwpt2020/{lang}/{prefix}_dep'
# if do_train and os.path.isdir(save_dir):
# return
strategy = tf.distribute.MirroredStrategy()
print("Number of devices: {}".format(strategy.num_replicas_in_sync))
with strategy.scope():
parser = BiaffineTransformerDependencyParser(strategy=strategy)
if do_train:
parser.fit(
trnfile,
devfile,
save_dir,
transformer,
batch_size=4096,
warmup_steps_ratio=.1,
samples_per_batch=150,
# max_samples_per_batch=75,
transformer_dropout=.33,
learning_rate=2e-3,
learning_rate_transformer=1e-5,
# max_seq_length=512,
# epochs=1
)
logger = init_logger(name='test', root_dir=save_dir, mode='w')
parser.config.tree = 'mst'
# dep_dev_output = f'{save_dir}/{os.path.basename(devfile.replace(".conllu", ".dep.pred.conllu"))}'
# if not os.path.isfile(dep_dev_output) or do_eval:
# parser.evaluate(devfile, save_dir, warm_up=False, output=dep_dev_output, logger=logger)
dep_test_output = f'{save_dir}/{os.path.basename(testfile.replace(".conllu", ".dep.pred.conllu"))}'
if not os.path.isfile(dep_test_output) or do_eval:
parser.load(save_dir, tree='mst')
parser.evaluate(testfile,
save_dir,
warm_up=False,
output=dep_test_output,
logger=None)
# score = evaluate(devfile, dep_dev_output)
# dep_dev_elas = score["ELAS"].f1
# dep_dev_clas = score["CLAS"].f1
# logger.info(f'DEP score for {lang}:')
# logger.info(f'ELAS: {dep_dev_elas * 100:.2f} - CLAS:{dep_dev_clas * 100:.2f}')
if do_train:
print(f'Model saved in {save_dir}')
save_dir = f'data/model/iwpt2020/{lang}/{prefix}_sdp'
parser = BiaffineTransformerSemanticDependencyParser()
if do_train and not os.path.isdir(save_dir):
parser.fit(
trnfile,
devfile,
save_dir,
transformer,
batch_size=1000 if lang == 'cs' else 3000,
warmup_steps_ratio=.1,
samples_per_batch=150,
# max_samples_per_batch=150,
transformer_dropout=.33,
learning_rate=2e-3,
learning_rate_transformer=1e-5,
# max_seq_length=512,
# epochs=1
)
# (sdp_dev_elas, final_sdp_dev_output), (ensemble_dev_elas, final_ensemble_dev_output) = \
# eval_sdp_and_ensemble(parser, devfile, dep_dev_output, save_dir, lang, logger)
(sdp_test_elas, final_sdp_test_output), (ensemble_test_elas, final_ensemble_test_output) = \
eval_sdp_and_ensemble(parser, testfile, dep_test_output, save_dir, lang, logger, do_eval)
save_dir = f'data/model/iwpt2020/{lang}/'
# copyfile(dep_dev_output, save_dir + 'dev.dep.conllu')
# copyfile(final_sdp_dev_output, save_dir + 'dev.sdp.conllu')
# copyfile(final_ensemble_dev_output, save_dir + 'dev.ens.conllu')
# dev_scores = [dep_dev_elas, sdp_dev_elas, ensemble_dev_elas]
# winner = max(dev_scores)
# widx = dev_scores.index(winner)
dep_test_output = merge_long_sent(dep_test_output)
evaluate(f'data/iwpt2020/test-udpipe/{lang}.fixed.conllu', dep_test_output)
dep_test_output = dep_test_output.replace('.conllu', '.fixed.conllu')
# if widx == 0:
# # dep wins, but we don't have output for dep, so let's do it below
# best_test_output = dep_test_output
# best_task = 'dep'
# elif widx == 1:
# # sdp wins
# best_test_output = final_sdp_test_output
# best_task = 'sdp'
# else:
# # ensemble wins
# best_test_output = final_ensemble_test_output
# best_task = 'ens'
#
# info = {
# 'best_task': best_task,
# 'dev_scores': dict((x, y) for x, y in zip(['dep', 'sdp', 'ens'], dev_scores))
# }
# save_json(info, save_dir + 'scores.json')
# copyfile(best_test_output, save_dir + lang + '.conllu')
# dev_json = 'data/model/iwpt2020/dev.json'
# try:
# total = load_json(dev_json)
# except FileNotFoundError:
# total = {}
# total[lang] = info
# save_json(total, dev_json)
final_root = f'data/model/iwpt2020/{prefix}'
dep_root = f'{final_root}/dep'
sdp_root = f'{final_root}/sdp'
ens_root = f'{final_root}/ens'
outputs = [
dep_test_output, final_sdp_test_output, final_ensemble_test_output
]
folders = [dep_root, sdp_root, ens_root]
for o, f in zip(outputs, folders):
os.makedirs(f, exist_ok=True)
tmp = f'/tmp/{lang}.conllu'
copyfile(o, tmp)
remove_complete_edges(tmp, tmp)
restore_collapse_edges(tmp, tmp)
conllu_quick_fix(tmp, f'{f}/{lang}.conllu')