def get_evalb_dir():
home = os.path.realpath(os.path.join(get_resource(_PTB_HOME), '../EVALB'))
evalb_path = os.path.join(home, 'evalb')
if not os.path.isfile(evalb_path):
flash(f'Compiling evalb to {home}')
with pushd(home):
run_cmd(f'make')
flash('')
if not os.path.isfile(evalb_path):
raise RuntimeError(f'Failed to compile evalb at {home}')
return home
def post_process(pred, amr_version):
pred = os.path.realpath(pred)
utils_tar_gz = get_amr_utils(amr_version)
util_dir = get_resource(utils_tar_gz)
stog_home = get_resource(
'https://github.com/jcyk/AMR-gs/archive/master.zip')
with pushd(stog_home):
run_cmd(
f'python3 -u -m stog.data.dataset_readers.amr_parsing.postprocess.postprocess '
f'--amr_path {pred} --util_dir {util_dir} --v 2')
return pred + '.post'
def remove_all_ec(path):
"""
Remove empty categories for all trees in this file and save them into a "noempty" file.
Args:
path: File path.
"""
script = get_resource('https://file.hankcs.com/bin/remove_ec.zip')
with pushd(script):
run_cmd(f'java -cp elit-ddr-0.0.5-SNAPSHOT.jar:elit-sdk-0.0.5-SNAPSHOT.jar:hanlp-1.7.8.jar:'
f'fastutil-8.1.1.jar:. demo.RemoveEmptyCategoriesTreebank {path}')
def make_gold_conll(ontonotes_path, language):
ontonotes_path = os.path.abspath(get_resource(ontonotes_path))
to_conll = get_resource(
'https://gist.githubusercontent.com/hankcs/46b9137016c769e4b6137104daf43a92/raw/66369de6c24b5ec47696ae307591f0d72c6f3f02/ontonotes_to_conll.sh'
)
to_conll = os.path.abspath(to_conll)
# shutil.rmtree(os.path.join(ontonotes_path, 'conll-2012'), ignore_errors=True)
with pushd(ontonotes_path):
try:
flash(
f'Converting [blue]{language}[/blue] to CoNLL format, '
f'this might take half an hour [blink][yellow]...[/yellow][/blink]'
)
run_cmd(f'bash {to_conll} {ontonotes_path} {language}')
flash('')
except RuntimeError as e:
flash(
f'[red]Failed[/red] to convert {language} of {ontonotes_path} to CoNLL. See exceptions for detail'
)
raise e
def smatch_eval(pred, gold, use_fast=False) -> Union[SmatchScores, F1_]:
script = get_resource(_FAST_SMATCH_SCRIPT if use_fast else _SMATCH_SCRIPT)
home = os.path.dirname(script)
pred = os.path.realpath(pred)
gold = os.path.realpath(gold)
with pushd(home):
flash('Running evaluation script [blink][yellow]...[/yellow][/blink]')
cmd = f'bash {script} {pred} {gold}'
text = run_cmd(cmd)
flash('')
return format_fast_scores(text) if use_fast else format_official_scores(
text)
def official_conll_05_evaluate(pred_path, gold_path):
script_root = get_resource(
'http://www.lsi.upc.edu/~srlconll/srlconll-1.1.tgz')
lib_path = f'{script_root}/lib'
if lib_path not in os.environ.get("PERL5LIB", ""):
os.environ['PERL5LIB'] = f'{lib_path}:{os.environ.get("PERL5LIB", "")}'
bin_path = f'{script_root}/bin'
if bin_path not in os.environ.get('PATH', ''):
os.environ['PATH'] = f'{bin_path}:{os.environ.get("PATH", "")}'
eval_info_gold_pred = run_cmd(
f'perl {script_root}/bin/srl-eval.pl {gold_path} {pred_path}')
eval_info_pred_gold = run_cmd(
f'perl {script_root}/bin/srl-eval.pl {pred_path} {gold_path}')
conll_recall = float(
eval_info_gold_pred.strip().split("\n")[6].strip().split()[5]) / 100
conll_precision = float(
eval_info_pred_gold.strip().split("\n")[6].strip().split()[5]) / 100
if conll_recall + conll_precision > 0:
conll_f1 = 2 * conll_recall * conll_precision / (conll_recall +
conll_precision)
else:
conll_f1 = 0
return conll_precision, conll_recall, conll_f1
