def set_up():
global setup_flag
if setup_flag:
return
try:
import laserembeddings
laserembeddings.Laser().embed_sentences(['This is a test.'], lang='en')
except FileNotFoundError:
import runpy
import sys
argv = [v for v in sys.argv]
sys.argv = [sys.argv[0], 'download-models']
runpy.run_module('laserembeddings', run_name='__main__')
sys.argv = argv
setup_flag = True
def _get_sent_laser(
hypothesis: List[str], references: List[List[str]],
extra_args: Optional[Dict[str, str]] = None
) -> List[float]:
set_up()
import laserembeddings
import langid
import logging
logging.getLogger('langid').setLevel(logging.WARNING)
lang = langid.classify(references[0][0])[0]
laser = laserembeddings.Laser()
hypo_emb = laser.embed_sentences(hypothesis, lang=lang)
ref_emb = laser.embed_sentences(references[0], lang=lang)
inner_product = np.sum(hypo_emb * ref_emb, axis=1)
hypo_l2 = np.linalg.norm(hypo_emb, axis=1)
ref_l2 = np.linalg.norm(ref_emb, axis=1)
return (inner_product / (hypo_l2 * ref_l2)).tolist()