def set_up(): global setup_flag if setup_flag: return try: import laserembeddings laserembeddings.Laser().embed_sentences(['This is a test.'], lang='en') except FileNotFoundError: import runpy import sys argv = [v for v in sys.argv] sys.argv = [sys.argv[0], 'download-models'] runpy.run_module('laserembeddings', run_name='__main__') sys.argv = argv setup_flag = True
def _get_sent_laser( hypothesis: List[str], references: List[List[str]], extra_args: Optional[Dict[str, str]] = None ) -> List[float]: set_up() import laserembeddings import langid import logging logging.getLogger('langid').setLevel(logging.WARNING) lang = langid.classify(references[0][0])[0] laser = laserembeddings.Laser() hypo_emb = laser.embed_sentences(hypothesis, lang=lang) ref_emb = laser.embed_sentences(references[0], lang=lang) inner_product = np.sum(hypo_emb * ref_emb, axis=1) hypo_l2 = np.linalg.norm(hypo_emb, axis=1) ref_l2 = np.linalg.norm(ref_emb, axis=1) return (inner_product / (hypo_l2 * ref_l2)).tolist()