def test_present_embeddings_run_server(self): def sweded_which(prog): return None which = shutil.which shutil.which = sweded_which browser = os.getenv("BROWSER", "") os.environ["BROWSER"] = "" try: with tempfile.TemporaryDirectory( prefix="sourced.ml-test-") as tmpdir: with captured_output() as (stdout, _, _): present_embeddings(tmpdir, True, ["one"], [str(i) for i in range(5)], [(i, i) for i in range(5)]) with open(os.path.join(tmpdir, "id2vec.json")) as fin: json.load(fin) with open(os.path.join(tmpdir, "id2vec_meta.tsv")) as fin: self.assertEqual(fin.read(), "0\n1\n2\n3\n4\n") with open(os.path.join(tmpdir, "id2vec_data.tsv")) as fin: self.assertEqual(fin.read(), "0\t0\n1\t1\n2\t2\n3\t3\n4\t4\n") self.assertIn( "\thttp://projector.tensorflow.org/?config=http://0.0.0.0:8000/id2vec.json\n", stdout.getvalue()) finally: shutil.which = which os.environ["BROWSER"] = browser web_server.stop()
def test_present_embeddings(self): with tempfile.TemporaryDirectory(prefix="sourced.ml-test-") as tmpdir: tmpdir = os.path.join(tmpdir, "1", "2") present_embeddings(tmpdir, False, ["one", "two"], [(str(i), "x") for i in range(5)], [(i, i) for i in range(5)]) with open(os.path.join(tmpdir, "id2vec.json")) as fin: json.load(fin) with open(os.path.join(tmpdir, "id2vec_meta.tsv")) as fin: self.assertEqual(fin.read(), "one\ttwo\n0\tx\n1\tx\n2\tx\n3\tx\n4\tx\n") with open(os.path.join(tmpdir, "id2vec_data.tsv")) as fin: self.assertEqual(fin.read(), "0\t0\n1\t1\n2\t2\n3\t3\n4\t4\n")
def projector_entry(args): MAX_TOKENS = 10000 # hardcoded in Tensorflow Projector log = logging.getLogger("id2vec_projector") id2vec = Id2Vec(log_level=args.log_level).load(source=args.input) if args.docfreq: from sourced.ml.models import DocumentFrequencies df = DocumentFrequencies(log_level=args.log_level).load(source=args.docfreq) else: df = None if len(id2vec) < MAX_TOKENS: tokens = numpy.arange(len(id2vec), dtype=int) if df is not None: freqs = [df.get(id2vec.tokens[i], 0) for i in tokens] else: freqs = None else: if df is not None: log.info("Filtering tokens through docfreq") items = [] for token, idx in id2vec.items(): try: items.append((df[token], idx)) except KeyError: continue log.info("Sorting") items.sort(reverse=True) tokens = [i[1] for i in items[:MAX_TOKENS]] freqs = [i[0] for i in items[:MAX_TOKENS]] else: log.warning("You have not specified --df => picking random %d tokens", MAX_TOKENS) numpy.random.seed(777) tokens = numpy.random.choice( numpy.arange(len(id2vec), dtype=int), MAX_TOKENS, replace=False) freqs = None log.info("Gathering the embeddings") embeddings = numpy.vstack([id2vec.embeddings[i] for i in tokens]) tokens = [id2vec.tokens[i] for i in tokens] labels = ["subtoken"] if freqs is not None: labels.append("docfreq") tokens = list(zip(tokens, (str(i) for i in freqs))) import sourced.ml.utils.projector as projector projector.present_embeddings(args.output, not args.no_browser, labels, tokens, embeddings) if not args.no_browser: projector.wait()