def emb(): meta = Meta( id='test_1B_3k_6d_2q' ) pq = PQ( vectors=3, dim=6, qdim=2, # 1 0 0 | 1 0 0 # 0 1 1 | 0 0 0 # 0 0 0 | 0 1 0 centroids=3, indexes=np.array([ # vectors x qdim [0, 1], [1, 0], [2, 2] ]).astype(np.uint8), codes=np.array([ # qdim x centroids x chunk [[1, 0, 0], [0, 1, 1], [0, 0, 0]], [[0, 0, 0], [1, 0, 0], [0, 1, 0]], ]).astype(np.float32), ) vocab = Vocab( words=['a', 'b', 'c'], counts=[1, 2, 3] ) return Navec(meta, vocab, pq)
def pack_(vocab, pq, id): meta = Meta(id) with open_bin(vocab) as file: vocab = Vocab.from_file(file) with open_bin(pq) as file: pq = PQ.from_file(file) path = 'navec_%s.tar' % id log_info('Dumping %s', path) Navec(meta, vocab, pq).dump(path)
def pack(args): meta = Meta(args.id) with open_bin(args.vocab) as file: vocab = Vocab.from_file(file) with open_bin(args.pq) as file: pq = PQ.from_file(file) path = 'navec_%s.tar' % args.id log_info('Dumping %s', path) Navec(meta, vocab, pq).dump(path)
def quantize_(emb, output, subdim, sample, iterations): with open(emb) as file: log_info('Load %s', emb) words, weights = parse_glove_emb(file) log_info( 'PQ, subdim: %d, sample: %d, iterations: %d', subdim, sample, iterations ) pq = quantize__(weights, subdim, sample, iterations) vocab = Vocab(words) log_info('Dump %s', output) Navec(vocab, pq).dump(output)