def tokenize_goals(data : StrictEmbeddedDataset, args : Namespace) \
-> Tuple[Tokenizer, List[Sentence]]:
if args.load_tokens and Path2(args.load_tokens).exists():
print("Loading tokens from {}".format(args.load_tokens))
with open(args.load_tokens, 'rb') as f:
tokenizer = pickle.load(f)
assert isinstance(tokenizer, Tokenizer)
else:
start = time.time()
print("Picking tokens...", end="")
sys.stdout.flush()
subset : Sequence[EmbeddedSample]
if args.num_relevance_samples > len(data):
subset = data
else:
subset = random.sample(data, args.num_relevance_samples)
tokenizer = make_keyword_tokenizer_relevance(
[(goal, next_tactic) for
prev_tactics, hypotheses, goal, next_tactic in subset],
tokenizers[args.tokenizer], args.num_keywords, TOKEN_START, args.num_threads)
print("{}s".format(time.time() - start))
if args.save_tokens:
print("Saving tokens to {}".format(args.save_tokens))
with open(args.save_tokens, 'wb') as f:
pickle.dump(tokenizer, f)
if args.print_keywords:
print("Keywords are {}".format(tokenizer.listTokens()))
start = time.time()
print("Tokenizing...", end="")
sys.stdout.flush()
tokenized_data = tokenize_data(tokenizer, data, args.num_threads)
print("{:.2f}s".format(time.time() - start))
return tokenizer, [goal for prev_tactics, hypotheses, goal, tactic in tokenized_data]
def _encode_data(self, data : RawDataset, args : Namespace) \
-> Tuple[DatasetType, TokenizerEmbeddingState]:
preprocessed_data = self._preprocess_data(data, args)
embedding = SimpleEmbedding()
embedded_data: EmbeddedDataset
with multiprocessing.Pool(args.num_threads) as pool:
stemmed_data = pool.imap(stemmify_data,
preprocessed_data,
chunksize=10240)
lazy_embedded_data = LazyEmbeddedDataset(
(EmbeddedSample(prev_tactics, hypotheses, goal,
embedding.encode_token(tactic))
for (prev_tactics, hypotheses, goal, tactic) in stemmed_data))
if args.load_tokens:
print("Loading tokens from {}".format(args.load_tokens))
with open(args.load_tokens, 'rb') as f:
tokenizer = pickle.load(f)
assert isinstance(tokenizer, Tokenizer)
embedded_data = lazy_embedded_data
else:
# Force the embedded data for picking keywords
forced_embedded_data = StrictEmbeddedDataset(
list(lazy_embedded_data.data))
subset = StrictEmbeddedDataset(
random.sample(forced_embedded_data,
args.num_relevance_samples))
embedded_data = forced_embedded_data
start = time.time()
print("Picking tokens...", end="")
sys.stdout.flush()
tokenizer = make_keyword_tokenizer_relevance([
(goal, next_tactic)
for prev_tactics, hypotheses, goal, next_tactic in subset
], tokenizers[args.tokenizer], args.num_keywords, TOKEN_START,
args.num_threads)
del subset
print("{}s".format(time.time() - start))
if args.save_tokens:
print("Saving tokens to {}".format(args.save_tokens))
assert isinstance(tokenizer, Tokenizer)
with open(args.save_tokens, 'wb') as f:
pickle.dump(tokenizer, f)
if args.print_keywords:
print("Keywords are {}".format(tokenizer.listTokens()))
print("Tokenizing...")
tokenized_data = tokenize_data(tokenizer, embedded_data,
args.num_threads)
gc.collect()
return self._encode_tokenized_data(tokenized_data, args, tokenizer, embedding), \
TokenizerEmbeddingState(tokenizer, embedding)