def encode_seq_structural_data(data : RawDataset, context_tokenizer_type : \ Callable[[List[str], int], Tokenizer], num_keywords : int, num_reserved_tokens: int) -> \ Tuple[StructDataset, Tokenizer, SimpleEmbedding]: embedding = SimpleEmbedding() hyps_and_goals = [ hyp_or_goal for hyp_and_goal in [ zip(hyps + [goal], itertools.repeat(embedding.encode_token(tactic))) for prev_tactics, hyps, goal, tactic in data ] for hyp_or_goal in hyp_and_goal ] context_tokenizer = make_keyword_tokenizer_relevance( hyps_and_goals, context_tokenizer_type, num_keywords, num_reserved_tokens) encodedData = [] for prev_tactics, hyps, goal, tactic in data: stem, rest = serapi_instance.split_tactic(tactic) encodedData.append( ([context_tokenizer.toTokenList(hyp) for hyp in hyps], context_tokenizer.toTokenList(goal), (embedding.encode_token(stem), [hyp_index(hyps, arg) for arg in get_symbols(rest)]))) return encodedData, context_tokenizer, embedding
def encode_hyparg_data(data : RawDataset, tokenizer_type : Callable[[List[str], int], Tokenizer], num_keywords : int, num_reserved_tokens : int, max_args : int, max_hyps : int, encoded_length : int, entropy_data_size : int, num_threads : Optional[int] = None) -> \ Tuple[StructDataset, Tokenizer, SimpleEmbedding]: stem_embedding = SimpleEmbedding() data_list = list(data) if len(data_list) <= entropy_data_size: subset = data_list else: subset = random.sample(data_list, entropy_data_size) tokenizer = make_keyword_tokenizer_relevance( [(context, stem_embedding.encode_token(serapi_instance.get_stem(tactic))) for relevant_lemmas, prev_tactics, hyps, context, tactic in subset], tokenizer_type, num_keywords, num_reserved_tokens) termEncoder = functools.partial(getNGramTokenbagVector, 1, tokenizer.numTokens()) with multiprocessing.Pool(num_threads) as pool: hyps, contexts, tactics = zip(*data_list) encoded_contexts = pool.imap(functools.partial( _encode, tokenizer, termEncoder), contexts) encoded_hyps = pool.imap(functools.partial( _encode_hyps, tokenizer, termEncoder, max_hyps, encoded_length), contexts) encoded_tactics = pool.imap( functools.partial(encode_tactic_structure, stem_embedding, max_args), zip(hyps, tactics)) result = list(zip(encoded_hyps, encoded_contexts, encoded_tactics)) tokenizer.freezeTokenList() return result, tokenizer, stem_embedding
def tokenize_goals(data : StrictEmbeddedDataset, args : Namespace) \ -> Tuple[Tokenizer, List[Sentence]]: if args.load_tokens and Path2(args.load_tokens).exists(): print("Loading tokens from {}".format(args.load_tokens)) with open(args.load_tokens, 'rb') as f: tokenizer = pickle.load(f) assert isinstance(tokenizer, Tokenizer) else: start = time.time() print("Picking tokens...", end="") sys.stdout.flush() subset : Sequence[EmbeddedSample] if args.num_relevance_samples > len(data): subset = data else: subset = random.sample(data, args.num_relevance_samples) tokenizer = make_keyword_tokenizer_relevance( [(goal, next_tactic) for prev_tactics, hypotheses, goal, next_tactic in subset], tokenizers[args.tokenizer], args.num_keywords, TOKEN_START, args.num_threads) print("{}s".format(time.time() - start)) if args.save_tokens: print("Saving tokens to {}".format(args.save_tokens)) with open(args.save_tokens, 'wb') as f: pickle.dump(tokenizer, f) if args.print_keywords: print("Keywords are {}".format(tokenizer.listTokens())) start = time.time() print("Tokenizing...", end="") sys.stdout.flush() tokenized_data = tokenize_data(tokenizer, data, args.num_threads) print("{:.2f}s".format(time.time() - start)) return tokenizer, [goal for prev_tactics, hypotheses, goal, tactic in tokenized_data]
def encode_seq_classify_data(data : RawDataset, tokenizer_type : Callable[[List[str], int], Tokenizer], num_keywords : int, num_reserved_tokens : int, save_tokens : Optional[str] = None, load_tokens : Optional[str] = None, num_relevance_samples : int = 1000) \ -> Tuple[ClassifySequenceDataset, Tokenizer, SimpleEmbedding]: embedding = SimpleEmbedding() subset = RawDataset(random.sample(data, num_relevance_samples)) if load_tokens: print("Loading tokens from {}".format(load_tokens)) tokenizer = torch.load(load_tokens) else: start = time.time() print("Picking tokens...", end="") sys.stdout.flush() tokenizer = make_keyword_tokenizer_relevance( [(context, embedding.encode_token(get_stem(tactic))) for prev_tactics, hyps, context, tactic in subset], tokenizer_type, num_keywords, num_reserved_tokens) print("{}s".format(time.time() - start)) if save_tokens: print("Saving tokens to {}".format(save_tokens)) torch.save(tokenizer, save_tokens) with multiprocessing.Pool(None) as pool: result = [(goal, embedding.encode_token(tactic)) for goal, tactic in chain.from_iterable( pool.imap( functools.partial(encode_seq_classify_data_worker__, tokenizer), chunks(data, 1024)))] tokenizer.freezeTokenList() return result, tokenizer, embedding
def _encode_data(self, data : RawDataset, args : Namespace) \ -> Tuple[DatasetType, TokenizerEmbeddingState]: preprocessed_data = self._preprocess_data(data, args) embedding = SimpleEmbedding() embedded_data: EmbeddedDataset with multiprocessing.Pool(args.num_threads) as pool: stemmed_data = pool.imap(stemmify_data, preprocessed_data, chunksize=10240) lazy_embedded_data = LazyEmbeddedDataset( (EmbeddedSample(prev_tactics, hypotheses, goal, embedding.encode_token(tactic)) for (prev_tactics, hypotheses, goal, tactic) in stemmed_data)) if args.load_tokens: print("Loading tokens from {}".format(args.load_tokens)) with open(args.load_tokens, 'rb') as f: tokenizer = pickle.load(f) assert isinstance(tokenizer, Tokenizer) embedded_data = lazy_embedded_data else: # Force the embedded data for picking keywords forced_embedded_data = StrictEmbeddedDataset( list(lazy_embedded_data.data)) subset = StrictEmbeddedDataset( random.sample(forced_embedded_data, args.num_relevance_samples)) embedded_data = forced_embedded_data start = time.time() print("Picking tokens...", end="") sys.stdout.flush() tokenizer = make_keyword_tokenizer_relevance([ (goal, next_tactic) for prev_tactics, hypotheses, goal, next_tactic in subset ], tokenizers[args.tokenizer], args.num_keywords, TOKEN_START, args.num_threads) del subset print("{}s".format(time.time() - start)) if args.save_tokens: print("Saving tokens to {}".format(args.save_tokens)) assert isinstance(tokenizer, Tokenizer) with open(args.save_tokens, 'wb') as f: pickle.dump(tokenizer, f) if args.print_keywords: print("Keywords are {}".format(tokenizer.listTokens())) print("Tokenizing...") tokenized_data = tokenize_data(tokenizer, embedded_data, args.num_threads) gc.collect() return self._encode_tokenized_data(tokenized_data, args, tokenizer, embedding), \ TokenizerEmbeddingState(tokenizer, embedding)