def __init__(self, ckpt_path='./Res/PretrainModel/2019_12_27_08_48_21/'): checkpoint = Checkpoint.load(ckpt_path) self.seq2seq = checkpoint.model self.input_vocab = checkpoint.input_vocab self.output_vocab = checkpoint.output_vocab self.predictor = Predictor(self.seq2seq, self.input_vocab, self.output_vocab)
def dev_predict(task_path, src_str, is_plot=True): """Helper used to visualize and understand why and what the model predicts. Args: task_path (str): path to the saved task directory containing, amongst other, the model. src_str (str): source sentence that will be used to predict. is_plot (bool, optional): whether to plots the attention pattern. Returns: out_words (list): decoder predictions. other (dictionary): additional information used for predictions. test (dictionary): additional information that is only stored in dev mode. These can include temporary variables that do not have to be stored in `other` but that can still be interesting to inspect. """ check = Checkpoint.load(task_path) check.model.set_dev_mode() predictor = Predictor(check.model, check.input_vocab, check.output_vocab) out_words, other = predictor.predict(src_str.split()) test = dict() for k, v in other["test"].items(): tensor = v if isinstance(v, torch.Tensor) else torch.cat(v) test[k] = tensor.detach().cpu().numpy().squeeze()[:other["length"][0]] # except: # for using "step" # test[k] = v if is_plot: visualizer = AttentionVisualizer(task_path) visualizer(src_str) return out_words, other, test
def test_predict(self): predictor = Predictor(self.seq2seq, self.dataset.input_vocab, self.dataset.output_vocab) src_seq = ["I", "am", "fat"] tgt_seq = predictor.predict(src_seq) for tok in tgt_seq: self.assertTrue(tok in self.dataset.output_vocab._token2index)
def predict_with_checkpoint(checkpoint_path, sequence, hierarchial = False, remote = None, word_vectors = None): checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = None, None if word_vectors != None: input_vects = Word2Vectors(input_vocab, word_vectors, word_vectors.dim_size) output_vects = Word2Vectors(output_vocab, word_vectors, word_vectors.dim_size) seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = input_vects, output_vects seq2seq.decoder = TopKDecoder(seq2seq.decoder, 5) if not hierarchial: predictor = Predictor(seq2seq, input_vocab, output_vocab) seq = sequence.strip().split() else: predictor = HierarchialPredictor(seq2seq, input_vocab, output_vocab) seq = ['|'.join(x.split()) for x in sequence] return ' '.join(predictor.predict(seq))
def predict(expt_dir, seq_str, date, epoch, step, n=3): seq = seq_str.strip().split() checkpoint_path = os.path.join(expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, date, epoch, step) seq2seq, input_vocab, output_vocab = get_model(checkpoint_path) beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4)) predictor = Predictor(beam_search, input_vocab, output_vocab) return predictor.predict_n(seq, n=n)
def setUpClass(self): test_path = os.path.dirname(os.path.realpath(__file__)) src = SourceField() trg = TargetField() dataset = torchtext.data.TabularDataset( path=os.path.join(test_path, 'data/eng-fra.txt'), format='tsv', fields=[('src', src), ('trg', trg)], ) src.build_vocab(dataset) trg.build_vocab(dataset) encoder = EncoderRNN(len(src.vocab), 10, 10, rnn_cell='lstm') decoder = DecoderRNN(len(trg.vocab), 10, 10, trg.sos_id, trg.eos_id, rnn_cell='lstm') seq2seq = Seq2seq(encoder, decoder) self.predictor = Predictor(seq2seq, src.vocab, trg.vocab)
class TestPredictor(unittest.TestCase): @classmethod def setUpClass(self): test_path = os.path.dirname(os.path.realpath(__file__)) src = SourceField() trg = TargetField() dataset = torchtext.data.TabularDataset( path=os.path.join(test_path, 'data/eng-fra.txt'), format='tsv', fields=[('src', src), ('trg', trg)], ) src.build_vocab(dataset) trg.build_vocab(dataset) encoder = EncoderRNN(len(src.vocab), 10, 10, rnn_cell='lstm') decoder = DecoderRNN(len(trg.vocab), 10, 10, trg.sos_id, trg.eos_id, rnn_cell='lstm') seq2seq = Seq2seq(encoder, decoder) self.predictor = Predictor(seq2seq, src.vocab, trg.vocab) def test_predict(self): src_seq = "I am fat" tgt_seq = self.predictor.predict(src_seq.split(' ')) for tok in tgt_seq: self.assertTrue(tok in self.predictor.tgt_vocab.stoi)
def evaluate_model(model, data, src_field, tgt_field, file_props={}): predictor = Predictor(model, src_field.vocab, tgt_field.vocab) data["pred_lemma"] = [ "".join(predictor.predict(list(e.word))[:-1]) for e in data.itertuples() ] acc = 0 for word in data.itertuples(): acc += int(word.pred_lemma == word.lemma) acc /= len(data.lemma) EXPERIMENT.metric("Dev accuracy", acc) data.to_csv("./dev_{}.csv".format("-".join("{}={}".format(k, v) for k, v in file_props))) EXPERIMENT.log("Incorrect predictions") EXPERIMENT.log( str(data[data["lemma"] != data["pred_lemma"]][[ "word", "lemma", "pred_lemma" ]]))
class eval_tool: def __init__(self, ckpt_path='./Res/PretrainModel/2019_12_27_08_48_21/'): checkpoint = Checkpoint.load(ckpt_path) self.seq2seq = checkpoint.model self.input_vocab = checkpoint.input_vocab self.output_vocab = checkpoint.output_vocab self.predictor = Predictor(self.seq2seq, self.input_vocab, self.output_vocab) def predict(self, input_str): return self.predictor.predict(input_str.strip().split())
def test(expt_dir, checkpoint, test_file, output_file): if checkpoint is not None: checkpoint_path = os.path.join(expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, checkpoint) logging.info("loading checkpoint from {}".format(checkpoint_path)) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab else: raise Exception("checkpoint path does not exist") predictor = Predictor(seq2seq, input_vocab, output_vocab) output = open(output_file, 'ab') with open(test_file) as f: for line_ in f: line = line_.strip().split('<s>') if len(line) != 0: question = basic_tokenizer(line[-2]) answer = predictor.predict(question)[:-1] output.write(''.join(answer) + '\n')
def test(opt, test_path): if opt.load_checkpoint is not None: # load model logging.info("loading check point from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab # Prepare predictor predictor = Predictor(seq2seq, input_vocab, output_vocab) with open(test_path, 'r') as reader, open(test_path + '_pred', 'w') as writer: for line in reader: source = treebank_tokenizer(line.split("\t")[0]) writer.write(generate(source, predictor) + '\n')
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=40, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) predictor = Predictor(seq2seq, input_vocab, output_vocab) #while True: #seq_str = raw_input("Type in a source sequence:") #seq = seq_str.strip().split() #print(predictor.predict(seq)) with open(opt.test_path) as f: content = f.readlines() content = [x.strip() for x in content] output = "" for row in content: seq_in = row.split("\t")[0] seq_out = row.split("\t")[1] seq_pred = predictor.predict(seq_in.strip().split()) seq_pred = " ".join(seq_pred[:-1])
best_model_dir=opt.best_model_dir, batch_size=opt.batch_size, checkpoint_every=opt.checkpoint_every, print_every=opt.print_every, max_epochs=opt.max_epochs, max_steps=opt.max_steps, max_checkpoints_num=opt.max_checkpoints_num, best_ppl=opt.best_ppl, device=device, multi_gpu=multi_gpu, logger=logger) seq2seq = t.train(seq2seq, data=train, start_step=opt.skip_steps, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=opt.teacher_forcing_ratio) elif opt.phase == "infer": # Predict predictor = Predictor(seq2seq, src_vocab.word2idx, tgt_vocab.idx2word, device) while True: seq_str = input("Type in a source sequence:") seq = seq_str.strip().split() ans = predictor.predict_n(seq, n=opt.beam_width) \ if opt.beam_width > 1 else predictor.predict(seq) print(ans)
# train t = SupervisedTrainer( loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir, ) seq2seq = t.train( seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume, ) evaluator = Evaluator(loss=loss, batch_size=32) dev_loss, accuracy = evaluator.evaluate(seq2seq, dev) assert dev_loss < 1.5 beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 3)) predictor = Predictor(beam_search, input_vocab, output_vocab) inp_seq = "1 3 5 7 9" seq = predictor.predict(inp_seq.split()) assert " ".join(seq[:-1]) == inp_seq[::-1]
if par_in_list == '(': word_list.append(')') elif par_in_list == '[': word_list.append(']') elif par_in_list == '{': word_list.append('}') word_list = [word for word in word_list if word != ''] return ' '.join(word_list) # In[22]: predictor = Predictor(seq2seq_model, input_vocab, output_vocab) num_samples = 0 perfect_samples = 0 dfa_perfect_samples = 0 match = 0 total = 0 model_correct = 0 model_wrong = 0 with torch.no_grad(): for batch in batch_iterator: num_samples = num_samples + 1
# Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=10, checkpoint_every=10, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=1, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) if __name__ == '__main__': predictor = Predictor(seq2seq, input_vocab, output_vocab) # while True: # seq_str = raw_input("Type in a source sequence:") # seq = seq_str.strip().split() # print(predictor.predict(seq))
def train(): src = SourceField(sequential=True, tokenize=lambda x: [i for i in jieba.lcut(x)]) tgt = TargetField(sequential=True, tokenize=lambda x: [i for i in jieba.lcut(x)]) max_len = 50 def len_filter(example): return len(example.src) <= max_len and len(example.tgt) <= max_len train = torchtext.data.TabularDataset(path=opt.train_path, format='csv', fields=[('src', src), ('tgt', tgt)], filter_pred=len_filter) dev = torchtext.data.TabularDataset(path=opt.dev_path, format='csv', fields=[('src', src), ('tgt', tgt)], filter_pred=len_filter) src.build_vocab(train, max_size=50000) tgt.build_vocab(train, max_size=50000) input_vocab = src.vocab output_vocab = tgt.vocab # NOTE: If the source field name and the target field name # are different from 'src' and 'tgt' respectively, they have # to be set explicitly before any training or inference # seq2seq.src_field_name = 'src' # seq2seq.tgt_field_name = 'tgt' # Prepare loss weight = torch.ones(len(tgt.vocab)) pad = tgt.vocab.stoi[tgt.pad_token] loss = Perplexity(weight, pad) if torch.cuda.is_available(): loss.cuda() seq2seq = None optimizer = None if not opt.resume: # Initialize model hidden_size = 128 bidirectional = True encoder = EncoderRNN(len(src.vocab), max_len, hidden_size, bidirectional=bidirectional, variable_lengths=True) decoder = DecoderRNN(len(tgt.vocab), max_len, hidden_size * 2 if bidirectional else hidden_size, dropout_p=0.2, use_attention=True, bidirectional=bidirectional, eos_id=tgt.eos_id, sos_id=tgt.sos_id) seq2seq = Seq2seq(encoder, decoder) if torch.cuda.is_available(): seq2seq.cuda() for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) predictor = Predictor(seq2seq, input_vocab, output_vocab)
# # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=batch_size, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=num_epochs, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) predictor = Predictor(seq2seq, input_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq_1 = [first_field.SYM_SOS ] + seq_str.strip().split() + [first_field.SYM_EOS] seq_str = raw_input("Type in a source sequence:") seq_2 = [first_field.SYM_SOS ] + seq_str.strip().split() + [first_field.SYM_EOS] print(predictor.predict([seq_1, seq_2]))
parser = argparse.ArgumentParser() parser.add_argument('--input', action='store', dest='input', help='Input file', default='INPUT_FILE') opt = parser.parse_args() checkpoint_path = os.path.join(ROOT, "experiment/checkpoints/72") checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab predictor = Predictor(seq2seq, input_vocab, output_vocab) data = pd.read_csv(ROOT + INPUT_FILE, delimiter=",", header=0) pred = [] def sentence_gen(sen): #sen[max_len,batch_size] a = [] for i in range(len(sen)): phrase = "" for j in range(len(sen[i])): if sen[i][j] != "<eos>": #print("printing word :",sen[i][j]) if sen[i][j] == "." or sen[i][j] == ",":
class AttentionVisualizer(object): """Object for visualizing the attention pattern of a given prediction. Args: task_path (str): name of the checkpoint file. figsizeh (tuple, optional): (width, height) of the final matplotlib figure. decimals (int, optional): number of decimals to whoe when pritning any number. is_show_attn_split (bool, optional): whether to show the the content and positional attention if there is one in addition to the full attention. is_show_evaluation (bool, optional): whether to show the evaluation metric if the target is given. output_length_key, attention_key, content_attn_key (str, optional): keys of the respective values in the the dictionary returned by the prediction. positional_table_labels (dictionary, optional): mapping from the keys in the return dictionary (the values) to the name the name of it should be shown as in the figure (the keys). The order is the one that will be used to plot the table (in python > 3.6). is_show_name (bool, optional): whether to show the name of the mdoel as the title of the figure. max_src, max_out, max_tgt (int, optional): maximum number of token to show for the source, the output and the target. Used in order not to clotter too much the plots. kwargs: Additional arguments to `MetricComputer`. """ def __init__( self, task_path, figsize=(15, 13), decimals=2, is_show_attn_split=True, is_show_evaluation=True, output_length_key='length', attention_key="attention_score", position_attn_key='position_attention', content_attn_key='content_attention', positional_table_labels={ "λ%": "position_percentage", "C.γ": "content_confidence", #"lgt": "approx_max_logit", "C.λ": "pos_confidence", "μ": "mu", "σ": "sigma", "w_α": "mean_attn_old_weight", "w_j/n": "rel_counter_decoder_weight", "w_1/n": "single_step_weight", "w_μ": "mu_old_weight", "w_γ": "mean_content_old_weight", "w_1": "bias_weight" }, # "% carry": "carry_rates", is_show_name=True, max_src=17, max_out=13, max_tgt=13, **kwargs): check = Checkpoint.load(task_path) self.model = check.model # store some interesting variables self.model.set_dev_mode() self.predictor = Predictor(self.model, check.input_vocab, check.output_vocab) self.model_name = task_path.split("/")[-2] self.figsize = figsize self.decimals = decimals self.is_show_attn_split = is_show_attn_split self.is_show_evaluation = is_show_evaluation self.positional_table_labels = positional_table_labels self.is_show_name = is_show_name self.max_src = max_src self.max_out = max_out self.max_tgt = max_tgt self.output_length_key = output_length_key self.attention_key = attention_key self.position_attn_key = position_attn_key self.content_attn_key = content_attn_key if self.is_show_evaluation: self.is_symbol_rewriting = "symbol rewriting" in task_path.lower() self.metric_computer = MetricComputer( check, is_symbol_rewriting=self.is_symbol_rewriting, **kwargs) if self.model.decoder.is_attention is None: raise AttentionException("Model is not using attention.") def __call__(self, src_str, tgt_str=None): """Plots the attention for the current example. Args: src_str (str): source of the example. tgt_str (str, optional): (width, height) target of the example, must be given in order to show the final metric. Returns: fig (plt.Figure): plotted attention figure. """ out_words, other = self.predictor.predict(src_str.split()) full_src_str = src_str full_out_str = " ".join(out_words) full_tgt_str = tgt_str additional, additional_text = self._format_additional(other) additional, src_words, out_words, tgt_str = self._subset( additional, src_str.split(), out_words, tgt_str) if self.is_show_name: title = "" else: title = None if tgt_str is not None: if self.is_show_name: title += "\n tgt_str: {} - ".format(tgt_str) else: title = "tgt_str: {} - ".format(tgt_str) if self.metric_computer.is_predict_eos: is_output_good_length = (len(full_out_str.split()) != len( full_tgt_str.split())) if self.is_symbol_rewriting and is_output_good_length: warnings.warn( "Cannot currently show the metric for symbol rewriting if output is not the right length." ) else: metrics = self.metric_computer(full_src_str, full_out_str, full_tgt_str) for name, val in metrics.items(): title += "{}: {:.2g} ".format(name, val) else: warnings.warn( "Cannot currently show the metric in the attention plots when `is_predict_eos=False`" ) if self.attention_key not in additional: raise ValueError( "`{}` not returned by predictor. Make sure the model uses attention." .format(self.attention_key)) attention = additional[self.attention_key] if self.position_attn_key in additional: filtered_pos_table_labels = { k: v for k, v in self.positional_table_labels.items() if v in additional } table_values = np.stack([ np.around(additional[name], decimals=self.decimals) for name in filtered_pos_table_labels.values() ]).T if self.is_show_attn_split and (self.position_attn_key in additional and self.content_attn_key in additional): content_attention = additional.get(self.content_attn_key) positional_attention = additional.get(self.position_attn_key) fig, axs = plt.subplots(2, 2, figsize=self.figsize) _plot_attention(src_words, out_words, attention, axs[0, 0], is_colorbar=False, title="Final Attention") _plot_table(table_values, list(filtered_pos_table_labels.keys()), axs[0, 1]) _plot_attention(src_words, out_words, content_attention, axs[1, 0], title="Content Attention") _plot_attention(src_words, out_words, positional_attention, axs[1, 1], title="Positional Attention") elif self.position_attn_key in additional: fig, axs = plt.subplots(1, 2, figsize=self.figsize) _plot_attention(src_words, out_words, attention, axs[0], title="Final Attention") _plot_table(table_values, list(filtered_pos_table_labels.keys()), axs[1]) else: fig, ax = plt.subplots(1, 1, figsize=self.figsize) _plot_attention(src_words, out_words, attention, ax, title="Final Attention") fig.text(0.5, 0.02, ' | '.join(additional_text), ha='center', va='center', size=13) if title is not None: plt.suptitle(title, size=13, weight="bold") fig.tight_layout() fig.subplots_adjust(bottom=0.07, top=0.83) return fig def _format_additional(self, additional): """Format the additinal dictionary returned by the predictor.""" def _format_carry_rates(carry_rates): if carry_rates is None: return "Carry % : None" mean_carry_rates = np.around(carry_rates.mean().item(), decimals=self.decimals) median_carry_rates = np.around(carry_rates.median().item(), decimals=self.decimals) return "Carry % : mean: {}; median: {}".format( mean_carry_rates, median_carry_rates) def _format_bb_gates(gates): if gates is None: return "BB Weight Mean Gates : None" mean_gates = np.around(gates.mean(0), decimals=self.decimals) return "BB Weight Mean Gates : {}".format(mean_gates) def _format_mu_weights(mu_weights): if mu_weights is not None: building_blocks_labels = self.model.decoder.position_attention.bb_labels for i, label in enumerate(building_blocks_labels): output[label + "_weight"] = mu_weights[:, i] output = dict() additional.pop( "visualize", None) # this is only for training visualization not predict additional.pop("losses", None) additional_text = [] additional = flatten_dict(additional) output = dict() output[self.output_length_key] = additional.pop( self.output_length_key)[0] for k, v in additional.items(): tensor = v if isinstance(v, torch.Tensor) else torch.cat(v) output[k] = tensor.detach().cpu().numpy().squeeze( )[:output[self.output_length_key]] carry_txt = _format_carry_rates(additional.pop("carry_rates", None)) bb_gates_txt = _format_bb_gates(output.pop("bb_gates", None)) additional_text.append(carry_txt) additional_text.append(bb_gates_txt) _format_mu_weights(output.pop("mu_weights", None)) return output, additional_text def _subset(self, additional, src_words, out_words, tgt_str=None): """Subsets the objects in the additional dictionary in order not to clotter the visualization. """ n_src = len(src_words) n_out = len(out_words) if n_out > self.max_out: subset_out = self.max_out // 2 out_words = out_words[:subset_out] + out_words[-subset_out:] for k, v in additional.items(): if isinstance(v, np.ndarray): additional[k] = np.concatenate( (v[:subset_out], v[-subset_out:]), axis=0) if n_src > self.max_src: subset_src = self.max_src // 2 src_words = src_words[:subset_src] + src_words[-subset_src:] for k, v in additional.items(): if isinstance(v, np.ndarray) and v.ndim == 2: additional[k] = np.concatenate( (v[:, :subset_src], v[:, -subset_src:]), axis=1) if tgt_str is not None: tgt_words = tgt_str.split() n_tgt = len(tgt_words) if n_tgt > self.max_tgt: subset_target = self.max_tgt // 2 tgt_str = " ".join(tgt_words[:subset_target] + ["..."] + tgt_words[-subset_target:]) return additional, src_words, out_words, tgt_str
parser.add_argument('--log-level', dest='log_level', default='info', help='Logging level.') opt = parser.parse_args() LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=getattr(logging, opt.log_level.upper())) logging.info(opt) if opt.load_checkpoint is not None: logging.info("loading checkpoint from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab predictor = Predictor(seq2seq, input_vocab, output_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
max_grad_norm=args.max_grad_norm) ############### train model ################ t = SpkTrainer(args=args, loss=loss, batch_size=args.batch_size, checkpoint_every=args.ckpt_every, random_seed=args.seed, print_every=args.verbose, expt_dir=args.expt_dir) discrim = t.train(model=model, data=train, num_epochs=args.epochs, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=args.resume) ################ initialize predictor ################### predictor = Predictor(model=model, src_vocab=input_vocab, tgt_vocab=output_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
dropout_p=0.2, use_attention=True, bidirectional=bidirectional, rnn_cell='lstm', eos_id=tgt.eos_id, sos_id=tgt.sos_id) seq2seq = Seq2seq(encoder, decoder) if torch.cuda.is_available(): seq2seq.cuda() for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # train t = SupervisedTrainer(loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) evaluator = Evaluator(loss=loss, batch_size=32) dev_loss, accuracy = evaluator.evaluate(seq2seq, dev) assert dev_loss < 1.5 predictor = Predictor(seq2seq, input_vocab, output_vocab) inp_seq = "1 3 5 7 9" seq = predictor.predict(inp_seq.split()) assert " ".join(seq[:-1]) == inp_seq[::-1]
for param in seq2tree.parameters(): param.data.uniform_(-0.08, 0.08) # encoder.embedding.weight.data.set_(input_vocab.vectors) # encoder.embedding.weight.data.set_(output_vocab.vectors) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) optimizer = Optimizer(optim.Adam(seq2tree.parameters(), lr=1e-4), max_grad_norm=5) # train t = SupervisedTrainer(loss=loss, batch_size=1, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2tree = t.train(seq2tree, train, num_epochs=20, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) predictor = Predictor(seq2tree, input_vocab, input_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
# TODO add dev eval here for early stopping if config['train model']: seq2seq = t.train(input_vocab, feats_vocab, seq2seq, train, num_epochs=config['epochs'], vectors=vectors, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) # pdb.set_trace() predictor = Predictor(seq2seq, input_vocab, feats_vocab, output_vocab, vectors) # seq2top = Seq2seq(seq2seq.encoder, ) # topk_predictor = Predictor(seq2top, input_vocab, output_vocab, vectors) if config['pull embeddings']: out_vecs = {} if config['feat embeddings']: feats = {} of = open(config['feat output'], 'wb') # TODO add option to save output src = SourceField() feat = SourceField() tgt = TargetField() # pdb.set_trace()
def sample( # train_source, # train_target, # dev_source, # dev_target, experiment_directory='/home/xweiwang/RL/seq2seq/experiment', checkpoint='2019_05_18_20_32_54', resume=True, log_level='info', ): """ # Sample usage TRAIN_SRC=data/toy_reverse/train/src.txt TRAIN_TGT=data/toy_reverse/train/tgt.txt DEV_SRC=data/toy_reverse/dev/src.txt DEV_TGT=data/toy_reverse/dev/tgt.txt ## Training ```shell $ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt $EXPT_PATH ``` ## Resuming from the latest checkpoint of the experiment ```shell $ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt $EXPT_PATH -r ``` ## Resuming from a specific checkpoint ```shell $ python examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt $EXPT_PATH -c $CHECKPOINT_DIR ``` """ logging.basicConfig( format=LOG_FORMAT, level=getattr(logging, log_level.upper()), ) # logging.info('train_source: %s', train_source) # logging.info('train_target: %s', train_target) # logging.info('dev_source: %s', dev_source) # logging.info('dev_target: %s', dev_target) logging.info('experiment_directory: %s', experiment_directory) logging.info('checkpoint: %s', checkpoint) # if checkpoint: seq2seq, input_vocab, output_vocab = load_checkpoint( experiment_directory, checkpoint) # else: # seq2seq, input_vocab, output_vocab = train_model( # train_source, # train_target, # dev_source, # dev_target, # experiment_directory, # resume=resume, # ) predictor = Predictor(seq2seq, input_vocab, output_vocab) while True: seq_str = input('Type in a source sequence: ') seq = seq_str.strip().split() print(predictor.predict(seq))
def run_training(opt, default_data_dir, num_epochs=100): if opt.load_checkpoint is not None: logging.info("loading checkpoint from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab else: # Prepare dataset src = SourceField() tgt = TargetField() max_len = 50 data_file = os.path.join(default_data_dir, opt.train_path, 'data.txt') logging.info("Starting new Training session on %s", data_file) def len_filter(example): return (len(example.src) <= max_len) and (len(example.tgt) <= max_len) \ and (len(example.src) > 0) and (len(example.tgt) > 0) train = torchtext.data.TabularDataset( path=data_file, format='json', fields={'src': ('src', src), 'tgt': ('tgt', tgt)}, filter_pred=len_filter ) dev = None if opt.no_dev is False: dev_data_file = os.path.join(default_data_dir, opt.train_path, 'dev-data.txt') dev = torchtext.data.TabularDataset( path=dev_data_file, format='json', fields={'src': ('src', src), 'tgt': ('tgt', tgt)}, filter_pred=len_filter ) src.build_vocab(train, max_size=50000) tgt.build_vocab(train, max_size=50000) input_vocab = src.vocab output_vocab = tgt.vocab # NOTE: If the source field name and the target field name # are different from 'src' and 'tgt' respectively, they have # to be set explicitly before any training or inference # seq2seq.src_field_name = 'src' # seq2seq.tgt_field_name = 'tgt' # Prepare loss weight = torch.ones(len(tgt.vocab)) pad = tgt.vocab.stoi[tgt.pad_token] loss = Perplexity(weight, pad) if torch.cuda.is_available(): logging.info("Yayyy We got CUDA!!!") loss.cuda() else: logging.info("No cuda available device found running on cpu") seq2seq = None optimizer = None if not opt.resume: hidden_size = 128 decoder_hidden_size = hidden_size * 2 logging.info("EncoderRNN Hidden Size: %s", hidden_size) logging.info("DecoderRNN Hidden Size: %s", decoder_hidden_size) bidirectional = True encoder = EncoderRNN(len(src.vocab), max_len, hidden_size, bidirectional=bidirectional, rnn_cell='lstm', variable_lengths=True) decoder = DecoderRNN(len(tgt.vocab), max_len, decoder_hidden_size, dropout_p=0, use_attention=True, bidirectional=bidirectional, rnn_cell='lstm', eos_id=tgt.eos_id, sos_id=tgt.sos_id) seq2seq = Seq2seq(encoder, decoder) if torch.cuda.is_available(): seq2seq.cuda() for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) scheduler = StepLR(optimizer.optimizer, 1) optimizer.set_scheduler(scheduler) # train num_epochs = num_epochs batch_size = 32 checkpoint_every = num_epochs / 10 print_every = num_epochs / 100 properties = dict(batch_size=batch_size, checkpoint_every=checkpoint_every, print_every=print_every, expt_dir=opt.expt_dir, num_epochs=num_epochs, teacher_forcing_ratio=0.5, resume=opt.resume) logging.info("Starting training with the following Properties %s", json.dumps(properties, indent=2)) t = SupervisedTrainer(loss=loss, batch_size=num_epochs, checkpoint_every=checkpoint_every, print_every=print_every, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=num_epochs, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) evaluator = Evaluator(loss=loss, batch_size=batch_size) if opt.no_dev is False: dev_loss, accuracy = evaluator.evaluate(seq2seq, dev) logging.info("Dev Loss: %s", dev_loss) logging.info("Accuracy: %s", dev_loss) beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4)) predictor = Predictor(beam_search, input_vocab, output_vocab) while True: try: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() results = predictor.predict_n(seq, n=3) for i, res in enumerate(results): print('option %s: %s\n', i + 1, res) except KeyboardInterrupt: logging.info("Bye Bye") exit(0)
def main(): '''Main Function''' parser = argparse.ArgumentParser(description='sum_file.py') parser.add_argument('-model', required=True, help='Path to model .pt file') parser.add_argument('-src', required=True, help='Source sequence to decode (one line per sequence)') parser.add_argument('-vocab', required=True, help='Source sequence to decode (one line per sequence)') parser.add_argument('-output', default='pred.txt', help="""Path to output the predictions (each line will be the decoded sequence""") parser.add_argument('-beam_size', type=int, default=5, help='Beam size') parser.add_argument('-batch_size', type=int, default=30, help='Batch size') parser.add_argument('-n_best', type=int, default=1, help="""If verbose is set, will output the n_best decoded sentences""") parser.add_argument('-no_cuda', action='store_true') opt = parser.parse_args() opt.cuda = not opt.no_cuda # Prepare DataLoader preprocess_data = torch.load(opt.vocab) preprocess_settings = preprocess_data['settings'] test_src_word_insts = read_instances_from_file( opt.src, preprocess_settings.max_word_seq_len, preprocess_settings.keep_case, preprocess_settings.mode) test_src_insts = convert_instance_to_idx_seq( test_src_word_insts, preprocess_data['dict']['src']) # prepare model device = torch.device('cuda' if opt.cuda else 'cpu') checkpoint = torch.load(opt.model) model_opt = checkpoint['settings'] model_opt.bidirectional = True encoder = EncoderRNN(model_opt.src_vocab_size, model_opt.max_token_seq_len, model_opt.d_model, bidirectional=model_opt.bidirectional, variable_lengths=True) decoder = DecoderRNN(model_opt.tgt_vocab_size, model_opt.max_token_seq_len, model_opt.d_model * 2 if model_opt.bidirectional else model_opt.d_model, n_layers=model_opt.n_layer, dropout_p=model_opt.dropout, use_attention=True, bidirectional=model_opt.bidirectional, eos_id=Constants.BOS, sos_id=Constants.EOS) model = Seq2seq(encoder, decoder).to(device) model = nn.DataParallel(model) # using Dataparallel because training used model.load_state_dict(checkpoint['model']) print('[Info] Trained model state loaded.') predictor = Predictor(model, preprocess_data['dict']['tgt']) with open(opt.output, 'w') as f: for src_seq in tqdm(test_src_insts, mininterval=2, desc=' - (Test)', leave=False): pred_line = ' '.join(predictor.predict(src_seq)) f.write(pred_line + '\n') print('[Info] Finished.')
def __init__( self, task_path, figsize=(15, 13), decimals=2, is_show_attn_split=True, is_show_evaluation=True, output_length_key='length', attention_key="attention_score", position_attn_key='position_attention', content_attn_key='content_attention', positional_table_labels={ "λ%": "position_percentage", "C.γ": "content_confidence", #"lgt": "approx_max_logit", "C.λ": "pos_confidence", "μ": "mu", "σ": "sigma", "w_α": "mean_attn_old_weight", "w_j/n": "rel_counter_decoder_weight", "w_1/n": "single_step_weight", "w_μ": "mu_old_weight", "w_γ": "mean_content_old_weight", "w_1": "bias_weight" }, # "% carry": "carry_rates", is_show_name=True, max_src=17, max_out=13, max_tgt=13, **kwargs): check = Checkpoint.load(task_path) self.model = check.model # store some interesting variables self.model.set_dev_mode() self.predictor = Predictor(self.model, check.input_vocab, check.output_vocab) self.model_name = task_path.split("/")[-2] self.figsize = figsize self.decimals = decimals self.is_show_attn_split = is_show_attn_split self.is_show_evaluation = is_show_evaluation self.positional_table_labels = positional_table_labels self.is_show_name = is_show_name self.max_src = max_src self.max_out = max_out self.max_tgt = max_tgt self.output_length_key = output_length_key self.attention_key = attention_key self.position_attn_key = position_attn_key self.content_attn_key = content_attn_key if self.is_show_evaluation: self.is_symbol_rewriting = "symbol rewriting" in task_path.lower() self.metric_computer = MetricComputer( check, is_symbol_rewriting=self.is_symbol_rewriting, **kwargs) if self.model.decoder.is_attention is None: raise AttentionException("Model is not using attention.")
for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) topk_decoder = seq2seq.decoder topk_decoder = TopKDecoder(decoder, 10, False , True ,1.0) seq2seq = Seq2seq(encoder, topk_decoder) predictor = Predictor(seq2seq, input_vocab, output_vocab, False,1,float('inf')) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))