def __init__(self, ckpt_path='./Res/PretrainModel/2019_12_27_08_48_21/'): checkpoint = Checkpoint.load(ckpt_path) self.seq2seq = checkpoint.model self.input_vocab = checkpoint.input_vocab self.output_vocab = checkpoint.output_vocab self.predictor = Predictor(self.seq2seq, self.input_vocab, self.output_vocab)
def predict_with_checkpoint(checkpoint_path, sequence, hierarchial = False, remote = None, word_vectors = None): checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = None, None if word_vectors != None: input_vects = Word2Vectors(input_vocab, word_vectors, word_vectors.dim_size) output_vects = Word2Vectors(output_vocab, word_vectors, word_vectors.dim_size) seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = input_vects, output_vects seq2seq.decoder = TopKDecoder(seq2seq.decoder, 5) if not hierarchial: predictor = Predictor(seq2seq, input_vocab, output_vocab) seq = sequence.strip().split() else: predictor = HierarchialPredictor(seq2seq, input_vocab, output_vocab) seq = ['|'.join(x.split()) for x in sequence] return ' '.join(predictor.predict(seq))
def predict(expt_dir, seq_str, date, epoch, step, n=3): seq = seq_str.strip().split() checkpoint_path = os.path.join(expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, date, epoch, step) seq2seq, input_vocab, output_vocab = get_model(checkpoint_path) beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4)) predictor = Predictor(beam_search, input_vocab, output_vocab) return predictor.predict_n(seq, n=n)
def test_predict(self): predictor = Predictor(self.seq2seq, self.dataset.input_vocab, self.dataset.output_vocab) src_seq = ["I", "am", "fat"] tgt_seq = predictor.predict(src_seq) for tok in tgt_seq: self.assertTrue(tok in self.dataset.output_vocab._token2index)
def dev_predict(task_path, src_str, is_plot=True): """Helper used to visualize and understand why and what the model predicts. Args: task_path (str): path to the saved task directory containing, amongst other, the model. src_str (str): source sentence that will be used to predict. is_plot (bool, optional): whether to plots the attention pattern. Returns: out_words (list): decoder predictions. other (dictionary): additional information used for predictions. test (dictionary): additional information that is only stored in dev mode. These can include temporary variables that do not have to be stored in `other` but that can still be interesting to inspect. """ check = Checkpoint.load(task_path) check.model.set_dev_mode() predictor = Predictor(check.model, check.input_vocab, check.output_vocab) out_words, other = predictor.predict(src_str.split()) test = dict() for k, v in other["test"].items(): tensor = v if isinstance(v, torch.Tensor) else torch.cat(v) test[k] = tensor.detach().cpu().numpy().squeeze()[:other["length"][0]] # except: # for using "step" # test[k] = v if is_plot: visualizer = AttentionVisualizer(task_path) visualizer(src_str) return out_words, other, test
def setUpClass(self): test_path = os.path.dirname(os.path.realpath(__file__)) src = SourceField() trg = TargetField() dataset = torchtext.data.TabularDataset( path=os.path.join(test_path, 'data/eng-fra.txt'), format='tsv', fields=[('src', src), ('trg', trg)], ) src.build_vocab(dataset) trg.build_vocab(dataset) encoder = EncoderRNN(len(src.vocab), 10, 10, rnn_cell='lstm') decoder = DecoderRNN(len(trg.vocab), 10, 10, trg.sos_id, trg.eos_id, rnn_cell='lstm') seq2seq = Seq2seq(encoder, decoder) self.predictor = Predictor(seq2seq, src.vocab, trg.vocab)
def evaluate_model(model, data, src_field, tgt_field, file_props={}): predictor = Predictor(model, src_field.vocab, tgt_field.vocab) data["pred_lemma"] = [ "".join(predictor.predict(list(e.word))[:-1]) for e in data.itertuples() ] acc = 0 for word in data.itertuples(): acc += int(word.pred_lemma == word.lemma) acc /= len(data.lemma) EXPERIMENT.metric("Dev accuracy", acc) data.to_csv("./dev_{}.csv".format("-".join("{}={}".format(k, v) for k, v in file_props))) EXPERIMENT.log("Incorrect predictions") EXPERIMENT.log( str(data[data["lemma"] != data["pred_lemma"]][[ "word", "lemma", "pred_lemma" ]]))
def test(opt, test_path): if opt.load_checkpoint is not None: # load model logging.info("loading check point from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab # Prepare predictor predictor = Predictor(seq2seq, input_vocab, output_vocab) with open(test_path, 'r') as reader, open(test_path + '_pred', 'w') as writer: for line in reader: source = treebank_tokenizer(line.split("\t")[0]) writer.write(generate(source, predictor) + '\n')
def test(expt_dir, checkpoint, test_file, output_file): if checkpoint is not None: checkpoint_path = os.path.join(expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, checkpoint) logging.info("loading checkpoint from {}".format(checkpoint_path)) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab else: raise Exception("checkpoint path does not exist") predictor = Predictor(seq2seq, input_vocab, output_vocab) output = open(output_file, 'ab') with open(test_file) as f: for line_ in f: line = line_.strip().split('<s>') if len(line) != 0: question = basic_tokenizer(line[-2]) answer = predictor.predict(question)[:-1] output.write(''.join(answer) + '\n')
def train(): src = SourceField(sequential=True, tokenize=lambda x: [i for i in jieba.lcut(x)]) tgt = TargetField(sequential=True, tokenize=lambda x: [i for i in jieba.lcut(x)]) max_len = 50 def len_filter(example): return len(example.src) <= max_len and len(example.tgt) <= max_len train = torchtext.data.TabularDataset(path=opt.train_path, format='csv', fields=[('src', src), ('tgt', tgt)], filter_pred=len_filter) dev = torchtext.data.TabularDataset(path=opt.dev_path, format='csv', fields=[('src', src), ('tgt', tgt)], filter_pred=len_filter) src.build_vocab(train, max_size=50000) tgt.build_vocab(train, max_size=50000) input_vocab = src.vocab output_vocab = tgt.vocab # NOTE: If the source field name and the target field name # are different from 'src' and 'tgt' respectively, they have # to be set explicitly before any training or inference # seq2seq.src_field_name = 'src' # seq2seq.tgt_field_name = 'tgt' # Prepare loss weight = torch.ones(len(tgt.vocab)) pad = tgt.vocab.stoi[tgt.pad_token] loss = Perplexity(weight, pad) if torch.cuda.is_available(): loss.cuda() seq2seq = None optimizer = None if not opt.resume: # Initialize model hidden_size = 128 bidirectional = True encoder = EncoderRNN(len(src.vocab), max_len, hidden_size, bidirectional=bidirectional, variable_lengths=True) decoder = DecoderRNN(len(tgt.vocab), max_len, hidden_size * 2 if bidirectional else hidden_size, dropout_p=0.2, use_attention=True, bidirectional=bidirectional, eos_id=tgt.eos_id, sos_id=tgt.sos_id) seq2seq = Seq2seq(encoder, decoder) if torch.cuda.is_available(): seq2seq.cuda() for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) predictor = Predictor(seq2seq, input_vocab, output_vocab)
for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) topk_decoder = seq2seq.decoder topk_decoder = TopKDecoder(decoder, 10, False , True ,1.0) seq2seq = Seq2seq(encoder, topk_decoder) predictor = Predictor(seq2seq, input_vocab, output_vocab, False,1,float('inf')) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
def main(): '''Main Function''' parser = argparse.ArgumentParser(description='sum_file.py') parser.add_argument('-model', required=True, help='Path to model .pt file') parser.add_argument('-src', required=True, help='Source sequence to decode (one line per sequence)') parser.add_argument('-vocab', required=True, help='Source sequence to decode (one line per sequence)') parser.add_argument('-output', default='pred.txt', help="""Path to output the predictions (each line will be the decoded sequence""") parser.add_argument('-beam_size', type=int, default=5, help='Beam size') parser.add_argument('-batch_size', type=int, default=30, help='Batch size') parser.add_argument('-n_best', type=int, default=1, help="""If verbose is set, will output the n_best decoded sentences""") parser.add_argument('-no_cuda', action='store_true') opt = parser.parse_args() opt.cuda = not opt.no_cuda # Prepare DataLoader preprocess_data = torch.load(opt.vocab) preprocess_settings = preprocess_data['settings'] test_src_word_insts = read_instances_from_file( opt.src, preprocess_settings.max_word_seq_len, preprocess_settings.keep_case, preprocess_settings.mode) test_src_insts = convert_instance_to_idx_seq( test_src_word_insts, preprocess_data['dict']['src']) # prepare model device = torch.device('cuda' if opt.cuda else 'cpu') checkpoint = torch.load(opt.model) model_opt = checkpoint['settings'] model_opt.bidirectional = True encoder = EncoderRNN(model_opt.src_vocab_size, model_opt.max_token_seq_len, model_opt.d_model, bidirectional=model_opt.bidirectional, variable_lengths=True) decoder = DecoderRNN(model_opt.tgt_vocab_size, model_opt.max_token_seq_len, model_opt.d_model * 2 if model_opt.bidirectional else model_opt.d_model, n_layers=model_opt.n_layer, dropout_p=model_opt.dropout, use_attention=True, bidirectional=model_opt.bidirectional, eos_id=Constants.BOS, sos_id=Constants.EOS) model = Seq2seq(encoder, decoder).to(device) model = nn.DataParallel(model) # using Dataparallel because training used model.load_state_dict(checkpoint['model']) print('[Info] Trained model state loaded.') predictor = Predictor(model, preprocess_data['dict']['tgt']) with open(opt.output, 'w') as f: for src_seq in tqdm(test_src_insts, mininterval=2, desc=' - (Test)', leave=False): pred_line = ' '.join(predictor.predict(src_seq)) f.write(pred_line + '\n') print('[Info] Finished.')
def sample( # train_source, # train_target, # dev_source, # dev_target, experiment_directory='/home/xweiwang/RL/seq2seq/experiment', checkpoint='2019_05_18_20_32_54', resume=True, log_level='info', ): """ # Sample usage TRAIN_SRC=data/toy_reverse/train/src.txt TRAIN_TGT=data/toy_reverse/train/tgt.txt DEV_SRC=data/toy_reverse/dev/src.txt DEV_TGT=data/toy_reverse/dev/tgt.txt ## Training ```shell $ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt $EXPT_PATH ``` ## Resuming from the latest checkpoint of the experiment ```shell $ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt $EXPT_PATH -r ``` ## Resuming from a specific checkpoint ```shell $ python examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt $EXPT_PATH -c $CHECKPOINT_DIR ``` """ logging.basicConfig( format=LOG_FORMAT, level=getattr(logging, log_level.upper()), ) # logging.info('train_source: %s', train_source) # logging.info('train_target: %s', train_target) # logging.info('dev_source: %s', dev_source) # logging.info('dev_target: %s', dev_target) logging.info('experiment_directory: %s', experiment_directory) logging.info('checkpoint: %s', checkpoint) # if checkpoint: seq2seq, input_vocab, output_vocab = load_checkpoint( experiment_directory, checkpoint) # else: # seq2seq, input_vocab, output_vocab = train_model( # train_source, # train_target, # dev_source, # dev_target, # experiment_directory, # resume=resume, # ) predictor = Predictor(seq2seq, input_vocab, output_vocab) while True: seq_str = input('Type in a source sequence: ') seq = seq_str.strip().split() print(predictor.predict(seq))
# scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) # train t = SupervisedTrainer(loss=loss, batch_size=16, checkpoint_every=50, print_every=1, expt_dir=opt.expt_dir) model = t.train(gnn, data_train, data_valid, num_epochs=30, num_antecedents=opt.num_antecedents, optimizer=optimizer, teacher_forcing_ratio=1, resume=opt.resume, save_file=opt.save_file, embed_file=opt.embed_file, log_dir=opt.log_dir) predictor = Predictor(model, input_vocab, output_vocab) """ while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq)) """ print("Training Finished.")
best_model_dir=opt.best_model_dir, batch_size=opt.batch_size, checkpoint_every=opt.checkpoint_every, print_every=opt.print_every, max_epochs=opt.max_epochs, max_steps=opt.max_steps, max_checkpoints_num=opt.max_checkpoints_num, best_ppl=opt.best_ppl, device=device, multi_gpu=multi_gpu, logger=logger) seq2seq = t.train(seq2seq, data=train, start_step=opt.skip_steps, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=opt.teacher_forcing_ratio) elif opt.phase == "infer": # Predict predictor = Predictor(seq2seq, src_vocab.word2idx, tgt_vocab.idx2word, device) while True: seq_str = input("Type in a source sequence:") seq = seq_str.strip().split() ans = predictor.predict_n(seq, n=opt.beam_width) \ if opt.beam_width > 1 else predictor.predict(seq) print(ans)
if par_in_list == '(': word_list.append(')') elif par_in_list == '[': word_list.append(']') elif par_in_list == '{': word_list.append('}') word_list = [word for word in word_list if word != ''] return ' '.join(word_list) # In[22]: predictor = Predictor(seq2seq_model, input_vocab, output_vocab) num_samples = 0 perfect_samples = 0 dfa_perfect_samples = 0 match = 0 total = 0 model_correct = 0 model_wrong = 0 with torch.no_grad(): for batch in batch_iterator: num_samples = num_samples + 1
def train(opt): LOG_FORMAT = '%(asctime)s %(levelname)-8s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=getattr(logging, opt.log_level.upper())) logging.info(opt) if int(opt.GPU) >= 0: torch.cuda.set_device(int(opt.GPU)) if opt.load_checkpoint is not None: logging.info("loading checkpoint from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2tree = checkpoint.model input_vocab = checkpoint.input_vocab else: # Prepare dataset src = SourceField() nt = NTField() pos = PosField() tgt_tree = TreeField() comp = CompField() max_len = opt.max_len def len_filter(example): return len(example.src) <= max_len train = torchtext.data.TabularDataset(path=opt.train_path, format='tsv', fields=[('src', src), ('nt', nt), ('pos', pos), ('tree', tgt_tree)], filter_pred=len_filter) dev = torchtext.data.TabularDataset(path=opt.dev_path, format='tsv', fields=[('src', src), ('nt', nt), ('pos', pos), ('tree', tgt_tree)], filter_pred=len_filter) src.build_vocab(train, max_size=50000) comp.build_vocab(train, max_size=50000) nt.build_vocab(train, max_size=50000) pos.build_vocab(train, max_size=50000) # src_tree.build_vocab(train, max_size=50000) pos_in_nt = set() for Pos in pos.vocab.stoi: if nt.vocab.stoi[Pos] > 1: pos_in_nt.add(nt.vocab.stoi[Pos]) hidden_size = opt.hidden_size input_vocab = src.vocab nt_vocab = nt.vocab def tree_to_id(tree): tree.set_label(nt_vocab.stoi[tree.label()]) if len(tree) == 1 and str(tree[0])[0] is not '(': tree[0] = input_vocab.stoi[tree[0]] return else: for subtree in tree: tree_to_id(subtree) tree.append(Tree(nt_vocab.stoi['<eos>'], [])) return tree # train.examples = [str(tree_to_id(ex.tree)) for ex in train.examples] # dev.examples = [str(tree_to_id(ex.tree)) for ex in dev.examples] for ex in train.examples: ex.tree = str(tree_to_id(Tree.fromstring(ex.tree))) for ex in dev.examples: ex.tree = str(tree_to_id(Tree.fromstring(ex.tree))) # train.examples = [tree_to_id(Tree.fromstring(ex.tree)) for ex in train.examples] # dev.examples = [str(tree_to_id(Tree.fromstring(ex.tree))) for ex in dev.examples] if opt.word_embedding is not None: input_vocab.load_vectors([opt.word_embedding]) loss = NLLLoss() if torch.cuda.is_available(): loss.cuda() loss.reset() seq2tree = None optimizer = None if not opt.resume: # Initialize model bidirectional = opt.bidirectional_encoder encoder = EncoderRNN(len(src.vocab), opt.word_embedding_size, max_len, hidden_size, bidirectional=bidirectional, variable_lengths=True) decoder = DecoderTree(len(src.vocab), opt.word_embedding_size, opt.nt_embedding_size, len(nt.vocab), max_len, hidden_size * 2 if bidirectional else hidden_size, sos_id=nt_vocab.stoi['<sos>'], eos_id=nt_vocab.stoi['<eos>'], dropout_p=0.2, use_attention=True, bidirectional=bidirectional, pos_in_nt=pos_in_nt) seq2tree = Seq2tree(encoder, decoder) if torch.cuda.is_available(): seq2tree.cuda() for param in seq2tree.parameters(): param.data.uniform_(-0.08, 0.08) # encoder.embedding.weight.data.set_(input_vocab.vectors) # encoder.embedding.weight.data.set_(output_vocab.vectors) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) optimizer = Optimizer(optim.Adam(seq2tree.parameters(), lr=opt.lr), max_grad_norm=5) # train t = SupervisedTrainer(loss=loss, batch_size=opt.batch_size, checkpoint_every=opt.checkpoint_every, print_every=10, expt_dir=opt.expt_dir, lr=opt.lr) seq2tree = t.train(seq2tree, train, num_epochs=opt.epoch, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0, resume=opt.resume) predictor = Predictor(seq2tree, input_vocab, nt_vocab) return predictor, dev, train
# TODO add dev eval here for early stopping if config['train model']: seq2seq = t.train(input_vocab, feats_vocab, seq2seq, train, num_epochs=config['epochs'], vectors=vectors, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) # pdb.set_trace() predictor = Predictor(seq2seq, input_vocab, feats_vocab, output_vocab, vectors) # seq2top = Seq2seq(seq2seq.encoder, ) # topk_predictor = Predictor(seq2top, input_vocab, output_vocab, vectors) if config['pull embeddings']: out_vecs = {} if config['feat embeddings']: feats = {} of = open(config['feat output'], 'wb') # TODO add option to save output src = SourceField() feat = SourceField() tgt = TargetField() # pdb.set_trace()
# train t = SupervisedTrainer( loss=loss, batch_size=32, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir, ) seq2seq = t.train( seq2seq, train, num_epochs=6, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume, ) evaluator = Evaluator(loss=loss, batch_size=32) dev_loss, accuracy = evaluator.evaluate(seq2seq, dev) assert dev_loss < 1.5 beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 3)) predictor = Predictor(beam_search, input_vocab, output_vocab) inp_seq = "1 3 5 7 9" seq = predictor.predict(inp_seq.split()) assert " ".join(seq[:-1]) == inp_seq[::-1]
def run_training(opt, default_data_dir, num_epochs=100): if opt.load_checkpoint is not None: logging.info("loading checkpoint from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab else: # Prepare dataset src = SourceField() tgt = TargetField() max_len = 50 data_file = os.path.join(default_data_dir, opt.train_path, 'data.txt') logging.info("Starting new Training session on %s", data_file) def len_filter(example): return (len(example.src) <= max_len) and (len(example.tgt) <= max_len) \ and (len(example.src) > 0) and (len(example.tgt) > 0) train = torchtext.data.TabularDataset( path=data_file, format='json', fields={'src': ('src', src), 'tgt': ('tgt', tgt)}, filter_pred=len_filter ) dev = None if opt.no_dev is False: dev_data_file = os.path.join(default_data_dir, opt.train_path, 'dev-data.txt') dev = torchtext.data.TabularDataset( path=dev_data_file, format='json', fields={'src': ('src', src), 'tgt': ('tgt', tgt)}, filter_pred=len_filter ) src.build_vocab(train, max_size=50000) tgt.build_vocab(train, max_size=50000) input_vocab = src.vocab output_vocab = tgt.vocab # NOTE: If the source field name and the target field name # are different from 'src' and 'tgt' respectively, they have # to be set explicitly before any training or inference # seq2seq.src_field_name = 'src' # seq2seq.tgt_field_name = 'tgt' # Prepare loss weight = torch.ones(len(tgt.vocab)) pad = tgt.vocab.stoi[tgt.pad_token] loss = Perplexity(weight, pad) if torch.cuda.is_available(): logging.info("Yayyy We got CUDA!!!") loss.cuda() else: logging.info("No cuda available device found running on cpu") seq2seq = None optimizer = None if not opt.resume: hidden_size = 128 decoder_hidden_size = hidden_size * 2 logging.info("EncoderRNN Hidden Size: %s", hidden_size) logging.info("DecoderRNN Hidden Size: %s", decoder_hidden_size) bidirectional = True encoder = EncoderRNN(len(src.vocab), max_len, hidden_size, bidirectional=bidirectional, rnn_cell='lstm', variable_lengths=True) decoder = DecoderRNN(len(tgt.vocab), max_len, decoder_hidden_size, dropout_p=0, use_attention=True, bidirectional=bidirectional, rnn_cell='lstm', eos_id=tgt.eos_id, sos_id=tgt.sos_id) seq2seq = Seq2seq(encoder, decoder) if torch.cuda.is_available(): seq2seq.cuda() for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) scheduler = StepLR(optimizer.optimizer, 1) optimizer.set_scheduler(scheduler) # train num_epochs = num_epochs batch_size = 32 checkpoint_every = num_epochs / 10 print_every = num_epochs / 100 properties = dict(batch_size=batch_size, checkpoint_every=checkpoint_every, print_every=print_every, expt_dir=opt.expt_dir, num_epochs=num_epochs, teacher_forcing_ratio=0.5, resume=opt.resume) logging.info("Starting training with the following Properties %s", json.dumps(properties, indent=2)) t = SupervisedTrainer(loss=loss, batch_size=num_epochs, checkpoint_every=checkpoint_every, print_every=print_every, expt_dir=opt.expt_dir) seq2seq = t.train(seq2seq, train, num_epochs=num_epochs, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) evaluator = Evaluator(loss=loss, batch_size=batch_size) if opt.no_dev is False: dev_loss, accuracy = evaluator.evaluate(seq2seq, dev) logging.info("Dev Loss: %s", dev_loss) logging.info("Accuracy: %s", dev_loss) beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4)) predictor = Predictor(beam_search, input_vocab, output_vocab) while True: try: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() results = predictor.predict_n(seq, n=3) for i, res in enumerate(results): print('option %s: %s\n', i + 1, res) except KeyboardInterrupt: logging.info("Bye Bye") exit(0)
max_grad_norm=args.max_grad_norm) ############### train model ################ t = SpkTrainer(args=args, loss=loss, batch_size=args.batch_size, checkpoint_every=args.ckpt_every, random_seed=args.seed, print_every=args.verbose, expt_dir=args.expt_dir) discrim = t.train(model=model, data=train, num_epochs=args.epochs, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=args.resume) ################ initialize predictor ################### predictor = Predictor(model=model, src_vocab=input_vocab, tgt_vocab=output_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
import requests import json from flask import Flask, request, Response app = Flask(__name__, static_url_path='') from seq2seq.evaluator import Predictor from seq2seq.util.checkpoint import Checkpoint cp = Checkpoint.load("./experiment/checkpoints/2019_08_02_13_23_47/") predictor = Predictor(cp.model, cp.input_vocab, cp.output_vocab) def q(s): l = -len('<eos>') return "".join(predictor.predict(list(s)))[:l] @app.route("/api") def get(): r = request.args.get('text', '') if r == "": return Response(json.dumps({'status':"error", 'message':"empty input"})) return Response(json.dumps({'status':"ok", 'message':q(r)}), mimetype="application/json") @app.route('/') def index(): return app.send_static_file('index.html')# if __name__ == "__main__": app.run(host='127.0.0.1', port=5002, debug=True)
parser.add_argument('--log-level', dest='log_level', default='info', help='Logging level.') opt = parser.parse_args() LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=getattr(logging, opt.log_level.upper())) logging.info(opt) if opt.load_checkpoint is not None: logging.info("loading checkpoint from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab predictor = Predictor(seq2seq, input_vocab, output_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
for param in seq2tree.parameters(): param.data.uniform_(-0.08, 0.08) # encoder.embedding.weight.data.set_(input_vocab.vectors) # encoder.embedding.weight.data.set_(output_vocab.vectors) # Optimizer and learning rate scheduler can be customized by # explicitly constructing the objects and pass to the trainer. # # optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5) # scheduler = StepLR(optimizer.optimizer, 1) # optimizer.set_scheduler(scheduler) optimizer = Optimizer(optim.Adam(seq2tree.parameters(), lr=1e-4), max_grad_norm=5) # train t = SupervisedTrainer(loss=loss, batch_size=1, checkpoint_every=50, print_every=10, expt_dir=opt.expt_dir) seq2tree = t.train(seq2tree, train, num_epochs=20, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5, resume=opt.resume) predictor = Predictor(seq2tree, input_vocab, input_vocab) while True: seq_str = raw_input("Type in a source sequence:") seq = seq_str.strip().split() print(predictor.predict(seq))
def __init__( self, task_path, figsize=(15, 13), decimals=2, is_show_attn_split=True, is_show_evaluation=True, output_length_key='length', attention_key="attention_score", position_attn_key='position_attention', content_attn_key='content_attention', positional_table_labels={ "λ%": "position_percentage", "C.γ": "content_confidence", #"lgt": "approx_max_logit", "C.λ": "pos_confidence", "μ": "mu", "σ": "sigma", "w_α": "mean_attn_old_weight", "w_j/n": "rel_counter_decoder_weight", "w_1/n": "single_step_weight", "w_μ": "mu_old_weight", "w_γ": "mean_content_old_weight", "w_1": "bias_weight" }, # "% carry": "carry_rates", is_show_name=True, max_src=17, max_out=13, max_tgt=13, **kwargs): check = Checkpoint.load(task_path) self.model = check.model # store some interesting variables self.model.set_dev_mode() self.predictor = Predictor(self.model, check.input_vocab, check.output_vocab) self.model_name = task_path.split("/")[-2] self.figsize = figsize self.decimals = decimals self.is_show_attn_split = is_show_attn_split self.is_show_evaluation = is_show_evaluation self.positional_table_labels = positional_table_labels self.is_show_name = is_show_name self.max_src = max_src self.max_out = max_out self.max_tgt = max_tgt self.output_length_key = output_length_key self.attention_key = attention_key self.position_attn_key = position_attn_key self.content_attn_key = content_attn_key if self.is_show_evaluation: self.is_symbol_rewriting = "symbol rewriting" in task_path.lower() self.metric_computer = MetricComputer( check, is_symbol_rewriting=self.is_symbol_rewriting, **kwargs) if self.model.decoder.is_attention is None: raise AttentionException("Model is not using attention.")
def eval_fa_equiv(model, data, input_vocab, output_vocab): loss = NLLLoss() batch_size = 1 model.eval() loss.reset() match = 0 total = 0 device = None if torch.cuda.is_available() else -1 batch_iterator = torchtext.data.BucketIterator( dataset=data, batch_size=batch_size, sort=False, sort_key=lambda x: len(x.src), device=device, train=False) tgt_vocab = data.fields[seq2seq.tgt_field_name].vocab pad = tgt_vocab.stoi[data.fields[seq2seq.tgt_field_name].pad_token] predictor = Predictor(model, input_vocab, output_vocab) num_samples = 0 perfect_samples = 0 dfa_perfect_samples = 0 match = 0 total = 0 with torch.no_grad(): for batch in batch_iterator: num_samples = num_samples + 1 input_variables, input_lengths = getattr(batch, seq2seq.src_field_name) target_variables = getattr(batch, seq2seq.tgt_field_name) target_string = decode_tensor(target_variables, output_vocab) #target_string = target_string + " <eos>" input_string = decode_tensor(input_variables, input_vocab) generated_string = ' '.join([ x for x in predictor.predict(input_string.strip().split())[:-1] if x != '<pad>' ]) #str(pos_example)[2] generated_string = refine_outout(generated_string) #str(pos_example)[2] pos_example = subprocess.check_output([ 'python2', 'regexDFAEquals.py', '--gold', '{}'.format(target_string), '--predicted', '{}'.format(generated_string) ]) if target_string == generated_string: perfect_samples = perfect_samples + 1 dfa_perfect_samples = dfa_perfect_samples + 1 elif str(pos_example)[2] == '1': dfa_perfect_samples = dfa_perfect_samples + 1 target_tokens = target_string.split() generated_tokens = generated_string.split() shorter_len = min(len(target_tokens), len(generated_tokens)) for idx in range(len(generated_tokens)): total = total + 1 if idx >= len(target_tokens): total = total + 1 elif target_tokens[idx] == generated_tokens[idx]: match = match + 1 if total == 0: accuracy = float('nan') else: accuracy = match / total string_accuracy = perfect_samples / num_samples dfa_accuracy = dfa_perfect_samples / num_samples f = open('./time_logs/log_score_time.txt', 'a') f.write('{}\n'.format(dfa_accuracy)) f.close()