def main(): arg_parser = argparse.ArgumentParser( description="Neural Machine Translation Testing") arg_parser.add_argument("--model_file", required=True, help="Model File") arg_parser.add_argument("--valid_data", required=True, nargs="+", help="Validation_data") args = arg_parser.parse_args() args = vars(args) print(args) model = Seq2Seq.load(args["model_file"]) print(model) model.device = "cpu" tr_dev_dataset_fn, en_dev_dataset_fn = args["valid_data"] tr_valid_data = read_text(tr_dev_dataset_fn) en_valid_data = read_text(en_dev_dataset_fn) valid_data = list(zip(tr_valid_data, en_valid_data)) src_valid, tgt_valid = add_start_end_tokens(valid_data) hypotheses = beam_search(model, src_valid, beam_size=3, max_decoding_time_step=70) top_hypotheses = [hyps[0] for hyps in hypotheses] bleu_score = compute_corpus_level_bleu_score(tgt_valid, top_hypotheses) print('Corpus BLEU: {}'.format(bleu_score * 100))
def main(): train_iterator, valid_iterator, test_iterator, params = prepare_data() (INPUT_DIM, OUTPUT_DIM, ENC_EMB_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT, DEC_DROPOUT) = params # INPUT_DIM = len(SRC.vocab), 7855 # OUTPUT_DIM = len(TRG.vocab), 5893 # ENC_EMB_DIM = 256 # DEC_EMB_DIM = 256 # ENC_HID_DIM = 512 # DEC_HID_DIM = 512 # ENC_DROPOUT = 0.5 # DEC_DROPOUT = 0.5 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') attn = Attention(ENC_HID_DIM, DEC_HID_DIM) enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT) dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn) model = Seq2Seq(enc, dec, device).to(device) model.apply(init_weights) print(f'The model has {count_parameters(model):,} trainable parameters') for i, batch in enumerate(train_iterator): print(f'ITER: {i}') example = batch print("Input Length:", example.src.shape, "[src_len, batch_size]") output = model.forward(example.src, example.trg) print(output.shape) print('') if i > 3: break
def load_models(load_path): model_args = json.load(open(os.path.join(load_path, 'options.json'), 'r')) vars(args).update(model_args) autoencoder = Seq2Seq(emsize=args.emsize, nhidden=args.nhidden, ntokens=args.ntokens, nlayers=args.nlayers, noise_r=args.noise_r, hidden_init=args.hidden_init, dropout=args.dropout, gpu=args.cuda) gan_gen = MLP_G(ninput=args.z_size, noutput=args.nhidden, layers=args.arch_g) gan_disc = MLP_D(ninput=args.nhidden, noutput=1, layers=args.arch_d) autoencoder = autoencoder.cuda() gan_gen = gan_gen.cuda() gan_disc = gan_disc.cuda() word2idx = json.load(open(os.path.join(args.save, 'vocab.json'), 'r')) idx2word = {v: k for k, v in word2idx.items()} print('Loading models from {}'.format(args.save)) loaded = torch.load(os.path.join(args.save, "model.pt")) autoencoder.load_state_dict(loaded.get('ae')) gan_gen.load_state_dict(loaded.get('gan_g')) gan_disc.load_state_dict(loaded.get('gan_d')) return model_args, idx2word, autoencoder, gan_gen, gan_disc
def main(): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128) optimizer = optim.Adam(model.parameters(), lr=0.001) criterion = nn.CrossEntropyLoss(reduction=None) nepochs = 25 batch_size = 64 if DEVICE == 'cuda' else 1 speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data( ) character_text_train = transform_letter_to_index(transcript_train, LETTER_LIST) character_text_valid = transform_letter_to_index(transcript_valid, LETTER_LIST) train_dataset = Speech2TextDataset(speech_train, character_text_train) # val_dataset = test_dataset = Speech2TextDataset(speech_test, None, False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate) # val_loader = test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_test) for epoch in range(nepochs): train(model, train_loader, criterion, optimizer, epoch) # val() test(model, test_loader, epoch)
def build_model(checkpoint): if checkpoint: voc = checkpoint['voc'] else: # Initialize word embeddings voc, pre_we_weight = init_word_embedding(config.WORD_EMBEDDING_FILES) print( f'word embedding size {config.WORD_EMBEDDING_SIZE}, hidden size {config.HIDDEN_SIZE}, layers {config.MODEL_LAYERS}' ) word_ebd_shape = (voc.size(), config.WORD_EMBEDDING_SIZE) model = Seq2Seq(word_ebd_shape, config.HIDDEN_SIZE, config.MODEL_LAYERS, config.MODEL_DROPOUT_RATE, config.RNN_TYPE, config.ATTN_TYPE) if checkpoint: model.load_state_dict(checkpoint['model']) else: model.load_pretrained_word_ebd(pre_we_weight) # Use appropriate device model = model.to(device) return model, voc
def main(): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=256, isAttended=True) # print(model) optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=5e-5) # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1) criterion = nn.CrossEntropyLoss(reduce=False, reduction=None) nepochs = 18 batch_size = 64 if DEVICE == 'cuda' else 1 speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data( ) character_text_train = transform_letter_to_index(transcript_train, LETTER_LIST) character_text_valid = transform_letter_to_index(transcript_valid, LETTER_LIST) train_dataset = Speech2TextDataset(speech_train, character_text_train) val_dataset = Speech2TextDataset(speech_valid, character_text_valid) test_dataset = Speech2TextDataset(speech_test, None, False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) #, collate_fn=collate_train) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True) #, collate_fn=collate_train) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False) #, collate_fn=collate_test) for epoch in range(nepochs): train(model, train_loader, criterion, optimizer, epoch) # scheduler.step() val(model, val_loader, criterion, epoch) # Test and Save results test_preds = test(model, test_loader) test_preds = test_preds.cpu().numpy() results = [] for i in range(test_preds.shape[0]): result = "" for j in range(test_preds.shape[1]): if (test_preds[i, j] == 0 or (test_preds[i, j] == 33)): continue if (test_preds[i, j] == 34): break result = result + index2letter[test_preds[i, j]] results.append(result) name = "Epoch_" + str(epoch) + "_LAS_submission.csv" ids = list(range(len(test_dataset))) ids.insert(0, 'Id') results.insert(0, 'Predicted') with open(name, 'w') as f: writer = csv.writer(f) writer.writerows(zip(ids, results))
def init_model(args): vocab = torch.load(args.vocab) cnn_encoder = CNNEncoder(len(vocab.src), args.embed_size) encoder = Encoder(cnn_encoder.out_size, args.hidden_size) devoder = Decoder(args.embed_size, args.hidden_size, len(vocab.tgt)) model = Seq2Seq(cnn_encoder, encoder, devoder, args, vocab) model.load_state_dict(torch.load(args.load_model_path)) model.eval() return vocab, model
def load_model(SRC, TRG, state): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") INPUT_DIM = len(SRC.vocab) OUTPUT_DIM = len(TRG.vocab) EMB_DIM = HID_DIM = state['hidden_dim'] DROPOUT = state['dropout'] enc_models = { 'lstm': LSTMEncoder, 'gru': GRUEncoder, 'lstm_attn': LSTMEncoder, 'gru_attn': GRUEncoder } dec_models = {'lstm': LSTMDecoder, 'gru_attn': GRUAttDecoder} enc_model = enc_models[state['rnn_type']] enc = enc_model(INPUT_DIM, EMB_DIM, HID_DIM, DROPOUT, bidirection=state['bidirection'], num_layers=state['num_layers']) enc = enc.to(device) dec_model = dec_models[state['rnn_type']] dec = dec_model(OUTPUT_DIM, EMB_DIM, HID_DIM, DROPOUT, device, num_layers=state['num_layers']) dec = dec.to(device) model = Seq2Seq(enc, dec, device, state['rnn_type']) model = model.to(device) #print('Encoder, Decoder, Model is on CUDA: ',enc.device, dec.device, model.device) def init_weights(m: nn.Module): for name, param in m.named_parameters(): if 'weight' in name: nn.init.normal_(param.data, mean=0, std=0.01) else: nn.init.constant_(param.data, 0) # Use default weight initialisation by pytorch # model.apply(init_weights) optimizer = optim.Adam(model.parameters(), lr=0.001) def count_parameters(model: nn.Module): return sum(p.numel() for p in model.parameters() if p.requires_grad) print(f'The model has {count_parameters(model):,} trainable parameters') PAD_IDX = TRG.vocab.stoi['PAD_TOKEN'] criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX) return model, optimizer, criterion
def create_new_model(): if args.model_type == "seq2seq": return Seq2Seq(word_vectors=word_vectors, hidden_size=args.hidden_size, output_size=vocab_size, device=device, num_layers=args.num_layers) elif args.model_type == "seq2seq_attn": return Seq2SeqAttn(word_vectors=word_vectors, hidden_size=args.hidden_size, output_size=vocab_size, device=device, num_layers=args.num_layers)
def build_model(config, word2id, id2word): # ------ Model ------# seq2seq = Seq2Seq( vocab_size=len(word2id), word_embed_size=config.embed_size, num_layers=config.num_layers, dropout_p=config.dropout, enc_hidden_size=config.enc_hidden_size, dec_hidden_size=config.dec_hidden_size, use_focus=config.use_focus, tie=config.weight_tie, task=config.task, rnn=config.rnn, model=config.model, feature_rich=config.feature_rich, n_mixture=config.n_mixture if config.mixture_decoder else None) print('Created Seq2Seq!') if config.use_focus and not config.eval_focus_oracle: selector = FocusSelector(word_embed_size=config.embed_size, num_layers=config.num_layers, enc_hidden_size=config.enc_hidden_size, dec_hidden_size=config.dec_hidden_size, n_mixture=config.n_mixture, dropout_p=0.2, rnn=config.rnn, seq2seq_model=config.model, task=config.task, threshold=config.threshold, feature_rich=config.feature_rich) # Selector share all embeddings with Seq2Seq model if config.feature_rich: selector.add_embedding(seq2seq.word_embed, seq2seq.answer_position_embed, seq2seq.ner_embed, seq2seq.pos_embed, seq2seq.case_embed) else: selector.add_embedding(seq2seq.word_embed) print('Created Focus Selector!') else: selector = None model = Model(seq2seq, selector) model.word2id = word2id model.id2word = id2word print(model) return model
def main(): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128, value_size=128, key_size=256, isAttended=True) model.load_state_dict(torch.load('model3')) model.eval() model = model.to(DEVICE) optimizer = optim.Adam(model.parameters(), lr=0.0005) criterion = nn.CrossEntropyLoss(reduction='sum') nepochs = 10 batch_size = 64 if DEVICE == 'cuda' else 1 speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data( ) character_text_train = transform_letter_to_index(transcript_train, LETTER_LIST) character_text_valid = transform_letter_to_index(transcript_valid, LETTER_LIST) train_dataset = Speech2TextDataset(speech_train, character_text_train) val_dataset = Speech2TextDataset(speech_valid, character_text_valid) test_dataset = Speech2TextDataset(speech_test, None, False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train) for x in train_loader: pass val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_test) for epoch in range(nepochs): print('==============', 'Epoch', epoch + 1, '================') train(model, train_loader, criterion, optimizer, epoch) val(model, val_loader, criterion, optimizer, epoch) torch.save(model.state_dict(), 'model3') load_model(model, test_loader)
def main(): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128) optimizer = optim.SGD(model.parameters(), lr=1e-4, weight_decay=5e-4) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.93) criterion = nn.CrossEntropyLoss(reduction='none') init_epoch = 0 nepochs = 50 batch_size = 64 if DEVICE == 'cuda' else 1 speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data( ) character_text_train = transform_letter_to_index(transcript_train, LETTER_LIST) character_text_valid = transform_letter_to_index(transcript_valid, LETTER_LIST) train_dataset = Speech2TextDataset(speech_train, character_text_train) val_dataset = Speech2TextDataset(speech_valid, character_text_valid) test_dataset = Speech2TextDataset(speech_test, None, False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train_val) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train_val) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_test) val_distances = [] exp = 27 # model.load_state_dict(torch.load('BestModel9.pth')) with open('stats_{}'.format(exp), 'w') as file: file.write('Experiment: {}\n'.format(exp)) for epoch in range(init_epoch, nepochs): train(model, train_loader, criterion, optimizer, scheduler, epoch, exp) val_distances.append(val(model, val_loader, epoch, exp)) if val_distances[-1] == min(val_distances): torch.save(model.state_dict(), 'BestModel{}.pth'.format(exp)) if epoch % 3 == 0 or epoch == nepochs - 1: test(model, test_loader, exp)
def main(): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128, value_size=128, key_size=256, is_attended=True) # cur_model_num = 6 # model.load_state_dict(torch.load('model_{}'.format(cur_model_num))) optimizer = optim.Adam(model.parameters(), lr=0.001) criterion = nn.CrossEntropyLoss(reduction="sum") n_epochs = 30 batch_size = 64 if DEVICE == 'cuda' else 1 speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data( ) character_text_train = transform_letter_to_index(transcript_train, LETTER2INDEX) character_text_valid = transform_letter_to_index(transcript_valid, LETTER2INDEX) train_dataset = Speech2TextDataset(speech_train, character_text_train) val_dataset = Speech2TextDataset(speech_valid, character_text_valid) test_dataset = Speech2TextDataset(speech_test, None, False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_test) for epoch in range(n_epochs): train(model, train_loader, criterion, optimizer, epoch) val(model, val_loader, criterion, epoch) # test(model, test_loader) torch.save(model.state_dict(), 'model_{}'.format(1)) result_gen(test_loader, 1)
def main(): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128) learningRate = 0.001 weightDecay = 5e-5 optimizer = torch.optim.Adam(model.parameters(), lr=learningRate, weight_decay=weightDecay) criterion = nn.CrossEntropyLoss(reduction='none') nepochs = 40 batch_size = 64 if DEVICE == 'cuda' else 1 speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data( ) character_text_train = transform_letter_to_index(transcript_train, LETTER_LIST) character_text_valid = transform_letter_to_index(transcript_valid, LETTER_LIST) train_dataset = Speech2TextDataset(speech_train, character_text_train) val_dataset = Speech2TextDataset(speech_valid, character_text_valid) test_dataset = Speech2TextDataset(speech_test, None, False) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_train) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_train) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_test) model.train() model.load_state_dict(torch.load('./new1.pth')) model.to(DEVICE) scheduler = StepLR(optimizer, step_size=10, gamma=0.5) for epoch in range(nepochs): train(model, train_loader, criterion, optimizer, epoch) scheduler.step() model.eval() data_list = test(model, test_loader) save_to_csv(data_list) print('done')
def __init__(self): self.autoencoder = Seq2Seq(emsize=args.emsize, nhidden=args.nhidden, ntokens=args.ntokens, nlayers=args.nlayers, noise_radius=args.noise_radius, hidden_init=args.hidden_init, dropout=args.dropout, gpu=args.cuda) self.optimizer = optim.SGD(self.autoencoder.parameters(), lr=args.lr) #self.optimizer = optim.Adam(self.autoencoder.parameters()) self.criterion = nn.CrossEntropyLoss() if args.cuda: self.autoencoder = self.autoencoder.cuda() self.criterion = self.criterion.cuda()
def create_new_model(): if args.model_type == "seq2seq": return Seq2Seq(word_vectors=word_vectors, hidden_size=args.hidden_size, output_size=vocab_size, device=device) elif args.model_type == "seq2seq_attn": return Seq2SeqAttn(word_vectors=word_vectors, hidden_size=args.hidden_size, output_size=vocab_size, device=device) elif args.model_type == "transformer": return TransformerModel(vocab_size, device, num_encoder_layers=2, num_decoder_layers=2, dropout=0.1)
def main(): args = parse_arguments() hidden_size = 512 embed_size = 256 print("[!] preparing dataset...") train_iter, val_iter, test_iter, DE, EN = load_dataset(args.batch_size) de_size, en_size = len(DE.vocab), len(EN.vocab) print("[TRAIN]:%d (dataset:%d)\t[TEST]:%d (dataset:%d)" % (len(train_iter), len( train_iter.dataset), len(test_iter), len(test_iter.dataset))) print("[DE_vocab]:%d [en_vocab]:%d" % (de_size, en_size)) print("[!] Instantiating models...") encoder = Encoder(de_size, embed_size, hidden_size, n_layers=2, dropout=0.5) decoder = Decoder(embed_size, hidden_size, en_size, n_layers=1, dropout=0.5) seq2seq = Seq2Seq(encoder, decoder) optimizer = optim.Adam(seq2seq.parameters(), lr=args.lr) print(seq2seq) best_val_loss = None for e in range(1, args.epochs + 1): train(e, seq2seq, optimizer, train_iter, en_size, args.grad_clip, DE, EN) val_loss = evaluate(seq2seq, val_iter, en_size, DE, EN) print("[Epoch:%d] val_loss:%5.3f | val_pp:%5.2fS" % (e, val_loss, math.exp(val_loss))) # Save the model if the validation loss is the best we've seen so far. if not best_val_loss or val_loss < best_val_loss: print("[!] saving model...") if not os.path.isdir(".save"): os.makedirs(".save") torch.save(seq2seq.state_dict(), './.save/seq2seq_%d.pt' % (e)) best_val_loss = val_loss test_loss = evaluate(seq2seq, test_iter, en_size, DE, EN) print("[TEST] loss:%5.2f" % test_loss)
def setup(): global char2index global index2char global SOS_token global EOS_token global PAD_token global model global device char2index, index2char = label_loader.load_label_json( "../data/kor_syllable_zeroth.json") SOS_token = char2index['<s>'] EOS_token = char2index['</s>'] PAD_token = char2index['_'] print(f"device: {device}") input_size = int(161) enc = EncoderRNN(input_size, 512, n_layers=3, dropout_p=0.3, bidirectional=True, rnn_cell='LSTM', variable_lengths=False) dec = DecoderRNN(len(char2index), 128, 512, 512, SOS_token, EOS_token, n_layers=2, rnn_cell='LSTM', dropout_p=0.3, bidirectional_encoder=True) model = Seq2Seq(enc, dec).to(device) model_path = "../models/zeroth_korean_trimmed/LSTM_512x3_512x2_zeroth_korean_trimmed/final.pth" print("Loading checkpoint model %s" % model_path) state = torch.load(model_path, map_location=device) model.load_state_dict(state['model']) print('Model loaded')
def translate(): if request.method == "GET": return render_template("index.html") elif request.method == "POST": args = request.form print(args) text_input = args["textarea"] print("Input: ", text_input) tokenized_sent = tokenizer.tokenize(text_input) print("Tokenized input: ", tokenized_sent) with open(VOCAB_FILE, "rb") as f: vocabs = pickle.load(f) model = Seq2Seq.load(MODEL_PATH) model.device = "cpu" hypothesis = beam_search(model, [tokenized_sent], beam_size=3, max_decoding_time_step=70)[0] print("Hypothesis") print(hypothesis) for i in range(3): new_target = [['<sos>'] + hypothesis[i].value + ['<eos>']] a_ts = generate_attention_map(model, vocabs, [tokenized_sent], new_target) save_attention(tokenized_sent, hypothesis[i].value, [ a[0].detach().cpu().numpy() for a in a_ts[:len(hypothesis[i].value)] ], save_path="static/list_{}.png".format(i)) result_hypothesis = [] for idx, hyp in enumerate(hypothesis): result_hypothesis.append((idx, " ".join(hyp.value))) return render_template("index.html", hypothesis=result_hypothesis, sentence=text_input)
def inference(params): config_gpu() # 配置GPU环境 # 构建模型 print("创建模型 ...") model = Seq2Seq(params=params) # 获取保存管理者 print("创建模型保存器") checkpoint = tf.train.Checkpoint(Seq2Seq=model) checkpoint_manager = tf.train.CheckpointManager(checkpoint, config.checkpoint_path, max_to_keep=3) if checkpoint_manager.latest_checkpoint: print("加载最新保存器数据 {} ...".format(checkpoint_manager.latest_checkpoint)) checkpoint.restore(checkpoint_manager.latest_checkpoint) # 训练模型 print("开始预测 ...") inference_model(model, params)
def beam_search(params): config_gpu() # 配置GPU # 加载数据集、模型 _, _, test_X = load_data() model = Seq2Seq(params) start = time.time() print('使用集束搜索开始预测...') results = [] dataset, steps_per_epoch = test_batch_generator(params['batch_size']) with tqdm(total=steps_per_epoch, position=0, leave=True) as tq: for (batch, batch_x) in enumerate(dataset.take(steps_per_epoch)): results += beam_decode(model, params, batch_x) tq.update(1) print('预测完成,耗时{}s\n处理至文件...'.format(time.time() - start)) def result_proc(text): """ 对预测结果做最后处理 :param text: 单条预测结果 :return: """ # text = text.lstrip(' ,!。') text = text.replace(' ', '') text = text.strip() if '<end>' in text: text = text[:text.index('<end>')] return text test_csv = pd.read_csv(config.test_set, encoding="UTF-8") # 赋值结果 test_csv['Prediction'] = results # 提取ID和预测结果两列 test_df = test_csv[['QID', 'Prediction']] # 结果处理 test_df['Prediction'] = test_df['Prediction'].apply(result_proc) # 保存结果 test_df.to_csv(config.inference_result_path, index=None, sep=',') print('已保存文件至{}'.format(config.inference_result_path))
def create_seq2seq_model(args, src, trg, loaded_vectors): """ Args: src: Field trg: Field """ input_dim = len(src.vocab) output_dim = len(trg.vocab) pad_idx = src.vocab.stoi['<pad>'] sos_idx = trg.vocab.stoi['<sos>'] eos_idx = trg.vocab.stoi['<eos>'] attn = Attention(args.enc_dim, args.dec_dim) enc = Encoder(input_dim, args.emb_dim, args.enc_dim, args.dec_dim, args.dropout, src.vocab.stoi, src.vocab.itos) dec = Decoder(output_dim, args.emb_dim, args.enc_dim, args.dec_dim, args.dropout, attn, trg.vocab.stoi, trg.vocab.itos) model = Seq2Seq(args, enc, dec, pad_idx, sos_idx, eos_idx, device, args.use_pretrained_embeddings, loaded_vectors, args.trainable_embeddings).to(device) print(f'The model has {count_parameters(model):,} trainable parameters')
def valid(valid_loader, encoder, decoder): encoder.eval() decoder.eval() model = Seq2Seq(encoder, decoder) losses = AverageMeter() # Batches for i, (features, trns, input_lengths) in enumerate(valid_loader): # Move to GPU, if available features = features.float().to(device) trns = trns.long().to(device) input_lengths = input_lengths.long().to(device) # Forward prop. loss = model(features, input_lengths, trns) # Keep track of metrics losses.update(loss.item()) return losses.avg
def train(train_loader, encoder, decoder, optimizer, epoch, logger): encoder.train() # train mode (dropout and batchnorm is used) decoder.train() model = Seq2Seq(encoder, decoder) losses = AverageMeter() # Batches for i, (features, trns, input_lengths) in enumerate(train_loader): # Move to GPU, if available features = features.float().to(device) trns = trns.long().to(device) input_lengths = input_lengths.long().to(device) # Forward prop. loss = model(features, input_lengths, trns) # Back prop. optimizer.zero_grad() loss.backward() # Clip gradients clip_gradient(optimizer, grad_clip) # Update weights optimizer.step() # Keep track of metrics losses.update(loss.item()) # Print status if i % print_freq == 0: logger.info('Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})'.format( epoch, i, len(train_loader), loss=losses)) return losses.avg
def result_gen(test_loader, model_num): model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128, value_size=128, key_size=256, is_attended=True) model.load_state_dict(torch.load('model_{}'.format(model_num + 1))) model.eval() model = model.to(DEVICE) test_text = test(model, test_loader) test_text_str = [] for cur_text in test_text: test_text_str.append(transform_index_to_letter(cur_text, LETTER_LIST)) res_df = pd.DataFrame(test_text_str) res_df.to_csv('result_{}.csv'.format(model_num + 1), index=True, header=False)
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT) if UNIFORM: dec = DecoderUniform(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn) SUFFIX = "_uniform" elif NO_ATTN or DECODE_WITH_NO_ATTN: dec = DecoderNoAttn(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn) if NO_ATTN: SUFFIX = "_no-attn" else: dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn) model = Seq2Seq(enc, dec, PAD_IDX, SOS_IDX, EOS_IDX, device).to(device) # init weights model.apply(init_weights) # count the params print(f'The model has {count_parameters(model):,} trainable parameters') optimizer = optim.Adam(model.parameters()) criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX) # --------- end of model definiition --------- # # NUM_EPOCHS = 5 CLIP = 1 best_valid_loss = float('inf') convergence_time = 0.0 epochs_taken_to_converge = 0
ENC_EMB_DIM = 8 DEC_EMB_DIM = 8 ENC_HID_DIM = 64 DEC_HID_DIM = 64 ENC_DROPOUT = 0 DEC_DROPOUT = 0 attn = Attention(ENC_HID_DIM, DEC_HID_DIM) enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT) dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn) model = Seq2Seq(enc, dec, device).to(device) print(model) optimizer = optim.Adam(model.parameters()) criterion = nn.CrossEntropyLoss() #ignore_index=digit2index.get('<PAD>') EPOCHS = 1 CLIP = 10 SAVE_DIR = './checkpoints/' MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'seq2seq_attention_model.pt') best_valid_loss = float('inf') if not os.path.isdir(f'{SAVE_DIR}'):
start = time.time() # load data training_set = SignalDataset_iq(path, time_step=total_time_step, train=True) test_set = SignalDataset_iq(path, time_step=total_time_step, train=False) train_loader = torch.utils.data.DataLoader(training_set, **params_dataloader) test_loader = torch.utils.data.DataLoader(test_set, **params_dataloader) end = time.time() # get num_classes from training data set num_classes = training_set.num_classes # init model encoder = Encoder_LSTM(**params_model) decoder = Decoder_LSTM(**params_model, output_dim=args.output_dim) model = Seq2Seq(encoder, decoder, device).to(device) print("Model size: {0}".format(count_parameters(model))) criterion = nn.CrossEntropyLoss(reduction="sum") op = torch.optim.Adam(model.parameters()) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(op, patience=2, factor=0.5, verbose=True) # training best_acc_train = -1 best_acc_test = -1 for epoch in range(args.epoch): start = time.time() print("Epoch %d" % epoch)
def main(argv=None): """ Training. """ ### parametres LEARNING_RATE = FLAGS.LEARNING_RATE NUMBER_OF_FRAMES = FLAGS.NUMBER_OF_FRAMES BATCH_SIZE = FLAGS.BATCH_SIZE EPOCH = FLAGS.EPOCH TRAINING_DEVICE = FLAGS.TRAINING_DEVICE VOCAB_SIZE = FLAGS.VOCAB_SIZE NUMBER_OF_WORDS = FLAGS.NUMBER_OF_WORDS HIDDEN_SIZE = FLAGS.HIDDEN_SIZE INPUT_SIZE = FLAGS.INPUT_SIZE NUMBER_OF_LAYERS = FLAGS.NUMBER_OF_LAYERS tsfm = transforms.Compose([ transforms.Resize([224, 224]), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), ]) train_corpus = FLAGS.train_corpus utils = Utils() all_text = utils.output_text(train_corpus) text_processor = TextProcessor(freq_threshold=10) dictionary = text_processor.vocab_creator(all_text) ### Model definition encoder = Encoder_LSTM(input_size=INPUT_SIZE, hidden_size=HIDDEN_SIZE, num_layers=NUMBER_OF_LAYERS) decoder = Decoder_LSTM(input_size=VOCAB_SIZE, hidden_size=HIDDEN_SIZE, num_layers=NUMBER_OF_LAYERS, number_of_words=NUMBER_OF_WORDS) model_seq_to_seq = Seq2Seq(encoder, decoder).to(device) model = model_seq_to_seq ### load the state_dict of model if model has been pretrained. model.load_state_dict(torch.load(FLAGS.load_weights)) #### Model Testing model.eval() from random import randint import matplotlib.pyplot as plt utils = Utils() video_path = FLAGS.video_file video_pre_data = utils.video_to_frames(video_path, frame_number=NUMBER_OF_FRAMES, device='cuda', INPUT_SIZE=INPUT_SIZE, model=md.model_vgg, transform=tsfm) X_2 = torch.zeros([NUMBER_OF_WORDS, VOCAB_SIZE]) for i in range(NUMBER_OF_WORDS): if (i == 0): X_2[i][2] = 1 else: X_2[i][1] = 1 input_data = video_pre_data.unsqueeze(0) final_sentence = [] X_2 = X_2.unsqueeze(0) X_2 = X_2.to(device) input_data = input_data.to(device) for i in range(NUMBER_OF_WORDS - 1): with torch.no_grad(): predicted = model(input_data, X_2) predicted = predicted.squeeze(0) final_sentence.append( next((key for key, value in dictionary.items() if value == torch.argmax(predicted[i])), None)) X_2[0][i + 1][torch.argmax(predicted[i])] = 1 X_2[0][i + 1][1] = 0 print(final_sentence)
def main(argv=None): """ Training. """ ### parametres LEARNING_RATE = FLAGS.LEARNING_RATE NUMBER_OF_FRAMES = FLAGS.NUMBER_OF_FRAMES BATCH_SIZE = FLAGS.BATCH_SIZE EPOCH = FLAGS.EPOCH TRAINING_DEVICE = FLAGS.TRAINING_DEVICE VOCAB_SIZE = FLAGS.VOCAB_SIZE NUMBER_OF_WORDS = FLAGS.NUMBER_OF_WORDS HIDDEN_SIZE = FLAGS.HIDDEN_SIZE INPUT_SIZE = FLAGS.INPUT_SIZE NUMBER_OF_LAYERS = FLAGS.NUMBER_OF_LAYERS tsfm = transforms.Compose([ transforms.Resize([224, 224]), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), ]) train_dir = FLAGS.train_dir #'D:/Machine_Learning/datasets/YouTubeClips_2/YouTubeClips/' train_corpus = FLAGS.train_corpus #'D:/Machine_Learning/datasets/video_corpus/video_corpus.csv' print("train_dir is =", train_dir) print("train_corpus =", train_corpus) utils = Utils() all_text = utils.output_text(train_corpus) text_processor = TextProcessor(freq_threshold=10) dictionary = text_processor.vocab_creator(all_text) ### training data preparation train_ds = CustomDataset(train_dir, train_corpus, device, dictionary, VOCAB_SIZE, NUMBER_OF_WORDS, INPUT_SIZE, NUMBER_OF_FRAMES, tsfm, model=md.model_vgg) train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE) ### Model definition encoder = Encoder_LSTM(input_size=INPUT_SIZE, hidden_size=HIDDEN_SIZE, num_layers=NUMBER_OF_LAYERS) decoder = Decoder_LSTM(input_size=VOCAB_SIZE, hidden_size=HIDDEN_SIZE, num_layers=NUMBER_OF_LAYERS, number_of_words=NUMBER_OF_WORDS) model_seq_to_seq = Seq2Seq(encoder, decoder).to(device) model = model_seq_to_seq ### load the state_dict of model if model has been pretrained. if (FLAGS.load_weights): print("there are weights to be loaded") model.load_state_dict(torch.load(FLAGS.load_weights)) ### optimizer and loss function optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) criterion = nn.CrossEntropyLoss() #### Model Training import time print_feq = 1 best_loss = np.inf for epoch in range(1, EPOCH + 1): model.train() epoch_loss = 0 for step, (img, label) in enumerate(train_dl): time_1 = time.time() ## timing X_1, X_2 = img ### get inputs X_1 = X_1.to(device) # Set device X_2 = X_2.to(device) # Set device label = label.to(device) # Set output device ### zero the parameter gradients optimizer.zero_grad() ### forward prediction = model(X_1, X_2) ### Optimize prediction = prediction.to(device) prediction = torch.squeeze(prediction, 0) label = torch.squeeze(label, 0) new_label = torch.zeros([label.shape[0]]) for l in range(label.shape[0]): new_label[l] = np.argmax(label[l].cpu()) new_label = new_label.to(device) loss = criterion(prediction, new_label.long()) # Backward prop. loss.backward() optimizer.step() ### print out statistics epoch_loss += loss.item() if step % print_feq == 0: print('epoch:', epoch, '\tstep:', step + 1, '/', len(train_dl) + 1, '\ttrain loss:', '{:.4f}'.format(loss.item()), '\ttime:', '{:.4f}'.format( (time.time() - time_1) * print_feq), 's') ### save best model if (epoch_loss < best_loss): best_loss = epoch_loss model_name = 'MODEL_SEQ2SEQ' + 'VOCAB_SIZE_' + str( VOCAB_SIZE) + 'NUMBER_OF_WORDS_' + str( NUMBER_OF_WORDS ) + 'HIDDEN_SIZE_' + str(HIDDEN_SIZE) + 'INPUT_SIZE_' + str( INPUT_SIZE) + 'NUMBER_OF_LAYERS_' + str(NUMBER_OF_LAYERS) torch.save(model.state_dict(), model_name + '.pth') print("The loss for this epoch is = :", epoch_loss / len(train_dl))