def main():
arg_parser = argparse.ArgumentParser(
description="Neural Machine Translation Testing")
arg_parser.add_argument("--model_file", required=True, help="Model File")
arg_parser.add_argument("--valid_data",
required=True,
nargs="+",
help="Validation_data")
args = arg_parser.parse_args()
args = vars(args)
print(args)
model = Seq2Seq.load(args["model_file"])
print(model)
model.device = "cpu"
tr_dev_dataset_fn, en_dev_dataset_fn = args["valid_data"]
tr_valid_data = read_text(tr_dev_dataset_fn)
en_valid_data = read_text(en_dev_dataset_fn)
valid_data = list(zip(tr_valid_data, en_valid_data))
src_valid, tgt_valid = add_start_end_tokens(valid_data)
hypotheses = beam_search(model,
src_valid,
beam_size=3,
max_decoding_time_step=70)
top_hypotheses = [hyps[0] for hyps in hypotheses]
bleu_score = compute_corpus_level_bleu_score(tgt_valid, top_hypotheses)
print('Corpus BLEU: {}'.format(bleu_score * 100))
def main():
train_iterator, valid_iterator, test_iterator, params = prepare_data()
(INPUT_DIM, OUTPUT_DIM, ENC_EMB_DIM, DEC_EMB_DIM,
ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT, DEC_DROPOUT) = params
# INPUT_DIM = len(SRC.vocab), 7855
# OUTPUT_DIM = len(TRG.vocab), 5893
# ENC_EMB_DIM = 256
# DEC_EMB_DIM = 256
# ENC_HID_DIM = 512
# DEC_HID_DIM = 512
# ENC_DROPOUT = 0.5
# DEC_DROPOUT = 0.5
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
attn = Attention(ENC_HID_DIM, DEC_HID_DIM)
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM,
DEC_HID_DIM, ENC_DROPOUT)
dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM,
DEC_HID_DIM, DEC_DROPOUT, attn)
model = Seq2Seq(enc, dec, device).to(device)
model.apply(init_weights)
print(f'The model has {count_parameters(model):,} trainable parameters')
for i, batch in enumerate(train_iterator):
print(f'ITER: {i}')
example = batch
print("Input Length:", example.src.shape, "[src_len, batch_size]")
output = model.forward(example.src, example.trg)
print(output.shape)
print('')
if i > 3: break
def load_models(load_path):
model_args = json.load(open(os.path.join(load_path, 'options.json'), 'r'))
vars(args).update(model_args)
autoencoder = Seq2Seq(emsize=args.emsize,
nhidden=args.nhidden,
ntokens=args.ntokens,
nlayers=args.nlayers,
noise_r=args.noise_r,
hidden_init=args.hidden_init,
dropout=args.dropout,
gpu=args.cuda)
gan_gen = MLP_G(ninput=args.z_size,
noutput=args.nhidden,
layers=args.arch_g)
gan_disc = MLP_D(ninput=args.nhidden, noutput=1, layers=args.arch_d)
autoencoder = autoencoder.cuda()
gan_gen = gan_gen.cuda()
gan_disc = gan_disc.cuda()
word2idx = json.load(open(os.path.join(args.save, 'vocab.json'), 'r'))
idx2word = {v: k for k, v in word2idx.items()}
print('Loading models from {}'.format(args.save))
loaded = torch.load(os.path.join(args.save, "model.pt"))
autoencoder.load_state_dict(loaded.get('ae'))
gan_gen.load_state_dict(loaded.get('gan_g'))
gan_disc.load_state_dict(loaded.get('gan_d'))
return model_args, idx2word, autoencoder, gan_gen, gan_disc
def main():
model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128)
optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss(reduction=None)
nepochs = 25
batch_size = 64 if DEVICE == 'cuda' else 1
speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data(
)
character_text_train = transform_letter_to_index(transcript_train,
LETTER_LIST)
character_text_valid = transform_letter_to_index(transcript_valid,
LETTER_LIST)
train_dataset = Speech2TextDataset(speech_train, character_text_train)
# val_dataset =
test_dataset = Speech2TextDataset(speech_test, None, False)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate)
# val_loader =
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=collate_test)
for epoch in range(nepochs):
train(model, train_loader, criterion, optimizer, epoch)
# val()
test(model, test_loader, epoch)
def build_model(checkpoint):
if checkpoint:
voc = checkpoint['voc']
else:
# Initialize word embeddings
voc, pre_we_weight = init_word_embedding(config.WORD_EMBEDDING_FILES)
print(
f'word embedding size {config.WORD_EMBEDDING_SIZE}, hidden size {config.HIDDEN_SIZE}, layers {config.MODEL_LAYERS}'
)
word_ebd_shape = (voc.size(), config.WORD_EMBEDDING_SIZE)
model = Seq2Seq(word_ebd_shape, config.HIDDEN_SIZE, config.MODEL_LAYERS,
config.MODEL_DROPOUT_RATE, config.RNN_TYPE,
config.ATTN_TYPE)
if checkpoint:
model.load_state_dict(checkpoint['model'])
else:
model.load_pretrained_word_ebd(pre_we_weight)
# Use appropriate device
model = model.to(device)
return model, voc
def main():
model = Seq2Seq(input_dim=40,
vocab_size=len(LETTER_LIST),
hidden_dim=256,
isAttended=True)
# print(model)
optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=5e-5)
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)
criterion = nn.CrossEntropyLoss(reduce=False, reduction=None)
nepochs = 18
batch_size = 64 if DEVICE == 'cuda' else 1
speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data(
)
character_text_train = transform_letter_to_index(transcript_train,
LETTER_LIST)
character_text_valid = transform_letter_to_index(transcript_valid,
LETTER_LIST)
train_dataset = Speech2TextDataset(speech_train, character_text_train)
val_dataset = Speech2TextDataset(speech_valid, character_text_valid)
test_dataset = Speech2TextDataset(speech_test, None, False)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True) #, collate_fn=collate_train)
val_loader = DataLoader(val_dataset, batch_size=batch_size,
shuffle=True) #, collate_fn=collate_train)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False) #, collate_fn=collate_test)
for epoch in range(nepochs):
train(model, train_loader, criterion, optimizer, epoch)
# scheduler.step()
val(model, val_loader, criterion, epoch)
# Test and Save results
test_preds = test(model, test_loader)
test_preds = test_preds.cpu().numpy()
results = []
for i in range(test_preds.shape[0]):
result = ""
for j in range(test_preds.shape[1]):
if (test_preds[i, j] == 0 or (test_preds[i, j] == 33)):
continue
if (test_preds[i, j] == 34):
break
result = result + index2letter[test_preds[i, j]]
results.append(result)
name = "Epoch_" + str(epoch) + "_LAS_submission.csv"
ids = list(range(len(test_dataset)))
ids.insert(0, 'Id')
results.insert(0, 'Predicted')
with open(name, 'w') as f:
writer = csv.writer(f)
writer.writerows(zip(ids, results))
def init_model(args):
vocab = torch.load(args.vocab)
cnn_encoder = CNNEncoder(len(vocab.src), args.embed_size)
encoder = Encoder(cnn_encoder.out_size, args.hidden_size)
devoder = Decoder(args.embed_size, args.hidden_size, len(vocab.tgt))
model = Seq2Seq(cnn_encoder, encoder, devoder, args, vocab)
model.load_state_dict(torch.load(args.load_model_path))
model.eval()
return vocab, model
def load_model(SRC, TRG, state):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TRG.vocab)
EMB_DIM = HID_DIM = state['hidden_dim']
DROPOUT = state['dropout']
enc_models = {
'lstm': LSTMEncoder,
'gru': GRUEncoder,
'lstm_attn': LSTMEncoder,
'gru_attn': GRUEncoder
}
dec_models = {'lstm': LSTMDecoder, 'gru_attn': GRUAttDecoder}
enc_model = enc_models[state['rnn_type']]
enc = enc_model(INPUT_DIM,
EMB_DIM,
HID_DIM,
DROPOUT,
bidirection=state['bidirection'],
num_layers=state['num_layers'])
enc = enc.to(device)
dec_model = dec_models[state['rnn_type']]
dec = dec_model(OUTPUT_DIM,
EMB_DIM,
HID_DIM,
DROPOUT,
device,
num_layers=state['num_layers'])
dec = dec.to(device)
model = Seq2Seq(enc, dec, device, state['rnn_type'])
model = model.to(device)
#print('Encoder, Decoder, Model is on CUDA: ',enc.device, dec.device, model.device)
def init_weights(m: nn.Module):
for name, param in m.named_parameters():
if 'weight' in name:
nn.init.normal_(param.data, mean=0, std=0.01)
else:
nn.init.constant_(param.data, 0)
# Use default weight initialisation by pytorch
# model.apply(init_weights)
optimizer = optim.Adam(model.parameters(), lr=0.001)
def count_parameters(model: nn.Module):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'The model has {count_parameters(model):,} trainable parameters')
PAD_IDX = TRG.vocab.stoi['PAD_TOKEN']
criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX)
return model, optimizer, criterion
def create_new_model():
if args.model_type == "seq2seq":
return Seq2Seq(word_vectors=word_vectors,
hidden_size=args.hidden_size,
output_size=vocab_size,
device=device,
num_layers=args.num_layers)
elif args.model_type == "seq2seq_attn":
return Seq2SeqAttn(word_vectors=word_vectors,
hidden_size=args.hidden_size,
output_size=vocab_size,
device=device,
num_layers=args.num_layers)
def build_model(config, word2id, id2word):
# ------ Model ------#
seq2seq = Seq2Seq(
vocab_size=len(word2id),
word_embed_size=config.embed_size,
num_layers=config.num_layers,
dropout_p=config.dropout,
enc_hidden_size=config.enc_hidden_size,
dec_hidden_size=config.dec_hidden_size,
use_focus=config.use_focus,
tie=config.weight_tie,
task=config.task,
rnn=config.rnn,
model=config.model,
feature_rich=config.feature_rich,
n_mixture=config.n_mixture if config.mixture_decoder else None)
print('Created Seq2Seq!')
if config.use_focus and not config.eval_focus_oracle:
selector = FocusSelector(word_embed_size=config.embed_size,
num_layers=config.num_layers,
enc_hidden_size=config.enc_hidden_size,
dec_hidden_size=config.dec_hidden_size,
n_mixture=config.n_mixture,
dropout_p=0.2,
rnn=config.rnn,
seq2seq_model=config.model,
task=config.task,
threshold=config.threshold,
feature_rich=config.feature_rich)
# Selector share all embeddings with Seq2Seq model
if config.feature_rich:
selector.add_embedding(seq2seq.word_embed,
seq2seq.answer_position_embed,
seq2seq.ner_embed, seq2seq.pos_embed,
seq2seq.case_embed)
else:
selector.add_embedding(seq2seq.word_embed)
print('Created Focus Selector!')
else:
selector = None
model = Model(seq2seq, selector)
model.word2id = word2id
model.id2word = id2word
print(model)
return model
def main():
model = Seq2Seq(input_dim=40,
vocab_size=len(LETTER_LIST),
hidden_dim=128,
value_size=128,
key_size=256,
isAttended=True)
model.load_state_dict(torch.load('model3'))
model.eval()
model = model.to(DEVICE)
optimizer = optim.Adam(model.parameters(), lr=0.0005)
criterion = nn.CrossEntropyLoss(reduction='sum')
nepochs = 10
batch_size = 64 if DEVICE == 'cuda' else 1
speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data(
)
character_text_train = transform_letter_to_index(transcript_train,
LETTER_LIST)
character_text_valid = transform_letter_to_index(transcript_valid,
LETTER_LIST)
train_dataset = Speech2TextDataset(speech_train, character_text_train)
val_dataset = Speech2TextDataset(speech_valid, character_text_valid)
test_dataset = Speech2TextDataset(speech_test, None, False)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train)
for x in train_loader:
pass
val_loader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=collate_test)
for epoch in range(nepochs):
print('==============', 'Epoch', epoch + 1, '================')
train(model, train_loader, criterion, optimizer, epoch)
val(model, val_loader, criterion, optimizer, epoch)
torch.save(model.state_dict(), 'model3')
load_model(model, test_loader)
def main():
model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128)
optimizer = optim.SGD(model.parameters(), lr=1e-4, weight_decay=5e-4)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.93)
criterion = nn.CrossEntropyLoss(reduction='none')
init_epoch = 0
nepochs = 50
batch_size = 64 if DEVICE == 'cuda' else 1
speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data(
)
character_text_train = transform_letter_to_index(transcript_train,
LETTER_LIST)
character_text_valid = transform_letter_to_index(transcript_valid,
LETTER_LIST)
train_dataset = Speech2TextDataset(speech_train, character_text_train)
val_dataset = Speech2TextDataset(speech_valid, character_text_valid)
test_dataset = Speech2TextDataset(speech_test, None, False)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train_val)
val_loader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train_val)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=collate_test)
val_distances = []
exp = 27
# model.load_state_dict(torch.load('BestModel9.pth'))
with open('stats_{}'.format(exp), 'w') as file:
file.write('Experiment: {}\n'.format(exp))
for epoch in range(init_epoch, nepochs):
train(model, train_loader, criterion, optimizer, scheduler, epoch, exp)
val_distances.append(val(model, val_loader, epoch, exp))
if val_distances[-1] == min(val_distances):
torch.save(model.state_dict(), 'BestModel{}.pth'.format(exp))
if epoch % 3 == 0 or epoch == nepochs - 1:
test(model, test_loader, exp)
def main():
model = Seq2Seq(input_dim=40,
vocab_size=len(LETTER_LIST),
hidden_dim=128,
value_size=128,
key_size=256,
is_attended=True)
# cur_model_num = 6
# model.load_state_dict(torch.load('model_{}'.format(cur_model_num)))
optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss(reduction="sum")
n_epochs = 30
batch_size = 64 if DEVICE == 'cuda' else 1
speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data(
)
character_text_train = transform_letter_to_index(transcript_train,
LETTER2INDEX)
character_text_valid = transform_letter_to_index(transcript_valid,
LETTER2INDEX)
train_dataset = Speech2TextDataset(speech_train, character_text_train)
val_dataset = Speech2TextDataset(speech_valid, character_text_valid)
test_dataset = Speech2TextDataset(speech_test, None, False)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train)
val_loader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=collate_test)
for epoch in range(n_epochs):
train(model, train_loader, criterion, optimizer, epoch)
val(model, val_loader, criterion, epoch)
# test(model, test_loader)
torch.save(model.state_dict(), 'model_{}'.format(1))
result_gen(test_loader, 1)
def main():
model = Seq2Seq(input_dim=40, vocab_size=len(LETTER_LIST), hidden_dim=128)
learningRate = 0.001
weightDecay = 5e-5
optimizer = torch.optim.Adam(model.parameters(),
lr=learningRate,
weight_decay=weightDecay)
criterion = nn.CrossEntropyLoss(reduction='none')
nepochs = 40
batch_size = 64 if DEVICE == 'cuda' else 1
speech_train, speech_valid, speech_test, transcript_train, transcript_valid = load_data(
)
character_text_train = transform_letter_to_index(transcript_train,
LETTER_LIST)
character_text_valid = transform_letter_to_index(transcript_valid,
LETTER_LIST)
train_dataset = Speech2TextDataset(speech_train, character_text_train)
val_dataset = Speech2TextDataset(speech_valid, character_text_valid)
test_dataset = Speech2TextDataset(speech_test, None, False)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_train)
val_loader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=collate_train)
test_loader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=collate_test)
model.train()
model.load_state_dict(torch.load('./new1.pth'))
model.to(DEVICE)
scheduler = StepLR(optimizer, step_size=10, gamma=0.5)
for epoch in range(nepochs):
train(model, train_loader, criterion, optimizer, epoch)
scheduler.step()
model.eval()
data_list = test(model, test_loader)
save_to_csv(data_list)
print('done')
def __init__(self):
self.autoencoder = Seq2Seq(emsize=args.emsize,
nhidden=args.nhidden,
ntokens=args.ntokens,
nlayers=args.nlayers,
noise_radius=args.noise_radius,
hidden_init=args.hidden_init,
dropout=args.dropout,
gpu=args.cuda)
self.optimizer = optim.SGD(self.autoencoder.parameters(), lr=args.lr)
#self.optimizer = optim.Adam(self.autoencoder.parameters())
self.criterion = nn.CrossEntropyLoss()
if args.cuda:
self.autoencoder = self.autoencoder.cuda()
self.criterion = self.criterion.cuda()
def create_new_model():
if args.model_type == "seq2seq":
return Seq2Seq(word_vectors=word_vectors,
hidden_size=args.hidden_size,
output_size=vocab_size,
device=device)
elif args.model_type == "seq2seq_attn":
return Seq2SeqAttn(word_vectors=word_vectors,
hidden_size=args.hidden_size,
output_size=vocab_size,
device=device)
elif args.model_type == "transformer":
return TransformerModel(vocab_size,
device,
num_encoder_layers=2,
num_decoder_layers=2,
dropout=0.1)
def main():
args = parse_arguments()
hidden_size = 512
embed_size = 256
print("[!] preparing dataset...")
train_iter, val_iter, test_iter, DE, EN = load_dataset(args.batch_size)
de_size, en_size = len(DE.vocab), len(EN.vocab)
print("[TRAIN]:%d (dataset:%d)\t[TEST]:%d (dataset:%d)" %
(len(train_iter), len(
train_iter.dataset), len(test_iter), len(test_iter.dataset)))
print("[DE_vocab]:%d [en_vocab]:%d" % (de_size, en_size))
print("[!] Instantiating models...")
encoder = Encoder(de_size,
embed_size,
hidden_size,
n_layers=2,
dropout=0.5)
decoder = Decoder(embed_size,
hidden_size,
en_size,
n_layers=1,
dropout=0.5)
seq2seq = Seq2Seq(encoder, decoder)
optimizer = optim.Adam(seq2seq.parameters(), lr=args.lr)
print(seq2seq)
best_val_loss = None
for e in range(1, args.epochs + 1):
train(e, seq2seq, optimizer, train_iter, en_size, args.grad_clip, DE,
EN)
val_loss = evaluate(seq2seq, val_iter, en_size, DE, EN)
print("[Epoch:%d] val_loss:%5.3f | val_pp:%5.2fS" %
(e, val_loss, math.exp(val_loss)))
# Save the model if the validation loss is the best we've seen so far.
if not best_val_loss or val_loss < best_val_loss:
print("[!] saving model...")
if not os.path.isdir(".save"):
os.makedirs(".save")
torch.save(seq2seq.state_dict(), './.save/seq2seq_%d.pt' % (e))
best_val_loss = val_loss
test_loss = evaluate(seq2seq, test_iter, en_size, DE, EN)
print("[TEST] loss:%5.2f" % test_loss)
def setup():
global char2index
global index2char
global SOS_token
global EOS_token
global PAD_token
global model
global device
char2index, index2char = label_loader.load_label_json(
"../data/kor_syllable_zeroth.json")
SOS_token = char2index['<s>']
EOS_token = char2index['</s>']
PAD_token = char2index['_']
print(f"device: {device}")
input_size = int(161)
enc = EncoderRNN(input_size,
512,
n_layers=3,
dropout_p=0.3,
bidirectional=True,
rnn_cell='LSTM',
variable_lengths=False)
dec = DecoderRNN(len(char2index),
128,
512,
512,
SOS_token,
EOS_token,
n_layers=2,
rnn_cell='LSTM',
dropout_p=0.3,
bidirectional_encoder=True)
model = Seq2Seq(enc, dec).to(device)
model_path = "../models/zeroth_korean_trimmed/LSTM_512x3_512x2_zeroth_korean_trimmed/final.pth"
print("Loading checkpoint model %s" % model_path)
state = torch.load(model_path, map_location=device)
model.load_state_dict(state['model'])
print('Model loaded')
def translate():
if request.method == "GET":
return render_template("index.html")
elif request.method == "POST":
args = request.form
print(args)
text_input = args["textarea"]
print("Input: ", text_input)
tokenized_sent = tokenizer.tokenize(text_input)
print("Tokenized input: ", tokenized_sent)
with open(VOCAB_FILE, "rb") as f:
vocabs = pickle.load(f)
model = Seq2Seq.load(MODEL_PATH)
model.device = "cpu"
hypothesis = beam_search(model, [tokenized_sent],
beam_size=3,
max_decoding_time_step=70)[0]
print("Hypothesis")
print(hypothesis)
for i in range(3):
new_target = [['<sos>'] + hypothesis[i].value + ['<eos>']]
a_ts = generate_attention_map(model, vocabs, [tokenized_sent],
new_target)
save_attention(tokenized_sent,
hypothesis[i].value, [
a[0].detach().cpu().numpy()
for a in a_ts[:len(hypothesis[i].value)]
],
save_path="static/list_{}.png".format(i))
result_hypothesis = []
for idx, hyp in enumerate(hypothesis):
result_hypothesis.append((idx, " ".join(hyp.value)))
return render_template("index.html",
hypothesis=result_hypothesis,
sentence=text_input)
def inference(params):
config_gpu() # 配置GPU环境
# 构建模型
print("创建模型 ...")
model = Seq2Seq(params=params)
# 获取保存管理者
print("创建模型保存器")
checkpoint = tf.train.Checkpoint(Seq2Seq=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint,
config.checkpoint_path,
max_to_keep=3)
if checkpoint_manager.latest_checkpoint:
print("加载最新保存器数据 {} ...".format(checkpoint_manager.latest_checkpoint))
checkpoint.restore(checkpoint_manager.latest_checkpoint)
# 训练模型
print("开始预测 ...")
inference_model(model, params)
def beam_search(params):
config_gpu() # 配置GPU
# 加载数据集、模型
_, _, test_X = load_data()
model = Seq2Seq(params)
start = time.time()
print('使用集束搜索开始预测...')
results = []
dataset, steps_per_epoch = test_batch_generator(params['batch_size'])
with tqdm(total=steps_per_epoch, position=0, leave=True) as tq:
for (batch, batch_x) in enumerate(dataset.take(steps_per_epoch)):
results += beam_decode(model, params, batch_x)
tq.update(1)
print('预测完成，耗时{}s\n处理至文件...'.format(time.time() - start))
def result_proc(text):
"""
对预测结果做最后处理
:param text: 单条预测结果
:return:
"""
# text = text.lstrip(' ，！。')
text = text.replace(' ', '')
text = text.strip()
if '<end>' in text:
text = text[:text.index('<end>')]
return text
test_csv = pd.read_csv(config.test_set, encoding="UTF-8")
# 赋值结果
test_csv['Prediction'] = results
# 提取ID和预测结果两列
test_df = test_csv[['QID', 'Prediction']]
# 结果处理
test_df['Prediction'] = test_df['Prediction'].apply(result_proc)
# 保存结果
test_df.to_csv(config.inference_result_path, index=None, sep=',')
print('已保存文件至{}'.format(config.inference_result_path))
def create_seq2seq_model(args, src, trg, loaded_vectors):
"""
Args:
src: Field
trg: Field
"""
input_dim = len(src.vocab)
output_dim = len(trg.vocab)
pad_idx = src.vocab.stoi['<pad>']
sos_idx = trg.vocab.stoi['<sos>']
eos_idx = trg.vocab.stoi['<eos>']
attn = Attention(args.enc_dim, args.dec_dim)
enc = Encoder(input_dim, args.emb_dim, args.enc_dim, args.dec_dim,
args.dropout, src.vocab.stoi, src.vocab.itos)
dec = Decoder(output_dim, args.emb_dim, args.enc_dim, args.dec_dim,
args.dropout, attn, trg.vocab.stoi, trg.vocab.itos)
model = Seq2Seq(args, enc, dec, pad_idx, sos_idx, eos_idx, device,
args.use_pretrained_embeddings, loaded_vectors,
args.trainable_embeddings).to(device)
print(f'The model has {count_parameters(model):,} trainable parameters')
def valid(valid_loader, encoder, decoder):
encoder.eval()
decoder.eval()
model = Seq2Seq(encoder, decoder)
losses = AverageMeter()
# Batches
for i, (features, trns, input_lengths) in enumerate(valid_loader):
# Move to GPU, if available
features = features.float().to(device)
trns = trns.long().to(device)
input_lengths = input_lengths.long().to(device)
# Forward prop.
loss = model(features, input_lengths, trns)
# Keep track of metrics
losses.update(loss.item())
return losses.avg
def train(train_loader, encoder, decoder, optimizer, epoch, logger):
encoder.train() # train mode (dropout and batchnorm is used)
decoder.train()
model = Seq2Seq(encoder, decoder)
losses = AverageMeter()
# Batches
for i, (features, trns, input_lengths) in enumerate(train_loader):
# Move to GPU, if available
features = features.float().to(device)
trns = trns.long().to(device)
input_lengths = input_lengths.long().to(device)
# Forward prop.
loss = model(features, input_lengths, trns)
# Back prop.
optimizer.zero_grad()
loss.backward()
# Clip gradients
clip_gradient(optimizer, grad_clip)
# Update weights
optimizer.step()
# Keep track of metrics
losses.update(loss.item())
# Print status
if i % print_freq == 0:
logger.info('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch, i, len(train_loader), loss=losses))
return losses.avg
def result_gen(test_loader, model_num):
model = Seq2Seq(input_dim=40,
vocab_size=len(LETTER_LIST),
hidden_dim=128,
value_size=128,
key_size=256,
is_attended=True)
model.load_state_dict(torch.load('model_{}'.format(model_num + 1)))
model.eval()
model = model.to(DEVICE)
test_text = test(model, test_loader)
test_text_str = []
for cur_text in test_text:
test_text_str.append(transform_index_to_letter(cur_text, LETTER_LIST))
res_df = pd.DataFrame(test_text_str)
res_df.to_csv('result_{}.csv'.format(model_num + 1),
index=True,
header=False)
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT)
if UNIFORM:
dec = DecoderUniform(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM,
DEC_DROPOUT, attn)
SUFFIX = "_uniform"
elif NO_ATTN or DECODE_WITH_NO_ATTN:
dec = DecoderNoAttn(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM,
DEC_DROPOUT, attn)
if NO_ATTN:
SUFFIX = "_no-attn"
else:
dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM,
DEC_DROPOUT, attn)
model = Seq2Seq(enc, dec, PAD_IDX, SOS_IDX, EOS_IDX, device).to(device)
# init weights
model.apply(init_weights)
# count the params
print(f'The model has {count_parameters(model):,} trainable parameters')
optimizer = optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX)
# --------- end of model definiition --------- #
# NUM_EPOCHS = 5
CLIP = 1
best_valid_loss = float('inf')
convergence_time = 0.0
epochs_taken_to_converge = 0
ENC_EMB_DIM = 8
DEC_EMB_DIM = 8
ENC_HID_DIM = 64
DEC_HID_DIM = 64
ENC_DROPOUT = 0
DEC_DROPOUT = 0
attn = Attention(ENC_HID_DIM, DEC_HID_DIM)
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM,
ENC_DROPOUT)
dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM,
DEC_DROPOUT, attn)
model = Seq2Seq(enc, dec, device).to(device)
print(model)
optimizer = optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss() #ignore_index=digit2index.get('<PAD>')
EPOCHS = 1
CLIP = 10
SAVE_DIR = './checkpoints/'
MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'seq2seq_attention_model.pt')
best_valid_loss = float('inf')
if not os.path.isdir(f'{SAVE_DIR}'):
start = time.time()
# load data
training_set = SignalDataset_iq(path, time_step=total_time_step, train=True)
test_set = SignalDataset_iq(path, time_step=total_time_step, train=False)
train_loader = torch.utils.data.DataLoader(training_set, **params_dataloader)
test_loader = torch.utils.data.DataLoader(test_set, **params_dataloader)
end = time.time()
# get num_classes from training data set
num_classes = training_set.num_classes
# init model
encoder = Encoder_LSTM(**params_model)
decoder = Decoder_LSTM(**params_model, output_dim=args.output_dim)
model = Seq2Seq(encoder, decoder, device).to(device)
print("Model size: {0}".format(count_parameters(model)))
criterion = nn.CrossEntropyLoss(reduction="sum")
op = torch.optim.Adam(model.parameters())
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(op,
patience=2,
factor=0.5,
verbose=True)
# training
best_acc_train = -1
best_acc_test = -1
for epoch in range(args.epoch):
start = time.time()
print("Epoch %d" % epoch)
def main(argv=None):
"""
Training.
"""
### parametres
LEARNING_RATE = FLAGS.LEARNING_RATE
NUMBER_OF_FRAMES = FLAGS.NUMBER_OF_FRAMES
BATCH_SIZE = FLAGS.BATCH_SIZE
EPOCH = FLAGS.EPOCH
TRAINING_DEVICE = FLAGS.TRAINING_DEVICE
VOCAB_SIZE = FLAGS.VOCAB_SIZE
NUMBER_OF_WORDS = FLAGS.NUMBER_OF_WORDS
HIDDEN_SIZE = FLAGS.HIDDEN_SIZE
INPUT_SIZE = FLAGS.INPUT_SIZE
NUMBER_OF_LAYERS = FLAGS.NUMBER_OF_LAYERS
tsfm = transforms.Compose([
transforms.Resize([224, 224]),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
train_corpus = FLAGS.train_corpus
utils = Utils()
all_text = utils.output_text(train_corpus)
text_processor = TextProcessor(freq_threshold=10)
dictionary = text_processor.vocab_creator(all_text)
### Model definition
encoder = Encoder_LSTM(input_size=INPUT_SIZE,
hidden_size=HIDDEN_SIZE,
num_layers=NUMBER_OF_LAYERS)
decoder = Decoder_LSTM(input_size=VOCAB_SIZE,
hidden_size=HIDDEN_SIZE,
num_layers=NUMBER_OF_LAYERS,
number_of_words=NUMBER_OF_WORDS)
model_seq_to_seq = Seq2Seq(encoder, decoder).to(device)
model = model_seq_to_seq
### load the state_dict of model if model has been pretrained.
model.load_state_dict(torch.load(FLAGS.load_weights))
#### Model Testing
model.eval()
from random import randint
import matplotlib.pyplot as plt
utils = Utils()
video_path = FLAGS.video_file
video_pre_data = utils.video_to_frames(video_path,
frame_number=NUMBER_OF_FRAMES,
device='cuda',
INPUT_SIZE=INPUT_SIZE,
model=md.model_vgg,
transform=tsfm)
X_2 = torch.zeros([NUMBER_OF_WORDS, VOCAB_SIZE])
for i in range(NUMBER_OF_WORDS):
if (i == 0):
X_2[i][2] = 1
else:
X_2[i][1] = 1
input_data = video_pre_data.unsqueeze(0)
final_sentence = []
X_2 = X_2.unsqueeze(0)
X_2 = X_2.to(device)
input_data = input_data.to(device)
for i in range(NUMBER_OF_WORDS - 1):
with torch.no_grad():
predicted = model(input_data, X_2)
predicted = predicted.squeeze(0)
final_sentence.append(
next((key for key, value in dictionary.items()
if value == torch.argmax(predicted[i])), None))
X_2[0][i + 1][torch.argmax(predicted[i])] = 1
X_2[0][i + 1][1] = 0
print(final_sentence)
def main(argv=None):
"""
Training.
"""
### parametres
LEARNING_RATE = FLAGS.LEARNING_RATE
NUMBER_OF_FRAMES = FLAGS.NUMBER_OF_FRAMES
BATCH_SIZE = FLAGS.BATCH_SIZE
EPOCH = FLAGS.EPOCH
TRAINING_DEVICE = FLAGS.TRAINING_DEVICE
VOCAB_SIZE = FLAGS.VOCAB_SIZE
NUMBER_OF_WORDS = FLAGS.NUMBER_OF_WORDS
HIDDEN_SIZE = FLAGS.HIDDEN_SIZE
INPUT_SIZE = FLAGS.INPUT_SIZE
NUMBER_OF_LAYERS = FLAGS.NUMBER_OF_LAYERS
tsfm = transforms.Compose([
transforms.Resize([224, 224]),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
train_dir = FLAGS.train_dir #'D:/Machine_Learning/datasets/YouTubeClips_2/YouTubeClips/'
train_corpus = FLAGS.train_corpus #'D:/Machine_Learning/datasets/video_corpus/video_corpus.csv'
print("train_dir is =", train_dir)
print("train_corpus =", train_corpus)
utils = Utils()
all_text = utils.output_text(train_corpus)
text_processor = TextProcessor(freq_threshold=10)
dictionary = text_processor.vocab_creator(all_text)
### training data preparation
train_ds = CustomDataset(train_dir,
train_corpus,
device,
dictionary,
VOCAB_SIZE,
NUMBER_OF_WORDS,
INPUT_SIZE,
NUMBER_OF_FRAMES,
tsfm,
model=md.model_vgg)
train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE)
### Model definition
encoder = Encoder_LSTM(input_size=INPUT_SIZE,
hidden_size=HIDDEN_SIZE,
num_layers=NUMBER_OF_LAYERS)
decoder = Decoder_LSTM(input_size=VOCAB_SIZE,
hidden_size=HIDDEN_SIZE,
num_layers=NUMBER_OF_LAYERS,
number_of_words=NUMBER_OF_WORDS)
model_seq_to_seq = Seq2Seq(encoder, decoder).to(device)
model = model_seq_to_seq
### load the state_dict of model if model has been pretrained.
if (FLAGS.load_weights):
print("there are weights to be loaded")
model.load_state_dict(torch.load(FLAGS.load_weights))
### optimizer and loss function
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
criterion = nn.CrossEntropyLoss()
#### Model Training
import time
print_feq = 1
best_loss = np.inf
for epoch in range(1, EPOCH + 1):
model.train()
epoch_loss = 0
for step, (img, label) in enumerate(train_dl):
time_1 = time.time() ## timing
X_1, X_2 = img ### get inputs
X_1 = X_1.to(device) # Set device
X_2 = X_2.to(device) # Set device
label = label.to(device) # Set output device
### zero the parameter gradients
optimizer.zero_grad()
### forward
prediction = model(X_1, X_2)
### Optimize
prediction = prediction.to(device)
prediction = torch.squeeze(prediction, 0)
label = torch.squeeze(label, 0)
new_label = torch.zeros([label.shape[0]])
for l in range(label.shape[0]):
new_label[l] = np.argmax(label[l].cpu())
new_label = new_label.to(device)
loss = criterion(prediction, new_label.long())
# Backward prop.
loss.backward()
optimizer.step()
### print out statistics
epoch_loss += loss.item()
if step % print_feq == 0:
print('epoch:', epoch, '\tstep:', step + 1, '/',
len(train_dl) + 1, '\ttrain loss:',
'{:.4f}'.format(loss.item()), '\ttime:', '{:.4f}'.format(
(time.time() - time_1) * print_feq), 's')
### save best model
if (epoch_loss < best_loss):
best_loss = epoch_loss
model_name = 'MODEL_SEQ2SEQ' + 'VOCAB_SIZE_' + str(
VOCAB_SIZE) + 'NUMBER_OF_WORDS_' + str(
NUMBER_OF_WORDS
) + 'HIDDEN_SIZE_' + str(HIDDEN_SIZE) + 'INPUT_SIZE_' + str(
INPUT_SIZE) + 'NUMBER_OF_LAYERS_' + str(NUMBER_OF_LAYERS)
torch.save(model.state_dict(), model_name + '.pth')
print("The loss for this epoch is = :", epoch_loss / len(train_dl))
