def post(self):
args = predict_parser.parse_args()
input_question = args.question
input_file = "../Data/prediction_api_input"
file = open(input_file, "w")
file.write(input_question)
file.close()
return s2s.predict_seq2seq(input_file, '../Data/vocab_map',
'../model/seq2seq', 'API')
def input_process(id):
while True:
if states[id]:
time.sleep(2)
else:
in_dict = reply_dict[id]
in_seq = input_str[id]
print(time.asctime() + ' 线程' + str(id) + '正在处理消息')
output = rnn.predict(in_seq)
data = {"group_id": in_dict['group_id'],
"message": "[CQ:at,qq=" + str(in_dict['sender']['user_id']) + "] " + output}
requests.post(url, data=data)
states[id] = 1
def main():
print("STARTED \n")
if len(sys.argv) > 1:
mode = sys.argv[1]
else:
raise ValueError("Main mode option not provided")
tf.logging._logger.setLevel(logging.INFO)
if mode.upper() == "TRAINING":
#FOR TRAINING
print("Training Started")
s2s.train_seq2seq('Data/final_question_file', 'Data/final_answer_file', 'Data/vocab_map', 'model/seq2seq')
elif mode.upper() == "INFERENCE":
if len(sys.argv) > 2:
pass
else:
sys.argv.append("COMMAND")
#FOR PREDICTION --- 'file'/'command'
infer_mode = sys.argv[2]
if infer_mode.upper() == "FILE":
print("Entered Inference File Mode")
s2s.predict_seq2seq('Data/prediction_input','Data/vocab_map', 'model/seq2seq', 'FILE')
elif infer_mode.upper() == "COMMAND":
print("Entered Inference Command Mode")
command_line_input = input("Question: ")
ans = s2s.predict_seq2seq('Data/prediction_input','Data/vocab_map', 'model/seq2seq', 'COMMAND', None,
command_line_input)
print("\nQuestion: ", command_line_input)
print("\nAnswer: ", ans.replace('<EOS>',''))
else:
raise ValueError("Correct Inference mode (FILE/COMMAND) was not supplied")
elif mode.upper() == 'TESTING':
s2s.predict_seq2seq('Data/testing_input_file', 'Data/vocab_map', 'model/seq2seq', 'TESTING',
'Data/testing_ref_file')
else:
raise ValueError("Correct Main mode (Training/Inference) was not supplied")
print("\nFINISHED \n")
print("-----------------")
print("本软件完全免费")
print("代码遵循MIT协议")
print("-----------------")
print("")
mode = '0'
while True:
print("-----------------")
print("模式1：搭建模型")
print("模式2：训练模型")
print('模式3：开启Coolq接口')
print("模式4：进行对话")
print("-----------------")
mode = input('输入工作模式：')
if mode == '1':
rnn.pre_precess()
rnn.setup_model()
elif mode == '2':
epo = input('输入循环轮数：')
bat = input('输入batch size:')
rnn.train_model(bat, epo)
elif mode == '3':
t0 = threading.Thread(target=input_process, args=(0,), name='http_receive0')
t0.start()
http_receive()
elif mode == '4':
print('输入数字0 退出')
while True:
str_in = input('你说：')
if str_in == '0':
break
sort_within_batch=True,
repeat=False)
### Encoder
### Decoder
### Seq2Seq
INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TRG.vocab)
enc = Model.Encoder(INPUT_DIM, Paras.ENC_EMB_DIM, Paras.HID_DIM,
Paras.N_LAYERS, Paras.ENC_DROPOUT)
dec = Model.Decoder(OUTPUT_DIM, Paras.DEC_EMB_DIM, Paras.HID_DIM,
Paras.N_LAYERS, Paras.DEC_DROPOUT)
model = Seq2Seq.Seq2Seq(enc, dec, device).to(device)
### init_weights
model.apply(Model.init_weights)
### count_parameters
print(f'The model has {Model.count_parameters(model):,} trainable parameters')
optimizer = optim.Adam(model.parameters())
# calculating loss，ignoring loss of padding token
PAD_IDX = TRG.vocab.stoi['<pad>']
criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--resume_ret', action='store_true')
parser.add_argument('--fourthofdata', action='store_true')
parser.add_argument('--halfdata', action='store_true')
opt = parser.parse_args()
############################### RETRIEVER #################################
ret_enc = RNNEncoder(ret_INPUT_DIM, ret_ENC_EMB_DIM, ret_HID_DIM,
ret_N_LAYERS, ret_ENC_DROPOUT)
ret_dec = RNNDecoder(ret_OUTPUT_DIM, ret_DEC_EMB_DIM, ret_HID_DIM,
ret_N_LAYERS, ret_DEC_DROPOUT)
ret_model = Seq2Seq(ret_enc, ret_dec, cuda_device).to(cuda_device)
print('The model has {0:9d} trainable parameters'.format(
count_parameters(ret_model)))
ret_optimizer = optim.Adam(ret_model.parameters())
ret_criterion = nn.CrossEntropyLoss()
if not os.path.isdir('models'):
os.makedirs('models')
if opt.resume_ret:
with open("results/" + "ret" + "_data.pickle", "rb") as k:
data = pickle.load(k)
train_data = data["train"]
valid_data = data["valid"]
test_data = data["test"]
train_data_adj = data["train_adj"]
valid_data_adj = data["valid_adj"]
test_data_adj = data["test_adj"]
print("valid data", valid_data)
MODEL_SAVE_PATH = os.path.join(SAVE_DIR, "ret" + '_model.pt')
ret_model.load_state_dict(torch.load(MODEL_SAVE_PATH))
with open("results/" + "ret" + "_latent_space_vect.pickle", "rb") as j:
latent_space_vects = pickle.load(j)
enc_train_vect = latent_space_vects["train"]
enc_valid_vect = latent_space_vects["valid"]
else:
train_data, valid_data, test_data, train_data_adj, valid_data_adj, test_data_adj = split_data(
opt)
data = {}
data["train"] = train_data
data["valid"] = valid_data
data["test"] = test_data
data["train_adj"] = train_data_adj
data["valid_adj"] = valid_data_adj
data["test_adj"] = test_data_adj
with open("results/" + "ret" + "_data.pickle", "wb") as k:
pickle.dump(data, k)
enc_train_vect, enc_valid_vect = train_valid_model(
filename="ret",
which_train=ret_train,
which_evaluate=ret_evaluate,
model=ret_model,
train_data=train_data,
valid_data=valid_data,
train_data_adj=train_data_adj,
valid_data_adj=valid_data_adj,
optimizer=ret_optimizer,
criterion=ret_criterion)
enc_test_vect = test_model(filename="ret",
which_evaluate=ret_evaluate,
model=ret_model,
test_data=test_data,
test_data_adj=test_data_adj,
criterion=ret_criterion)
######################## NEAREST NEIGHBOUR #################################
train_ann = create_annoy_index("AttnEncAttnDecTrain", enc_train_vect)
valid_ann = create_annoy_index("AttnEncAttnDecValid", enc_valid_vect)
test_ann = create_annoy_index("AttnEncAttnDecTest", enc_test_vect)
wordlist2comment_dict = pickle.load(open("wordlist2comment.pickle", "rb"))
word2idcommentvocab_dict = pickle.load(
open("word2idcommentvocab.pickle", "rb"))
sim_train_data = torch.zeros_like(train_data)
sim_valid_data = torch.zeros_like(valid_data)
sim_test_data = torch.zeros_like(test_data)
for training_sample_id in range(train_data.shape[0]):
training_sample_comment = train_data[
training_sample_id][:max_comment_len]
training_sample_code = train_data[training_sample_id][max_comment_len +
1:]
annoy_vect = train_ann.get_item_vector(training_sample_id)
sim_vect_id = train_ann.get_nns_by_vector(annoy_vect, 1)
if sim_vect_id == training_sample_id:
print("Same id for training vect and similar vect")
exit(0)
sim_train_data[training_sample_id] = train_data[sim_vect_id]
new_train_data = torch.cat((train_data, sim_train_data), dim=1)
#print("new_train_data ", new_train_data.shape)
for valid_sample_id in range(valid_data.shape[0]):
valid_sample_comment = valid_data[valid_sample_id][:max_comment_len]
valid_sample_code = valid_data[valid_sample_id][max_comment_len + 1:]
annoy_vect = valid_ann.get_item_vector(valid_sample_id)
sim_vect_id = train_ann.get_nns_by_vector(annoy_vect, 1)
if sim_vect_id == valid_sample_id:
print("Same id for training vect and similar vect")
exit(0)
sim_valid_data[valid_sample_id] = train_data[sim_vect_id]
new_valid_data = torch.cat((valid_data, sim_valid_data), dim=1)
for test_sample_id in range(test_data.shape[0]):
test_sample_comment = test_data[test_sample_id][:max_comment_len]
test_sample_code = test_data[test_sample_id][max_comment_len + 1:]
annoy_vect = test_ann.get_item_vector(test_sample_id)
sim_vect_id = train_ann.get_nns_by_vector(annoy_vect, 1)
if sim_vect_id == test_sample_id:
print("Same id for training vect and similar vect")
exit(0)
sim_test_data[test_sample_id] = train_data[sim_vect_id]
new_test_data = torch.cat((test_data, sim_test_data), dim=1)
############################### TSNE #################################
#tsne_test_sample = enc_test_vect[0]
num_tsne_train_data = 100
which_tsne_test_sample = random.randint(0, enc_test_vect.shape[0])
annoy_tsne_test_vect = test_ann.get_item_vector(which_tsne_test_sample)
tsne_data = enc_train_vect[:num_tsne_train_data]
tsne_data_add = torch.zeros(11, enc_test_vect.shape[1], device=cuda_device)
tsne_data_add[0] = enc_test_vect[which_tsne_test_sample]
nr = 1
for id in train_ann.get_nns_by_vector(annoy_tsne_test_vect, 10):
tsne_data_add[nr] = enc_train_vect[id]
nr += 1
tsne_data = torch.cat((tsne_data, tsne_data_add), dim=0)
colour_labels = []
for i in range(num_tsne_train_data):
colour_labels += ["#0099cc"] #train
colour_labels += ["#e60b42"] #test
for i in range(10):
colour_labels += ["#f09a00"] #nearest neighbours
vis_tsne(data=tsne_data, labels=colour_labels, name="10nearest")
############################### EDITOR #################################
ed_enc = GraphCondAttnEncoder(src_vocab_size, trg_vocab_size, ed_hid_dim,
ed_n_layers, ed_n_heads, ed_pf_dim,
AttnEncoderLayer, SelfAttention,
PositionwiseFeedforward, ed_dropout,
cuda_device)
ed_dec = AttnDecoder(ed_output_dim, ed_hid_dim, ed_n_layers, ed_n_heads,
ed_pf_dim, AttnDecoderLayer, SelfAttention,
PositionwiseFeedforward, ed_dropout, cuda_device)
ed_pad_idx = 0
ed_model = Editor(ed_enc, ed_dec, ed_pad_idx, cuda_device).to(cuda_device)
for p in ed_model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
print('The model has {0:9d} trainable parameters'.format(
count_parameters(ed_model)))
ed_optimizer = optim.Adam(ed_model.parameters())
ed_criterion = nn.CrossEntropyLoss()
output_train_vect, output_valid_vect_candidates = train_valid_model(
filename="ed",
which_train=ed_train,
which_evaluate=ed_evaluate,
model=ed_model,
train_data=new_train_data,
valid_data=new_valid_data,
train_data_adj=train_data_adj,
valid_data_adj=valid_data_adj,
optimizer=ed_optimizer,
criterion=ed_criterion)
#print("Test model")
output_test_vect_candidates = test_model(filename="ed",
which_evaluate=ed_evaluate,
model=ed_model,
test_data=new_test_data,
test_data_adj=test_data_adj,
criterion=ed_criterion)
output_test_vect_reference = test_data[:, max_comment_len:]
token_dict = pickle.load(open("codevocab.pickle", "rb"))
all_refs = []
all_cands = []
all_bleu_scores = []
for j in range(test_data.shape[0]):
ref = []
cand = []
for i in range(max_code_len):
ref_el = output_test_vect_reference[j][i].item()
cand_el = output_test_vect_candidates[j][i].item()
if ref_el > 0:
if ref_el in token_dict:
ref += [token_dict[ref_el]]
if cand_el in token_dict:
cand += [token_dict[cand_el]]
bleu = sentence_bleu([ref], cand)
all_bleu_scores += [bleu]
all_refs += [ref]
all_cands += [cand]
bleu_eval = {}
bleu_eval["scores"] = all_bleu_scores
bleu_eval["references"] = all_refs
bleu_eval["candidates"] = all_cands
print("Average BLEU score is ",
sum(all_bleu_scores) / len(all_bleu_scores))
pickle.dump(bleu_eval, open("results/bleu_evaluation_results.pickle",
"wb"))
def train():
parser = ArgumentParser()
parser.add_argument(
"--dataset_path",
type=str,
default="../data/time_transfor/Time Dataset.json",
help="Path or url of the dataset. If empty download from S3.")
parser.add_argument("--dataset_cache",
type=str,
default='../cache/',
help="Path or url of the dataset cache")
parser.add_argument("--batch_size",
type=int,
default=32,
help="Batch size for validation")
parser.add_argument("--embedding_dim",
type=int,
default=100,
help="Batch size for validation")
parser.add_argument("--hidden_dim",
type=int,
default=100,
help="Batch size for validation")
parser.add_argument("--gradient_accumulation_steps",
type=int,
default=1,
help="Accumulate gradients on several steps")
parser.add_argument("--lr",
type=float,
default=6.25e-5,
help="Learning rate")
parser.add_argument("--train_precent",
type=float,
default=0.7,
help="Batch size for validation")
parser.add_argument("--n_epochs",
type=int,
default=30,
help="Number of training epochs")
parser.add_argument("--device",
type=str,
default="cuda" if torch.cuda.is_available() else "cpu",
help="Device (cuda or cpu)")
parser.add_argument("--max_norm",
type=float,
default=1.0,
help="Clipping gradient norm")
parser.add_argument("--log_step",
type=int,
default=10,
help="Multiple-choice loss coefficient")
parser.add_argument("--raw_data",
action='store_true',
default=True,
help="If true read data by raw function")
args = parser.parse_args()
device = torch.device(args.device)
train_data_loader, valid_data_loader, input_lengths, target_lengths = get_data_loaders(
args.dataset_path, args.batch_size, args.train_precent)
encoder = Encoder(input_lengths + 1, args.embedding_dim, args.hidden_dim)
decoder = Decoder(target_lengths + 1, args.embedding_dim, args.hidden_dim)
model = Seq2Seq(encoder, decoder, device).to(device)
optimizer = optim.Adam(model.parameters())
criterion = nn.NLLLoss(ignore_index=0).to(device)
def update(engine, batch):
model.train()
src_seqs = batch[0].transpose(0, 1).to(device)
src_lengths = batch[1].to(device)
trg_seqs = batch[2].transpose(0, 1).to(device)
output = model(src_seqs, src_lengths, trg_seqs)
loss = criterion(output.contiguous().view(-1, output.shape[2]),
trg_seqs.contiguous().view(-1))
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_norm)
if engine.state.iteration % args.gradient_accumulation_steps == 0:
optimizer.step()
optimizer.zero_grad()
return loss.item()
trainer = Engine(update)
def inference(engine, batch):
model.eval()
with torch.no_grad():
src_seqs = batch[0].transpose(0, 1).to(device)
src_lengths = batch[1].to(device)
trg_seqs = batch[2].transpose(0, 1).to(device)
output = model(src_seqs, src_lengths, trg_seqs)
return output.contiguous().view(
-1, output.shape[2]), trg_seqs.contiguous().view(-1)
evaluator = Engine(inference)
metrics = {
"nll": Loss(criterion, output_transform=lambda x: (x[0], x[1])),
"accuracy": Accuracy(output_transform=lambda x: (x[0], x[1]))
}
for name, metric in metrics.items():
metric.attach(evaluator, name)
Loss(criterion, output_transform=lambda x: (x[0], x[1]))
@trainer.on(Events.EPOCH_COMPLETED)
def log_validation_results(trainer):
evaluator.run(valid_data_loader)
ms = evaluator.state.metrics
logger.info(
"Validation Results - Epoch: [{}/{}] Avg accuracy: {:.6f} Avg loss: {:.6f}"
.format(trainer.state.epoch, trainer.state.max_epochs,
ms['accuracy'], ms['nll']))
'''======================early stopping =========================='''
def score_function(engine):
val_loss = engine.state.metrics['nll']
return -val_loss
handler = EarlyStopping(patience=5,
score_function=score_function,
trainer=trainer)
evaluator.add_event_handler(Events.COMPLETED, handler)
'''==================print information by iterator========================='''
steps = len(train_data_loader.dataset) // train_data_loader.batch_size
steps = steps if steps > 0 else 1
logger.info('steps:%d' % steps)
@trainer.on(Events.ITERATION_COMPLETED)
def log_training_loss(trainer):
if trainer.state.iteration % args.log_step == 0:
logger.info("Epoch[{}/{}] Step[{}/{}] Loss: {:.6f}".format(
trainer.state.epoch, trainer.state.max_epochs,
trainer.state.iteration % steps, steps,
trainer.state.output * args.gradient_accumulation_steps))
'''================add check point========================'''
checkpoint_handler = ModelCheckpoint(checkpoint_dir,
'checkpoint',
save_interval=1,
n_saved=3)
trainer.add_event_handler(
Events.EPOCH_COMPLETED, checkpoint_handler,
{'mymodel': getattr(model, 'module', model)
}) # "getattr" take care of distributed encapsulation
'''==============run trainer============================='''
trainer.run(train_data_loader, max_epochs=args.n_epochs)
def main(N_EPOCHS, learning_rate, batch_size, device, save_dir):
dataset = util.load_jsonl(train_path)
dataset = Dataset.Dataset(dataset)
train_length = int(len(dataset) * 0.8)
valid_length = len(dataset) - train_length
train_set, val_set = torch.utils.data.random_split(
dataset, (train_length, valid_length))
# default batch size 보다 작은 부분 남았을 경우 check
#train_set, val_set,_ = torch.utils.data.random_split(dataset, (32, 32,len(dataset)-64))
test_set = util.load_jsonl(test_path)
test_set = Dataset.Dataset(test_set)
# 생성된 문장 차원 check
#test_set,_ = torch.utils.data.random_split(test_set, (2,len(test_set)-2))
train_dataloader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size,
shuffle=True)
val_dataloader = torch.utils.data.DataLoader(
val_set, batch_size=batch_size) # shuffle=True
test_dataloader = torch.utils.data.DataLoader(test_set,
batch_size=1) # 한줄씩 생성요약
# input dim, output dim 변경
INPUT_DIM = 49990 #80000
OUTPUT_DIM = 49990 #80000
ENC_EMB_DIM = 32
DEC_EMB_DIM = 32
HID_DIM = 512
N_LAYERS = 2
ENC_DROPOUT = 0.5
DEC_DROPOUT = 0.5
vocab = Vocab.Vocab()
enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, N_LAYERS, ENC_DROPOUT,
device)
dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, N_LAYERS, DEC_DROPOUT)
model = Seq2Seq(enc, dec, device).to(device)
model.apply(init_weights)
torch.autograd.set_detect_anomaly(True)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
criterion = nn.CrossEntropyLoss(ignore_index=0).to(device) # padding
#N_EPOCHS = 1
CLIP = 1
best_valid_loss = 100 #float('inf')
'''
for epoch in range(N_EPOCHS):
#start_time = time.time()
train_loss = train(model, train_dataloader, optimizer, criterion, CLIP, vocab, device)
valid_loss = evaluate(model, val_dataloader, criterion, vocab, device) # , valid_sents
#end_time = time.time()
#epoch_mins, epoch_secs = epoch_time(start_time, end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model, f'{save_dir}/seq2seq.pt')
#print(f'Epoch: {epoch+1:2} | Time: {epoch_mins}m {epoch_secs}s')
print(f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')
print(f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}')
'''
model = torch.load(f'{save_dir}/seq2seq.pt')
test_sents = test(model, test_dataloader, criterion, vocab, device)
save_csv(out_path, test_sents)
NB_HIDDEN = 250
BATCH_SIZE = 50
NB_STEP = 15
NB_FEATURES = 26 # Alphabet
LEARNING_RATE = 1e-2
STOP_TRESHOLD = 1.
LOG_DIR = "logs/" + str(LEARNING_RATE) + "_learning_rate"
# Random one hot batch generator tensor
random_one_hot_batch_generator = tf.one_hot(tf.random_uniform([NB_STEP], minval=0, \
maxval=NB_FEATURES - 1, \
dtype=tf.int32), NB_FEATURES)
if __name__ == "__main__":
seq2seq = Seq2Seq(NB_FEATURES, NB_HIDDEN, NB_HIDDEN, LEARNING_RATE)
init_global = tf.global_variables_initializer()
init_local = tf.local_variables_initializer()
loss_placeholder = tf.placeholder(tf.float32, [])
loss_summary = tf.summary.scalar("Loss", loss_placeholder)
accuracy_placeholder = tf.placeholder(tf.float32, [])
accuracy_summary = tf.summary.scalar("Accuracy", accuracy_placeholder)
summary_writer = tf.summary.FileWriter(LOG_DIR, tf.get_default_graph())
with tf.Session() as sess:
sess.run([init_global, init_local])
accuracy = 0.0
epoch = 0
while accuracy < STOP_TRESHOLD:
def test():
parser = ArgumentParser()
parser.add_argument(
"--dataset_path",
type=str,
default="../data/Time Dataset.json",
help="Path or url of the dataset. If empty download from S3.")
parser.add_argument(
"--check_point",
type=str,
default='../checkpoint/Oct30_21-01-28/checkpoint_mymodel_8.pth',
help="Path or url of the dataset cache")
parser.add_argument("--batch_size",
type=int,
default=100,
help="Batch size for validation")
parser.add_argument("--embedding_dim",
type=int,
default=100,
help="Batch size for validation")
parser.add_argument("--hidden_dim",
type=int,
default=100,
help="Batch size for validation")
parser.add_argument("--device",
type=str,
default="cuda" if torch.cuda.is_available() else "cpu",
help="Device (cuda or cpu)")
parser.add_argument("--train_precent",
type=float,
default=0.7,
help="Batch size for validation")
args = parser.parse_args()
device = torch.device(args.device)
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
logdir = os.path.join('../logs', current_time + '_' + socket.gethostname())
train_data_loader, valid_data_loader, input_lengths, target_lengths = get_data_loaders(
args.dataset_path, args.batch_size, args.train_precent)
encoder = Encoder(input_lengths + 1, args.embedding_dim, args.hidden_dim)
decoder = Decoder(target_lengths + 1, args.embedding_dim, args.hidden_dim)
model = Seq2Seq(encoder, decoder, device).to(device)
check_point = torch.load(args.check_point)
model.load_state_dict(check_point)
model.eval()
pairs = json.load(open('../data/Time Dataset.json', 'rt',
encoding='utf-8'))
data = array(pairs)
src_texts = data[:, 0]
trg_texts = data[:, 1]
src_c2ix, src_ix2c = build_vocab(src_texts)
trg_c2ix, trg_ix2c = build_vocab(trg_texts)
def get_decode(src):
result = []
for t in src:
result.append(src_ix2c[t])
sndx = 0
if '^' in result:
sndx = result.index('^') + 1
endx = result.index('$')
return ''.join(result[sndx:endx])
def get_decode_target(target):
result = []
for t in target:
result.append(trg_ix2c[int(t)])
sndx = 0
if '^' == result[0]:
sndx = result.index('^') + 1
endx = result.index('$')
return ''.join(result[sndx:endx])
max_src_len = max(list(map(len, src_texts))) + 2
max_trg_len = max(list(map(len, trg_texts))) + 2
max_src_len, max_trg_len
for batch in valid_data_loader:
src_seqs = batch[0].transpose(0, 1).to(device)
src_lengths = batch[1].to(device)
trg_seqs = batch[2].transpose(0, 1).to(device)
outputs, attn_weights = model.predict(src_seqs=src_seqs,
src_lengths=src_lengths)
# print(outputs.cpu().detach().numpy())
outputs_index = torch.argmax(outputs.cpu(), dim=2)
outputs_index_mat = outputs_index.permute(1, 0)
for i in range(outputs_index_mat.shape[0]):
print('src: \t',
get_decode(src_seqs.cpu().permute(1, 0)[i].numpy()))
print('target :\t',
get_decode_target(trg_seqs.cpu().permute(1, 0)[i].numpy()))
print('predict:\t',
get_decode_target(outputs_index_mat[i].detach().numpy()[1:]))
print('=' * 64)
def main():
parser = argparse.ArgumentParser(
description='process user given parameters')
parser.register('type', 'bool', str2bool)
parser.add_argument('--random_seed', type=float, default=42)
parser.add_argument('--teacher_forcing_ratio', type=float, default=0.5)
parser.add_argument('--max_decode_len', type=int, default=100)
parser.add_argument('--att_method', type=str, default='concat')
parser.add_argument('--seq_model', type=str, default='simple')
# I/O parameters
parser.add_argument('--train_dir', type=str, default='./eng-fra.txt')
parser.add_argument(
'--word_embed_file',
type=str,
default=
'/home/wang.9215/medical_phrases/pretrain_embeddings/glove.6B.300d.txt'
)
parser.add_argument('--save_best',
type='bool',
default=False,
help='save model in the best epoch or not')
parser.add_argument('--save_dir',
type=str,
default='./saved_models',
help='save model in the best epoch or not')
parser.add_argument('--save_interval',
type=int,
default=5,
help='intervals for saving models')
# model parameters
parser.add_argument('--use_pretrain_embed', type='bool', default=False)
parser.add_argument('--word_dim', type=int, default=256)
parser.add_argument('--lstm_hidden_dim', type=int, default=256)
# optim parameters
parser.add_argument('--batch_size', type=int, default=64)
parser.add_argument('--num_epochs',
type=int,
default=1000,
help="number of epochs for training")
parser.add_argument('--log_interval',
type=int,
default=100,
help='step interval for log')
parser.add_argument('--test_interval',
type=int,
default=10,
help='epoch interval for testing')
parser.add_argument('--early_stop_epochs', type=int, default=10)
parser.add_argument('--learning_rate', type=float, default=0.0001)
parser.add_argument('--min_epochs',
type=int,
default=50,
help='minimum number of epochs')
parser.add_argument('--clip_grad', type=float, default=5.0)
parser.add_argument('--lr_decay',
type=float,
default=0.05,
help='decay ratio of learning rate')
parser.add_argument('--metric', type=str, default='map', help='mrr or map')
parser.add_argument('--dropout', type=float, default=0.5)
args = parser.parse_args()
print('args: ', args)
print('********Key parameters:******')
print('Use GPU? {0}'.format(torch.cuda.is_available()))
# print('Model Parameters: ')
print('*****************************')
np.random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
# load the original data
data, src_vocab, tgt_vocab = loader.txt_loader(args.train_dir)
_, src_word2id, src_id2word = loader.raw_word_mapping(src_vocab)
args.src_vocab_size = len(src_word2id) + 1
args.src_word2id = src_word2id
args.src_id2word = src_id2word
_, tgt_word2id, tgt_id2word = loader.raw_word_mapping(tgt_vocab)
args.tgt_vocab_size = len(tgt_word2id) + 1
args.tgt_word2id = tgt_word2id
args.tgt_id2word = tgt_id2word
np.random.shuffle(data)
train_ratio = 0.8
train_data = data[:int(len(data) * train_ratio)]
test_data = data[int(len(data) * train_ratio)::]
# convert original data to digits
train_idx_data = loader.make_idx_data(train_data, args)
test_idx_data = loader.make_idx_data(test_data, args)
print(train_idx_data[0])
# global parameters
args.cuda = torch.cuda.is_available()
args.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# build model, initialize parameters
# model = Seq2Seq.SimpleSeq2Seq(args).to(args.device)
# model = Seq2Seq.ContextSeq2Seq(args).to(args.device)
# model = Seq2Seq.BahdanauAttSeq2Seq(args).to(args.device)
model = Seq2Seq.LuongAttSeq2Seq(args).to(args.device)
# if args.cuda:
# model = model.cuda()
print(model)
# print([name for name, p in model.named_parameters()])
criterion = nn.CrossEntropyLoss(ignore_index=config.PAD_WORD)
optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
train_loss = 0
train_logits = []
train_labels = []
num_batches = len(train_idx_data) // args.batch_size
print('Begin trainning...')
for epoch in range(args.num_epochs):
model.train()
steps = 0
np.random.shuffle(train_idx_data)
for i in range(num_batches):
train_batch = train_idx_data[i * args.batch_size:(i + 1) *
args.batch_size]
if i == num_batches - 1:
train_batch = train_idx_data[i * args.batch_size::]
list_xs, labels = loader.batch_processing(train_batch, args)
labels = torch.tensor(labels, dtype=torch.long,
device=args.device) # (B * L)
list_xs = [torch.tensor(x, device=args.device) for x in list_xs]
# if args.cuda:
# labels = labels.cuda()
# list_xs = [x.cuda() for x in list_xs]
optimizer.zero_grad()
logits = model(list_xs, args.teacher_forcing_ratio) # (B * L) * V
# loss = utils.masked_cross_entropy(logits, labels)
loss = criterion(logits.reshape(-1, args.tgt_vocab_size),
labels.reshape(-1))
train_loss += loss
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.clip_grad)
optimizer.step()
if args.cuda:
logits = logits.to('cpu').detach().data.numpy()
labels = labels.to('cpu').detach().data.numpy()
# logits = logits.cpu().detach().data.numpy()
# labels = labels.cpu().detach().data.numpy()
else:
logits = logits.detach().data.numpy()
labels = labels.detach().data.numpy()
# print(logits.shape)
train_logits.append(logits)
train_labels.append(labels)
# evaluation
steps += 1
if steps % args.log_interval == 0:
# cur_logits = np.concatenate(train_logits, axis=0)
# cur_labels = np.concatenate(train_labels, axis=0)
cur_train_bleu = utils.cal_bleu_score(train_logits,
train_labels,
config.EOS_WORD)
print(
"Epoch-{0}, steps-{1}: Train Loss - {2:.5}, Train BLEU - {3:.5}"
.format(epoch, steps, train_loss / len(train_batch),
cur_train_bleu))
train_loss = 0
train_logits = []
train_labels = []
# utils.adjust_learning_rate(optimizer, args.learning_rate / (1 + (epoch + 1) * args.lr_decay))
if epoch == 0: continue
if epoch % args.test_interval == 0:
test_bleu = train_utils.dataset_eval(model, test_idx_data, args)
print("Epoch-{0}: Test BLEU: {1:.5}".format(epoch, test_bleu))
# test_case = np.random.choice(test_idx_data)
idx = np.random.randint(0, len(test_idx_data))
print(' '.join(test_data[idx][0]))
print(' '.join(test_data[idx][1]))
decode_ids, decode_words = model.seq_decoding(test_data[idx][0])
# decode_words = train_utils.instance_eval(model, test_data[idx][0], test_data[idx][1], args, seq_decode=True)
print(decode_words)
# if epoch % args.test_interval == 0:
# # if args.logging:
# # args.log_name = '../logs/deep_predition_val_logs_{0}.txt'.format(epoch)
# all_dev = train_utils.evaluation(dev_idx, model, criterion, args)
# print("Epoch-{0}: All Dev {1}: {2:.5}".format(epoch, args.metric.upper(), all_dev))
#
# if all_dev > best_on_dev:
# print(datetime.now().strftime("%m/%d/%Y %X"))
# best_on_dev = all_dev
# last_epoch = epoch
#
# all_iv = train_utils.evaluation(iv_test_idx, model, criterion, args)
# print("--- Testing: All IV Test {0}: {1:.5}".format(args.metric.upper(), all_iv))
#
# if args.save_best:
# utils.save(model, args.save_dir, 'best', epoch)
#
# else:
# if epoch - last_epoch > args.early_stop_epochs and epoch > args.min_epochs:
# print('Early stop at {0} epoch.'.format(epoch))
# break
#
# elif epoch % args.save_interval == 0:
# utils.save(model, args.save_dir, 'snapshot', epoch)
return
enc_h_dim = 512
dec_h_dim = 512
e_dim = 256
dropout = 0.5
if att == True:
attention = Seq2SeqBA.Attention(enc_h_dim, dec_h_dim)
encoder = Seq2SeqBA.Encoder(i_dim, e_dim, enc_h_dim, dec_h_dim, dropout)
decoder = Seq2SeqBA.Decoder(o_dim, e_dim, enc_h_dim, dec_h_dim, dropout,
attention)
model = Seq2SeqBA.BahdanauS2S(encoder, decoder, device).to(device)
model_name = "S2SBA.pt"
else:
encoder = Seq2Seq.Encoder(i_dim,
e_dim,
enc_h_dim,
n_layers=2,
dropout=dropout)
decoder = Seq2Seq.Decoder(o_dim,
e_dim,
dec_h_dim,
n_layers=2,
dropout=dropout)
model = Seq2Seq.Seq2Seq(encoder, decoder, device).to(device)
model_name = "S2S.pt"
print("Initialize weights")
model.apply(initialize_weights)
optimizer = optim.Adam(model.parameters(), lr=lr)
target_pad_idx = en_field.vocab.stoi[en_field.pad_token]
abb_losses = [] #keeps track of loss values for ABB words
training_losses = [] #keeps track of loss on training data each run
make_ABB = lambda x: x[:1] + [1.0] + x[1 + 1:] #Adds semantic information
make_ABA = lambda x: x[:1] + [-1.0] + x[1 + 1:] #Adds semantic information
for rep in range(REPS):
print("Rep: " + str(rep))
#Erase the previous model:
keras.backend.clear_session()
#Build the new model:
model = Seq2Seq.seq2seq(input_dim=FEAT_NUM,
hidden_dim=FEAT_NUM,
output_length=2,
output_dim=FEAT_NUM,
batch_size=1,
learn_rate=0.001,
layer_type="lstm",
dropout=DROPOUT)
#PRETRAINING
if VOCAB_SIZE > 0:
print("Simulating real-life experience of infants...Rep=" + str(rep))
irl_X = []
irl_Y = []
for word in range(VOCAB_SIZE):
##Putting reduplication in training:
if np.random.rand() < REDUP_IN_PT:
syll_alpha = choice(all_sylls)
template = choice(["ABB", "ABA"])
if template == "ABB":
#Shuffle training data:
indexes = list(range(len(ordered_X)))
shuffle(indexes)
X = np.array([ordered_X[i] for i in indexes])
Y = np.array([ordered_Y[i] for i in indexes])
#Create the model object:
#model = Seq2Seq(input_dim=FEAT_NUM, hidden_dim=FEAT_NUM*3, output_length=3, output_dim=FEAT_NUM, depth=2)
#Build the new model:
model = Seq2Seq.seq2seq(
input_dim=FEAT_NUM,
hidden_dim=FEAT_NUM * 3,
output_length=3,
output_dim=FEAT_NUM,
batch_size=1,
learn_rate=0.005,
layer_type="lstm",
)
this_curve = []
for ep in range(EPOCHS):
#Train the model one epoch at a time,
#so we can give it a forced-choice task at each step:
print("Epoch: " + str(ep))
hist = model.train(X, Y, epoch_num=1, print_every=2)
this_curve.append(hist["Loss"][-1])
for trial_type in accuracies.keys():
corr_loss = np.square(
np.subtract(
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--resume_ret', action='store_true')
parser.add_argument('--fourthofdata', action='store_true')
parser.add_argument('--halfdata', action='store_true')
parser.add_argument('--threefourthsofdata', action='store_true')
opt = parser.parse_args()
train_data, valid_data, test_data = split_data(opt)
ret_enc = RNNEncoder(ret_INPUT_DIM, ret_ENC_EMB_DIM, ret_HID_DIM,
ret_N_LAYERS, ret_ENC_DROPOUT)
ret_dec = RNNDecoder(ret_OUTPUT_DIM, ret_DEC_EMB_DIM, ret_HID_DIM,
ret_N_LAYERS, ret_DEC_DROPOUT)
model = Seq2Seq(ret_enc, ret_dec, cuda_device).to(cuda_device)
print('The model has {0:9d} trainable parameters'.format(
count_parameters(model)))
optimizer = optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss()
if not os.path.isdir('models'):
os.makedirs('models')
enc_train_vect, enc_valid_vect = train_valid_model(model=model,
train_data=train_data,
valid_data=valid_data,
optimizer=optimizer,
criterion=criterion)
enc_test_vect = test_model(model=model,
test_data=test_data,
criterion=criterion)
train_ann = create_annoy_index("AttnEncAttnDecTrain", enc_train_vect)
valid_ann = create_annoy_index("AttnEncAttnDecValid", enc_valid_vect)
test_ann = create_annoy_index("AttnEncAttnDecTest", enc_test_vect)
wordlist2comment_dict = pickle.load(open("wordlist2comment.pickle", "rb"))
word2idcommentvocab_dict = pickle.load(
open("word2idcommentvocab.pickle", "rb"))
sim_train_data = torch.zeros_like(train_data)
sim_valid_data = torch.zeros_like(valid_data)
sim_test_data = torch.zeros_like(test_data)
for training_sample_id in range(train_data.shape[0]):
training_sample_comment = train_data[
training_sample_id][:max_comment_len]
training_sample_code = train_data[training_sample_id][max_comment_len +
1:]
annoy_vect = train_ann.get_item_vector(training_sample_id)
sim_vect_id = train_ann.get_nns_by_vector(annoy_vect, 1)
if sim_vect_id == training_sample_id:
print("Same id for training vect and similar vect")
exit(0)
sim_train_data[training_sample_id] = train_data[sim_vect_id]
for valid_sample_id in range(valid_data.shape[0]):
valid_sample_comment = valid_data[valid_sample_id][:max_comment_len]
valid_sample_code = valid_data[valid_sample_id][max_comment_len + 1:]
annoy_vect = valid_ann.get_item_vector(valid_sample_id)
sim_vect_id = train_ann.get_nns_by_vector(annoy_vect, 1)
if sim_vect_id == valid_sample_id:
print("Same id for training vect and similar vect")
exit(0)
sim_valid_data[valid_sample_id] = train_data[sim_vect_id]
for test_sample_id in range(test_data.shape[0]):
test_sample_comment = test_data[test_sample_id][:max_comment_len]
test_sample_code = test_data[test_sample_id][max_comment_len + 1:]
annoy_vect = test_ann.get_item_vector(test_sample_id)
sim_vect_id = train_ann.get_nns_by_vector(annoy_vect, 1)
if sim_vect_id == test_sample_id:
print("Same id for training vect and similar vect")
exit(0)
sim_test_data[test_sample_id] = train_data[sim_vect_id]
output_test_vect_reference = test_data[:, max_comment_len:]
output_test_vect_candidates = sim_test_data[:, max_comment_len:]
token_dict = pickle.load(open("codevocab.pickle", "rb"))
all_refs = []
all_cands = []
all_bleu_scores = []
for j in range(test_data.shape[0]):
ref = []
cand = []
for i in range(max_code_len):
ref_el = output_test_vect_reference[j][i].item()
cand_el = output_test_vect_candidates[j][i].item()
if ref_el > 0:
if ref_el in token_dict:
ref += [token_dict[ref_el]]
if cand_el in token_dict:
cand += [token_dict[cand_el]]
bleu = sentence_bleu([ref], cand)
all_bleu_scores += [bleu]
all_refs += [ref]
all_cands += [cand]
bleu_eval = {}
bleu_eval["scores"] = all_bleu_scores
bleu_eval["references"] = all_refs
bleu_eval["candidates"] = all_cands
print("Average BLEU score is ",
sum(all_bleu_scores) / len(all_bleu_scores))
pickle.dump(bleu_eval, open("results/bleu_evaluation_results.pickle",
"wb"))
def make_distance_matrix(vectors):
nb_vector = len(vectors)
out = np.zeros([nb_vector, nb_vector])
for i, v1 in enumerate(vectors):
for j, v2 in enumerate(vectors):
out[i][j] = np.linalg.norm(v1 - v2)
return out
if __name__ == "__main__":
batch, vocab_size = convert_to_batch(SENTENCES)
seq2seq = Seq2Seq(vocab_size, NB_HIDDEN, NB_HIDDEN, LEARNING_RATE)
init_global = tf.global_variables_initializer()
init_local = tf.local_variables_initializer()
loss_placeholder = tf.placeholder(tf.float32, [])
loss_summary = tf.summary.scalar("Loss", loss_placeholder)
accuracy_placeholder = tf.placeholder(tf.float32, [])
accuracy_summary = tf.summary.scalar("Accuracy", accuracy_placeholder)
summary_writer = tf.summary.FileWriter(LOG_DIR, tf.get_default_graph())
with tf.Session() as sess:
sess.run([init_global, init_local])
accuracy = 0.0
epoch = 0
while accuracy < STOP_TRESHOLD:
elif SCOPE == "segment":
feat_num, withheld_syll, syllables = novel_seg_data()
elif SCOPE == "syllable":
feat_num, withheld_syll, syllables = novel_syll_data()
else:
raise Exception(
"Wrong scope! Must be from the set {feature, segment, syllable}.")
X = np.array(syllables)
Y = np.array([syll + syll for syll in syllables])
#Build the model:
model = Seq2Seq.seq2seq(input_dim=feat_num,
hidden_dim=feat_num * 3,
output_length=Y.shape[1],
output_dim=Y.shape[2],
batch_size=1,
learn_rate=0.001,
layer_type="lstm",
dropout=DROPOUT)
#Train the model:
hist = model.train(X, Y, epoch_num=EPOCHS, print_every=10)
learning_curves.append(hist["Loss"])
#Test the model on trained data:
trained_IN = np.tile(X[0], (1, 1, 1))
trained_OUT = np.tile(Y[0], (1, 1, 1))
train_pred = model.predict(trained_IN)
#Test the model on withheld data:
withheld_IN = np.tile(np.array(withheld_syll), (1, 1, 1))
dim_y = len(w2i)
dim_tag = len(t2i)
num_sents = batch_size
print "#features = ", dim_x, "#labels = ", dim_y
print "#tag len = ", dim_tag
print "load test data..."
test_batch = 1
test_data_x_y = get_data.test_processing_long(r"data/post-test.txt", r"data/post-tag-test.txt", i2w, w2i, i2t, t2i, 100, test_batch)
reference_dic = cPickle.load(open(r'print_bleu_score/reference_dic.pkl', 'rb'))
print "done."
print "compiling..."
model = Seq2Seq(dim_x + dim_tag, dim_y + dim_tag, dim_y, dim_tag, hidden_size_encoder, hidden_size_decoder, cell, optimizer, drop_rate, num_sents)
# # load_error_model("GRU-200_best.model", model)
print "training..."
start = time.time()
g_error = 0.5
for i in xrange(10000):
error = 0.0
in_start = time.time()
for get_num_start in xrange((full_data_len/read_data_batch)+1):
read_data_batch_error = 0.0
in_b_start = time.time()
get_num_end = get_num_start*read_data_batch + read_data_batch
if get_num_end > full_data_len:
def train_model():
train_noisy_Id, train_noisy_len, train_clean_Id, train_clean_len, train_answer_Id, train_answer_len, test_noisy_Id, test_noisy_len, test_clean_Id, test_clean_len, test_answer_Id, test_answer_len, eval_noisy_Id, eval_noisy_len, eval_clean_Id, eval_clean_len, eval_answer_Id, eval_answer_len, vocab_size = load_data(
)
max_answer_length = np.asarray(train_answer_Id).shape[1]
max_target_length = np.asarray(train_clean_Id).shape[1]
max_source_length = np.asarray(train_noisy_Id).shape[1]
print "trian answer Lstm model"
an_Lstm = Answer_LSTM.answer_lstm(batch_size, max_answer_length,
vocab_size, embedding_size, num_units,
None, None, None, None)
an_Lstm.build_graph()
saver = tf.train.Saver(sharded=False)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# saver.restore(sess, "../Seq_ckpt/pretrain-model")
for batch in range(max_batches):
answer_shuffle, ans_len = next_batch_Lstm(train_answer_Id,
train_answer_len,
batch_size)
fd = {
an_Lstm.answer_inputs: answer_shuffle,
an_Lstm.answer_inputs_length: ans_len
}
l, _ = sess.run([an_Lstm.loss_answer, an_Lstm.train_lstm], fd)
if batch == 0 or batch % batches_in_epoch == 0:
print('batch {}'.format(batch))
answer_shuffle, ans_len = next_batch_Lstm(
eval_answer_Id, eval_answer_len, batch_size)
fd_eval = {
an_Lstm.answer_inputs: answer_shuffle,
an_Lstm.answer_inputs_length: ans_len
}
print(' minibatch loss: {}'.format(
sess.run(an_Lstm.loss_answer, fd_eval)))
saver.save(sess, "../Seq_ckpt/pretrain-lstm")
print "trian Seq2seq model"
Seq2Seq_model = Seq2Seq.Seq2Seq(batch_size, max_source_length,
max_target_length, vocab_size,
embedding_size, num_units, None, None,
None, None, None, None, None)
Seq2Seq_model.build_graph()
saver = tf.train.Saver(sharded=False)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# saver.restore(sess, "../Seq_ckpt/pretrain-model")
for batch in range(max_batches):
source_shuffle, source_len, target_shuffle, target_len = next_batch_Seq2seq(
train_noisy_Id, train_noisy_len, train_clean_Id,
train_clean_len, batch_size)
fd = {
Seq2Seq_model.encoder_inputs: source_shuffle,
Seq2Seq_model.encoder_inputs_length: source_len,
Seq2Seq_model.decoder_targets: target_shuffle,
Seq2Seq_model.decoder_length: target_len
}
l, _ = sess.run(
[Seq2Seq_model.loss_seq2seq, Seq2Seq_model.train_op], fd)
if batch == 0 or batch % batches_in_epoch == 0:
print('batch {}'.format(batch))
source_shuffle, source_len, target_shuffle, target_len = next_batch_Seq2seq(
eval_noisy_Id, eval_noisy_len, eval_clean_Id,
eval_clean_len, batch_size)
fd_eval = {
Seq2Seq_model.encoder_inputs: source_shuffle,
Seq2Seq_model.encoder_inputs_length: source_len,
Seq2Seq_model.decoder_targets: target_shuffle,
Seq2Seq_model.decoder_length: target_len
}
print(' minibatch loss: {}'.format(
sess.run(Seq2Seq_model.loss_seq2seq, fd_eval)))
saver.save(sess, "../Seq_ckpt/pretrain-seq2seq")