def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) for name, param in model.named_parameters(): if param.requires_grad: print(name) if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_dev_f1 = -1 test_f1 = [] best_epoch = 0 # start training for idx in range(data.HP_iteration): epoch_start = time.time() print("Epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 total_perplexity_1 = 0 total_perplexity_2 = 0 total_loss_1 = 0 total_loss_2 = 0 total_loss_3 = 0 total_loss_4 = 0 random.shuffle(data.source_train_idx) random.shuffle(data.target_train_idx) random.shuffle(data.source_lm_idx) random.shuffle(data.target_lm_idx) model.train() model.zero_grad() batch_size_1 = data.HP_batch_size train_num_1 = len(data.source_train_idx) train_num_2 = len(data.target_train_idx) train_num_3 = len(data.source_lm_idx) train_num_4 = len(data.target_lm_idx) batch_num = train_num_1 // batch_size_1 + 1 batch_size_2 = train_num_2 // batch_num batch_size_3 = train_num_3 // batch_num batch_size_4 = train_num_4 // batch_num for batch_id in range(batch_num): instance_1 = data.source_train_idx[batch_id * batch_size_1: (batch_id + 1) * batch_size_1 if ((batch_id + 1) * batch_size_1) < train_num_1 else train_num_1] instance_2 = data.target_train_idx[batch_id * batch_size_2: (batch_id + 1) * batch_size_2 if ((batch_id + 1) * batch_size_2) < train_num_2 else train_num_2] instance_3 = data.source_lm_idx[batch_id * batch_size_3: (batch_id + 1) * batch_size_3 if ((batch_id + 1) * batch_size_3) < train_num_3 else train_num_3] instance_4 = data.target_lm_idx[batch_id * batch_size_4: (batch_id + 1) * batch_size_4 if ((batch_id + 1) * batch_size_4) < train_num_4 else train_num_4] if not instance_1 or not instance_2: continue # NER batch_word_1, batch_wordlen_1, batch_wordrecover_1, batch_char_1, batch_charlen_1, \ batch_charrecover_1, batch_label_1, lm_seq_tensor_1, mask_1 = batchify_with_label(instance_1, data.HP_gpu) batch_word_2, batch_wordlen_2, batch_wordrecover_2, batch_char_2, batch_charlen_2, \ batch_charrecover_2, batch_label_2, lm_seq_tensor_2, mask_2 = batchify_with_label(instance_2, data.HP_gpu) # LM batch_word_3, batch_wordlen_3, batch_wordrecover_3, batch_char_3, batch_charlen_3, \ batch_charrecover_3, batch_label_3, lm_seq_tensor_3, mask_3 = batchify_with_label(instance_3 + instance_1, data.HP_gpu) batch_word_4, batch_wordlen_4, batch_wordrecover_4, batch_char_4, batch_charlen_4, \ batch_charrecover_4, batch_label_4, lm_seq_tensor_4, mask_4 = batchify_with_label(instance_4 + instance_2, data.HP_gpu) batch_word = [batch_word_1, batch_word_2, batch_word_3, batch_word_4] batch_wordlen = [batch_wordlen_1, batch_wordlen_2, batch_wordlen_3, batch_wordlen_4] batch_char = [batch_char_1, batch_char_2, batch_char_3, batch_char_4] batch_charlen = [batch_charlen_1, batch_charlen_2, batch_charlen_3, batch_charlen_4] batch_charrecover = [batch_charrecover_1, batch_charrecover_2, batch_charrecover_3, batch_charrecover_4] batch_label = [batch_label_1, batch_label_2, batch_label_3, batch_label_4] lm_seq_tensor = [lm_seq_tensor_1, lm_seq_tensor_2, lm_seq_tensor_3, lm_seq_tensor_4] mask = [mask_1, mask_2, mask_3, mask_4] instance_count += 1 loss_ = [] perplexity_ = [] # source language model loss, perplexity, tag_seq = model.loss('model1', batch_word[2], batch_wordlen[2], batch_char[2], batch_charlen[2], batch_charrecover[2], batch_label[2], lm_seq_tensor[2], mask[2]) loss_.append(loss) perplexity_.append(perplexity) # source NER loss, perplexity, tag_seq = model.loss('model2', batch_word[0], batch_wordlen[0], batch_char[0], batch_charlen[0], batch_charrecover[0], batch_label[0], lm_seq_tensor[0], mask[0]) loss_.append(loss) # target language model loss, perplexity, tag_seq = model.loss('model3', batch_word[3], batch_wordlen[3], batch_char[3], batch_charlen[3], batch_charrecover[3], batch_label[3], lm_seq_tensor[3], mask[3]) loss_.append(loss) perplexity_.append(perplexity) loss = 0 model_num = len(loss_) for loss_id in range(model_num): loss += loss_[loss_id] loss.backward() optimizer.step() model.zero_grad() total_loss_1 += loss_[0].data[0] total_loss_2 += loss_[1].data[0] total_loss_3 += loss_[2].data[0] total_perplexity_1 += perplexity_[0].data[0] total_perplexity_2 += perplexity_[1].data[0] epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start source_lm_perplexity = math.exp(total_perplexity_1 / batch_num) target_lm_perplexity = math.exp(total_perplexity_2 / batch_num) print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % ( idx, epoch_cost, train_num_1 / epoch_cost, total_loss_2)) print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total perplexity: %.4f" % ( idx, epoch_cost, train_num_3 / epoch_cost, source_lm_perplexity)) print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % ( idx, epoch_cost, train_num_2 / epoch_cost, total_loss_4)) print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total perplexity: %.4f" % ( idx, epoch_cost, train_num_4 / epoch_cost, target_lm_perplexity)) if total_loss_1 > 1e8 or str(total_loss_1) == "nan" or total_loss_2 > 1e8 or str( total_loss_2) == "nan" or total_loss_3 > 1e8 or str(total_loss_3) == "nan" or total_loss_4 > 1e8 or str( total_loss_4) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) # dev-test speed, acc, p, r, f, _, _ = evaluate(data, model, "dev-test") test_f1.append(f[1]) dev_finish = time.time() dev_cost = dev_finish - epoch_finish current_score = f[0] print("Dev-Source: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % ( dev_cost, speed, acc[0], p[0], r[0], f[0])) print("Test-Target: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % ( dev_cost, speed, acc[1], p[1], r[1], f[1])) if current_score > best_dev_f1: best_epoch = idx print("Exceed previous best f score:", best_dev_f1) model_name = data.model_dir + ".model" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev_f1 = current_score if current_score > 0.72: print("change optim sgd:") optimizer = optim.SGD(model.parameters(), lr=0.015, momentum=data.HP_momentum, weight_decay=data.HP_l2) print("The best Source-domain dev f-score: %.4f, Target-domain f-score: %.4f" % (best_dev_f1, test_f1[best_epoch]))
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) pytorch_total_params = sum(p.numel() for p in model.parameters()) print("--------pytorch total params--------") print(pytorch_total_params) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_dev = -10 best_test = -10 best_epoch = -1 no_imprv_epoch = 0 # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) # print (self.train_Ids) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue # label_instance = [[i for i in range(0, data.label_alphabet_size + 1)] for _ in range(len(instance))] batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask, input_label_seq_tensor = batchify_with_label( instance, data.HP_gpu, data.label_alphabet_size) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, input_label_seq_tensor) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end % 500 == 0: # temp_time = time.time() # temp_cost = temp_time - temp_start # temp_start = temp_time # print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) if sample_loss > 1e8 or str(sample_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() if data.whether_clip_grad: from torch.nn.utils import clip_grad_norm clip_grad_norm(model.parameters(), data.clip_grad) optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) print("totalloss:", total_loss) if total_loss > 1e8 or str(total_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) # exit(1) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) # ## decode test speed, acc_test, p, r, f_test, _, _ = evaluate(data, model, "test") test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc_test, p, r, f_test)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc_test)) if current_score > best_dev: if data.seg: best_test = f_test print("Exceed previous best f score:", best_dev) else: best_test = acc_test print("Exceed previous best acc score:", best_dev) best_epoch = idx # model_name = data.model_dir +'.'+ str(idx) + ".model" # print("Save current best model in file:", model_name) # torch.save(model.state_dict(), model_name) best_dev = current_score no_imprv_epoch = 0 else: # early stop no_imprv_epoch += 1 if no_imprv_epoch >= 10: print("early stop") print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) break if data.seg: print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) else: print("Current best acc score in dev", best_dev) print("Current best acc score in test", best_test) gc.collect()
def train(data, meansentfeats = False): print("Training model...") data.show_data_summary() save_data_name = data.model_dir +".dset" data.save(save_data_name) model = SeqModel(data, meansentfeats = meansentfeats) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum,weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s"%(data.optimizer)) exit(1) best_dev = -10 # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" %(idx,data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num//batch_size+1 for batch_id in range(total_batch): start = batch_id*batch_size end = (batch_id+1)*batch_size if end >train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label(instance, data.HP_gpu) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss(batch_word,batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end%500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) if sample_loss > 1e8 or str(sample_loss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s"%(idx, epoch_cost, train_num/epoch_cost, total_loss)) print("totalloss:", total_loss) if total_loss > 1e8 or str(total_loss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) # continue speed, acc, p, r, f, _,_ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print("Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f"%(dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print("Exceed previous best f score:", best_dev) else: print("Exceed previous best acc score:", best_dev) model_name = data.model_dir +'.'+ str(idx) + ".model" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score # ## decode test speed, acc, p, r, f, _,_ = evaluate(data, model, "test") test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(test_cost, speed, acc, p, r, f)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f"%(test_cost, speed, acc)) gc.collect()
def train(data): print "Training model..." model = SeqModel(data) print "model:{}".format(model) if data.gpu: model.cuda() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.lr, momentum=data.momentum, weight_decay=data.l2) if data.use_mapping: optimizer_wc = optim.SGD(model.word_hidden.wordrep.w.parameters(), lr=data.lr, momentum=data.momentum, weight_decay=data.l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.lr, weight_decay=data.l2) if data.use_mapping: optimizer_wc = optim.Adam(model.word_hidden.wordrep.w.parameters(), lr=data.lr, weight_decay=data.l2) else: print("Optimizer illegal: %s , use sgd or adam." % data.optimizer) exit(0) best_dev = -10 best_dev_epoch = -1 best_test = -10 best_test_epoch = -1 # start training for idx in range(data.iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx + 1, data.iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.lr_decay, data.lr) if data.use_mapping: optimizer_wc = lr_decay(optimizer_wc, idx, data.lr_decay, data.lr) instance_count = 0 sample_id = 0 # sample_loss = 0 sample_mapping_loss = 0 total_loss = 0 total_mapping_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) # set model in train mode model.train() model.zero_grad() batch_size = data.batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, batch_trans, trans_seq_lengths, trans_seq_recover, mask = batchify_with_label( instance, data.gpu) instance_count += 1 loss, tag_seq, wc_loss = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, batch_trans, trans_seq_lengths, trans_seq_recover) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole #sample_loss += loss.data[0] sample_loss += loss.data.item() if data.use_mapping: sample_mapping_loss += wc_loss.data[0] #total_loss += loss.data[0] total_loss += loss.data.item() if data.use_mapping: total_mapping_loss += wc_loss.data[0] if batch_id % data.show_loss_per_batch == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time if data.use_mapping: print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) else: print( " Instance: %s; Time: %.2fs; loss: %.4f;mapping_loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, sample_mapping_loss, right_token, whole_token, (right_token + 0.) / whole_token)) sys.stdout.flush() sample_loss = 0 sample_mapping_loss = 0 if data.use_trans and data.use_mapping: for param in model.word_hidden.wordrep.w.parameters(): param.requires_grad = False loss.backward(retain_graph=True) if data.clip != None: torch.nn.utils.clip_grad_norm(model.parameters(), data.clip) optimizer.step() model.zero_grad() for param in model.word_hidden.wordrep.w.parameters(): param.requires_grad = True wc_loss.backward() optimizer_wc.step() model.zero_grad() else: loss.backward() # torch.nn.utils.clip_grad_norm(model.parameters(), data.clip) optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start if data.use_mapping: print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) else: print( " Instance: %s; Time: %.2fs; loss: %.4f;mapping_loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, sample_mapping_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start if data.use_mapping: print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s,total mapping loss: %s" % (idx + 1, epoch_cost, train_num / epoch_cost, total_loss, total_mapping_loss)) else: print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx + 1, epoch_cost, train_num / epoch_cost, total_loss)) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev", data.nbest) dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev if data.save_model: model_name = data.model_dir + data.state_training_name + '.' + str( current_score)[2:-1] print "Save current best model in file:", model_name torch.save(model.state_dict(), model_name) best_dev = current_score best_dev_epoch = idx # ## decode test speed, acc, p, r, f, _, _ = evaluate(data, model, "test", data.nbest) if f > best_test: best_test = f best_test_epoch = idx test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc, p, r, f)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc)) print('best_dev_score: %.4f, best_dev_epoch:%d' % (best_dev, best_dev_epoch)) print('best_test_score: %.4f, best_test_epoch:%d' % (best_test, best_test_epoch)) gc.collect()
def train(data): ''' initialize model and train ''' print('Training model...') data.show_data_summary() # save dset save_data_name = data.model_dir + '.dset' data.save(save_data_name) # save exportable model architecture (for deployment) data.save_export(data.model_dir + '.xpt') model = SeqModel(data) if data.optimizer.lower() == 'sgd': optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'adagrad': optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'adadelta': optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'rmsprop': optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'adam': optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print('Optimizer illegal: {}'.format(data.optimizer)) exit(1) best_dev = -10 ## start training for idx in range(data.iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch {}/{}".format(idx, data.iteration)) if data.optimizer.lower() == 'sgd': optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train mode model.train() model.zero_grad() batch_size = data.batch_size #batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] instance_texts = data.train_texts[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu, volatile_flag=False, label_flag=True) #print(batch_char.size()) #print(batch_char.max()) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.item() total_loss += loss.item() if end % 500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print(' Instance {}; Time {:.2}s; loss {:.4}; acc {}/{}={:.4}'. format(end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) if sample_loss > 1e8 or str(sample_loss) == 'nan': print( 'ERROR: LOSS EXPLOSION (>1e8) ! Please set adapted parameters and structure! EXIT ...' ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(' Instance {}; Time {:.2}s; loss {:.4}; acc {}/{}={:.4}'.format( end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( ' Epoch: {} training finished. Time: {:.2}s; speed: {:.2}st/s; total loss: {}' .format(idx, epoch_cost, train_num / epoch_cost, total_loss)) if total_loss > 1e8 or str(sample_loss) == 'nan': print( 'ERROR: LOSS EXPLOSION (>1e8) ! Please set adapted parameters and structure! EXIT ...' ) exit(1) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish # saving dev results json for model analysis dev_res = tuple((speed, acc, p, r, f)) if data.seg: current_score = f print( "Dev: time: {:.2}s, speed {:.2}st/s; acc: {:.4}, p: {:.4}, r: {:.4}, f: {:.4}" .format(dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: {:.2}s, speed {:.2}st/s; acc: {:.4}".format( dev_cost, speed, acc)) # decode test speed, acc, p, r, f, _, _ = evaluate(data, model, 'test') test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: {:.2}s, speed {:.2}st/s; acc: {:.4}, p: {:.4}, r: {:.4}, f: {:.4}" .format(dev_cost, speed, acc, p, r, f)) else: print("Test: time: {:.2}s, speed {:.2}st/s; acc: {:.4}".format( dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print('"Exceed previous best f score:', best_dev) else: print('"Exceed previous best acc score:', best_dev) model_name = data.model_dir + '.' + str(idx) + '.model' print('Save current best model in file:', model_name) torch.save(model.state_dict(), model_name) best_dev = current_score path2info = data.model_dir + '.infos' save_infos(data, dev_res, path2info) gc.collect() print('Training done!') return best_dev
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) for name, param in model.named_parameters(): if param.requires_grad: print(name) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_dev = -10 dev_f = [] test_f = [] perplexity_1 = [] perplexity_2 = [] best_epoch = 0 # data.HP_iteration = 1 LM_data = data.train_Ids_2 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 total_perplexity_1 = 0 total_perplexity_2 = 0 total_loss_1 = 0 total_loss_2 = 0 total_loss_3 = 0 total_loss_4 = 0 random.shuffle(data.train_Ids_1) random.shuffle(data.train_Ids_2) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 ###co-train for 4 models train_num_1 = len(data.train_Ids_1) train_num_2 = len(data.train_Ids_2) train_num_3 = len(LM_data) total_batch_1 = train_num_1 // batch_size + 1 batch_size_2 = train_num_2 // total_batch_1 l_batch_num_2 = train_num_2 - total_batch_1 * batch_size_2 start_2 = end_2 = 0 for batch_id in range(total_batch_1): start = batch_id * batch_size end = (batch_id + 1) * batch_size start_2 = end_2 if batch_id < l_batch_num_2: end_2 = start_2 + (batch_size_2 + 1) else: end_2 = start_2 + batch_size_2 if end > train_num_1: end = train_num_1 if end_2 > train_num_2: end_2 = train_num_2 instance_1 = data.train_Ids_1[start:end] instance_2 = data.train_Ids_2[start_2:end_2] if not instance_1 or not instance_2: continue #seq label batch_word_1, batch_features_1, batch_wordlen_1, batch_wordrecover_1, batch_char_1, batch_charlen_1, batch_charrecover_1, batch_label_1, lm_seq_tensor_1, mask_1 = batchify_with_label( instance_1, data.HP_gpu) batch_word_2, batch_features_2, batch_wordlen_2, batch_wordrecover_2, batch_char_2, batch_charlen_2, batch_charrecover_2, batch_label_2, lm_seq_tensor_2, mask_2 = batchify_with_label( instance_2, data.HP_gpu) batch_word = [batch_word_1, batch_word_2] batch_features = [batch_features_1, batch_features_2] batch_wordlen = [batch_wordlen_1, batch_wordlen_2] batch_char = [batch_char_1, batch_char_2] batch_charlen = [batch_charlen_1, batch_charlen_2] batch_charrecover = [batch_charrecover_1, batch_charrecover_2] batch_label = [batch_label_1, batch_label_2] lm_seq_tensor = [lm_seq_tensor_1, lm_seq_tensor_2] mask = [mask_1, mask_2] instance_count += 1 loss_ = [] perplexity_ = [] # LM 1 loss, perplexity, tag_seq_forward, tag_seq_backward, tag_seq = model.loss( 'model1', batch_word[0], batch_features[0], batch_wordlen[0], batch_char[0], batch_charlen[0], batch_charrecover[0], batch_label[0], lm_seq_tensor[0], mask[0]) loss_.append(loss) perplexity_.append(perplexity) #seq label 1 loss, perplexity, tag_seq_forward, tag_seq_backward, tag_seq = model.loss( 'model2', batch_word[0], batch_features[0], batch_wordlen[0], batch_char[0], batch_charlen[0], batch_charrecover[0], batch_label[0], lm_seq_tensor[0], mask[0]) loss_.append(loss) # LM 2 loss, perplexity, tag_seq_forward, tag_seq_backward, tag_seq = model.loss( 'model3', batch_word[1], batch_features[1], batch_wordlen[1], batch_char[1], batch_charlen[1], batch_charrecover[1], batch_label[1], lm_seq_tensor[1], mask[1]) loss_.append(loss) perplexity_.append(perplexity) #seq label 2 loss, perplexity, tag_seq_forward, tag_seq_backward, tag_seq = model.loss( 'model4', batch_word[1], batch_features[1], batch_wordlen[1], batch_char[1], batch_charlen[1], batch_charrecover[1], batch_label[1], lm_seq_tensor[1], mask[1]) loss_.append(loss) loss_rate = [1.0, 1.0, 1.0, 2.0] loss = 0 model_num = len(loss_) for loss_id in range(model_num): loss += loss_[loss_id] * loss_rate[loss_id] loss.backward() optimizer.step() model.zero_grad() total_loss_1 += loss_[0].data[0] total_loss_2 += loss_[1].data[0] total_loss_3 += loss_[2].data[0] total_loss_4 += loss_[3].data[0] total_perplexity_1 += perplexity_[0].data[0] total_perplexity_2 += perplexity_[1].data[0] epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start LM_perplex_1 = math.exp(total_perplexity_1 / total_batch_1) LM_perplex_2 = math.exp(total_perplexity_2 / total_batch_1) perplexity_1.append(LM_perplex_1) perplexity_2.append(LM_perplex_2) print("Epoch: %s training finished. Time: %.2fs" % (idx, epoch_cost)) print("Epoch: %s training finished. Time: %.2fs, total loss: %s" % (idx, epoch_cost, total_loss_2)) print("totalloss:", total_loss_2) print( "Epoch: %s training finished. Time: %.2fs, total perplexity: %.4f" % (idx, epoch_cost, LM_perplex_1)) print("Epoch: %s training finished. Time: %.2fs, total loss: %s" % (idx, epoch_cost, total_loss_4)) print("totalloss:", total_loss_4) print( "Epoch: %s training finished. Time: %.2fs, total perplexity: %.4f" % (idx, epoch_cost, LM_perplex_2)) speed, acc, p, r, f, _, _ = evaluate(data, model, "dev") dev_f.append(f[1]) dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f[1] print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc[0], p[0], r[0], f[0])) print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc[1], p[1], r[1], f[1])) else: current_score = acc[1] print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc[0])) print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc[1])) if current_score > best_dev: best_epoch = idx if data.seg: print("Exceed previous best f score:", best_dev) else: print("Exceed previous best acc score:", best_dev) # model_name = data.model_dir +'.'+ str(idx) + ".model" model_name = data.model_dir + ".model" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score # ## decode test speed, acc, p, r, f, _, _ = evaluate(data, model, "test") test_f.append(f[1]) test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc[0], p[0], r[0], f[0])) print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc[1], p[1], r[1], f[1])) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc[0])) print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc[1])) gc.collect() print("the best dev score is in epoch %s, dev:%.4f, test:%.4f" % (best_epoch, dev_f[best_epoch], test_f[best_epoch])) with open('data/fscore_13PC.txt', 'w') as ft: ft.write('dev f scores:\n') for t in dev_f: ft.write(str(round(t, 6))) ft.write(' ') ft.write('\n') ft.write('test f scores:\n') for t in test_f: ft.write(str(round(t, 6))) ft.write(' ') ft.write('\n') ft.write('LM 1 perplexity:\n') for t in perplexity_1: ft.write(str(round(t, 6))) ft.write(' ') ft.write('\n') ft.write('LM 2 perplexity:\n') for t in perplexity_2: ft.write(str(round(t, 6))) ft.write(' ') if data.task_emb_save_dir is not None: with open('data/task_emb.txt', 'w') as ft: for task, i in data.task_alphabet.iteritems(): ft.write(task) ft.write(' ') for t in model.word_hidden.LSTM_param_generator.task_emb_vocab.weight.data[ i]: ft.write(str(round(t, 6))) ft.write(' ') ft.write('\n') if data.domain_emb_save_dir is not None: with open('data/domain_emb.txt', 'w') as fd: for domain, i in data.domain_alphabet.iteritems(): fd.write(domain) fd.write(' ') for t in model.word_hidden.LSTM_param_generator.domain_emb_vocab.weight.data[ i]: fd.write(str(round(t, 6))) fd.write(' ') fd.write('\n')
def pipeline(data, ner_dir, re_dir): seq_model = SeqModel(data) seq_wordseq = WordSequence(data, False, True, True, data.use_char) classify_wordseq = WordSequence(data, True, False, True, False) classify_model = ClassifyModel(data) if torch.cuda.is_available(): classify_model = classify_model.cuda(data.HP_gpu) iter_parameter = itertools.chain( *map(list, [seq_wordseq.parameters(), seq_model.parameters()])) seq_optimizer = optim.Adam(iter_parameter, lr=opt.ner_lr, weight_decay=data.HP_l2) iter_parameter = itertools.chain(*map( list, [classify_wordseq.parameters(), classify_model.parameters()])) classify_optimizer = optim.Adam(iter_parameter, lr=opt.re_lr, weight_decay=data.HP_l2) if data.tune_wordemb == False: my_utils.freeze_net(seq_wordseq.wordrep.word_embedding) my_utils.freeze_net(classify_wordseq.wordrep.word_embedding) re_X_positive = [] re_Y_positive = [] re_X_negative = [] re_Y_negative = [] relation_vocab = data.re_feature_alphabets[ data.re_feature_name2id['[RELATION]']] my_collate = my_utils.sorted_collate1 for i in range(len(data.re_train_X)): x = data.re_train_X[i] y = data.re_train_Y[i] if y != relation_vocab.get_index("</unk>"): re_X_positive.append(x) re_Y_positive.append(y) else: re_X_negative.append(x) re_Y_negative.append(y) re_test_loader = DataLoader(my_utils.RelationDataset( data.re_test_X, data.re_test_Y), data.HP_batch_size, shuffle=False, collate_fn=my_collate) best_ner_score = -1 best_re_score = -1 for idx in range(data.HP_iteration): epoch_start = time.time() seq_wordseq.train() seq_wordseq.zero_grad() seq_model.train() seq_model.zero_grad() classify_wordseq.train() classify_wordseq.zero_grad() classify_model.train() classify_model.zero_grad() batch_size = data.HP_batch_size random.shuffle(data.train_Ids) ner_train_num = len(data.train_Ids) ner_total_batch = ner_train_num // batch_size + 1 re_train_loader, re_train_iter = makeRelationDataset( re_X_positive, re_Y_positive, re_X_negative, re_Y_negative, data.unk_ratio, True, my_collate, data.HP_batch_size) re_total_batch = len(re_train_loader) total_batch = max(ner_total_batch, re_total_batch) min_batch = min(ner_total_batch, re_total_batch) for batch_id in range(total_batch): if batch_id < ner_total_batch: start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > ner_train_num: end = ner_train_num instance = data.train_Ids[start:end] batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask, \ batch_permute_label = batchify_with_label(instance, data.HP_gpu) hidden = seq_wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) hidden_adv = None loss, tag_seq = seq_model.neg_log_likelihood_loss( hidden, hidden_adv, batch_label, mask) loss.backward() seq_optimizer.step() seq_wordseq.zero_grad() seq_model.zero_grad() if batch_id < re_total_batch: [batch_word, batch_features, batch_wordlen, batch_wordrecover, \ batch_char, batch_charlen, batch_charrecover, \ position1_seq_tensor, position2_seq_tensor, e1_token, e1_length, e2_token, e2_length, e1_type, e2_type, \ tok_num_betw, et_num], [targets, targets_permute] = my_utils.endless_get_next_batch_without_rebatch1( re_train_loader, re_train_iter) hidden = classify_wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, position1_seq_tensor, position2_seq_tensor) hidden_adv = None loss, pred = classify_model.neg_log_likelihood_loss( hidden, hidden_adv, batch_wordlen, e1_token, e1_length, e2_token, e2_length, e1_type, e2_type, tok_num_betw, et_num, targets) loss.backward() classify_optimizer.step() classify_wordseq.zero_grad() classify_model.zero_grad() epoch_finish = time.time() print("epoch: %s training finished. Time: %.2fs" % (idx, epoch_finish - epoch_start)) # _, _, _, _, f, _, _ = ner.evaluate(data, seq_wordseq, seq_model, "test") ner_score = ner.evaluate1(data, seq_wordseq, seq_model, "test") print("ner evaluate: f: %.4f" % (ner_score)) re_score = relation_extraction.evaluate(classify_wordseq, classify_model, re_test_loader) print("re evaluate: f: %.4f" % (re_score)) if ner_score + re_score > best_ner_score + best_re_score: print("new best score: ner: %.4f , re: %.4f" % (ner_score, re_score)) best_ner_score = ner_score best_re_score = re_score torch.save(seq_wordseq.state_dict(), os.path.join(ner_dir, 'wordseq.pkl')) torch.save(seq_model.state_dict(), os.path.join(ner_dir, 'model.pkl')) torch.save(classify_wordseq.state_dict(), os.path.join(re_dir, 'wordseq.pkl')) torch.save(classify_model.state_dict(), os.path.join(re_dir, 'model.pkl'))
def train(data): print "Training model..." data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(0) best_dev = -10 for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 sample_loss = {idtask: 0 for idtask in range(data.HP_tasks)} right_token = {idtask: 0 for idtask in range(data.HP_tasks)} whole_token = {idtask: 0 for idtask in range(data.HP_tasks)} random.shuffle(data.train_Ids) model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu, inference=False) instance_count += 1 loss, losses, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, inference=False) for idtask in range(data.HP_tasks): right, whole = predict_check(tag_seq[idtask], batch_label[idtask], mask) sample_loss[idtask] += losses[idtask].data[0] right_token[idtask] += right whole_token[idtask] += whole if end % 500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print( " Instance: %s; Task %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, idtask, temp_cost, sample_loss[idtask], right_token[idtask], whole_token[idtask], (right_token[idtask] + 0.) / whole_token[idtask])) if sample_loss[idtask] > 1e8 or str(sample_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) sys.stdout.flush() sample_loss[idtask] = 0 if end % 500 == 0: print "--------------------------------------------------------------------------" total_loss += loss.data[0] loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start for idtask in range(data.HP_tasks): print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss[idtask], right_token[idtask], whole_token[idtask], (right_token[idtask] + 0.) / whole_token[idtask])) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) print "totalloss:", total_loss if total_loss > 1e8 or str(total_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) summary = evaluate(data, model, "dev", False, False) dev_finish = time.time() dev_cost = dev_finish - epoch_finish current_scores = [] for idtask in xrange(0, data.HP_tasks): speed, acc, p, r, f, pred_labels, _ = summary[idtask] if data.seg: current_score = f current_scores.append(f) print( "Task %d Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (idtask, dev_cost, speed, acc, p, r, f)) else: current_score = acc current_scores.append(acc) print("Task %d Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (idtask, dev_cost, speed, acc)) pred_results_tasks = [] pred_scores_tasks = [] for idtask in xrange(data.HP_tasks): speed, acc, p, r, f, pred_results, pred_scores = summary[idtask] pred_results_tasks.append(pred_results) pred_scores_tasks.append(pred_scores_tasks) # EVALUATING ON DEV SET FOR CHOOSING THE BEST MODEL # MULTITASK LEARNING OF BOTH CONSTITUENCY AND DEPENDENCY PARSING if data.dependency_parsing and data.constituency_parsing: # CONSTITUENCY PARSING with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') # transform between @ and {} rebuild.rebuild_tree(data.decode_dir, decoded_st_dir) else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') # evaluate the output comparing to the gold command = [ "PYTHONPATH=" + data.cons2label, "python", data.evaluate, " --input ", decoded_st_dir, " --gold ", data.gold_dev_cons, " --evalb ", data.evalb, ">", f_decode_mt.name ] os.system(" ".join(command)) current_score_cons = float([ l for l in f_decode_mt.read().split("\n") if l.startswith("Bracketing FMeasure") ][0].split("=")[1]) print(current_score_cons) # DEPENDENCY PARSING with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') else: print("else") if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') output_nn = open(data.decode_dir) tmp = tempfile.NamedTemporaryFile().name decodeDependencies.decode(output_nn, tmp, data.language) current_score_depen = float( decodeDependencies.evaluateDependencies( data.gold_dev_dep, tmp)) print(current_score_depen) # SINGLE OR MULTITASK CONSTITUENCY PARSING elif data.constituency_parsing: with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') # transform between @ and {} rebuild.rebuild_tree(data.decode_dir, decoded_st_dir) else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') command = [ "PYTHONPATH=" + data.cons2label, "python", data.evaluate, " --input ", decoded_st_dir, " --gold ", data.gold_dev_cons, " --evalb ", data.evalb, ">", f_decode_mt.name ] os.system(" ".join(command)) current_score = float([ l for l in f_decode_mt.read().split("\n") if l.startswith("Bracketing FMeasure") ][0].split("=")[1]) print "Current Score (from EVALB)", current_score, "Previous best dev (from EVALB)", best_dev # SINGLE OR MULTITASK DEPENDENCY PARSING elif data.dependency_parsing: with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: # If we are learning multiple task we move it as a sequence # labeling if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') output_nn = open(data.decode_dir) tmp = tempfile.NamedTemporaryFile().name decodeDependencies.decode(output_nn, tmp, data.language) current_score = decodeDependencies.evaluateDependencies( data.gold_dev_dep, tmp) print "Current Score (from LAS)", current_score, "Previous best dev (from LAS)", best_dev else: current_score = sum(current_scores) / len(current_scores) print "Current Score", current_score, "Previous best dev", best_dev # SAVE THE BEST MODEL # by default save model with highest harmonic mean when parsing both # dependency and constituency trees if data.dependency_parsing and data.constituency_parsing: harmonic_mean = (2 * current_score_cons * current_score_depen) / \ (current_score_cons + current_score_depen) if harmonic_mean > best_dev: print("New harmonic mean " + repr(harmonic_mean)) print("Exceed previous best harmonic mean score: " + repr(best_dev) + " LAS " + repr(current_score_depen) + " F1 " + repr(current_score_cons)) model_name = data.model_dir + ".model" print "Overwritting model to", model_name torch.save(model.state_dict(), model_name) best_dev = harmonic_mean else: print("sofar the best " + repr(best_dev)) else: if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev model_name = data.model_dir + ".model" print "Overwritting model to", model_name torch.save(model.state_dict(), model_name) best_dev = current_score else: print("sofar the best " + repr(best_dev)) summary = evaluate(data, model, "test", False) test_finish = time.time() test_cost = test_finish - dev_finish for idtask in xrange(0, data.HP_tasks): speed, acc, p, r, f, _, _ = summary[idtask] if data.seg: current_score = f print( "Task %d Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (idtask, test_cost, speed, acc, p, r, f)) else: current_score = acc print("Task %d Test: time: %.2fs speed: %.2fst/s; acc: %.4f" % (idtask, test_cost, speed, acc)) gc.collect()
def train(data, model_file): print "Training model..." model = SeqModel(data) wordseq = WordSequence(data, False, True, data.use_char) if opt.self_adv == 'grad': wordseq_adv = WordSequence(data, False, True, data.use_char) elif opt.self_adv == 'label': wordseq_adv = WordSequence(data, False, True, data.use_char) model_adv = SeqModel(data) else: wordseq_adv = None if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": if opt.self_adv == 'grad': iter_parameter = itertools.chain(*map(list, [wordseq.parameters(), wordseq_adv.parameters(), model.parameters()])) optimizer = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) elif opt.self_adv == 'label': iter_parameter = itertools.chain(*map(list, [wordseq.parameters(), model.parameters()])) optimizer = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) iter_parameter = itertools.chain(*map(list, [wordseq_adv.parameters(), model_adv.parameters()])) optimizer_adv = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) else: iter_parameter = itertools.chain(*map(list, [wordseq.parameters(), model.parameters()])) optimizer = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(0) best_dev = -10 if data.tune_wordemb == False: my_utils.freeze_net(wordseq.wordrep.word_embedding) if opt.self_adv != 'no': my_utils.freeze_net(wordseq_adv.wordrep.word_embedding) # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model wordseq.train() wordseq.zero_grad() if opt.self_adv == 'grad': wordseq_adv.train() wordseq_adv.zero_grad() elif opt.self_adv == 'label': wordseq_adv.train() wordseq_adv.zero_grad() model_adv.train() model_adv.zero_grad() model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask,\ batch_permute_label = batchify_with_label(instance, data.HP_gpu) instance_count += 1 if opt.self_adv == 'grad': hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen,batch_charrecover, None, None) hidden_adv = wordseq_adv.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) loss, tag_seq = model.neg_log_likelihood_loss(hidden, hidden_adv, batch_label, mask) loss.backward() my_utils.reverse_grad(wordseq_adv) optimizer.step() wordseq.zero_grad() wordseq_adv.zero_grad() model.zero_grad() elif opt.self_adv == 'label' : wordseq.unfreeze_net() wordseq_adv.freeze_net() hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen,batch_charrecover, None, None) hidden_adv = wordseq_adv.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) loss, tag_seq = model.neg_log_likelihood_loss(hidden, hidden_adv, batch_label, mask) loss.backward() optimizer.step() wordseq.zero_grad() wordseq_adv.zero_grad() model.zero_grad() wordseq.freeze_net() wordseq_adv.unfreeze_net() hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen,batch_charrecover, None, None) hidden_adv = wordseq_adv.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) loss_adv, _ = model_adv.neg_log_likelihood_loss(hidden, hidden_adv, batch_permute_label, mask) loss_adv.backward() optimizer_adv.step() wordseq.zero_grad() wordseq_adv.zero_grad() model_adv.zero_grad() else: hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) hidden_adv = None loss, tag_seq = model.neg_log_likelihood_loss(hidden, hidden_adv, batch_label, mask) loss.backward() optimizer.step() wordseq.zero_grad() model.zero_grad() # right, whole = predict_check(tag_seq, batch_label, mask) # right_token += right # whole_token += whole sample_loss += loss.data.item() total_loss += loss.data.item() if end % 500 == 0: # temp_time = time.time() # temp_cost = temp_time - temp_start # temp_start = temp_time # print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % ( # end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) if sample_loss > 1e8 or str(sample_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) sys.stdout.flush() sample_loss = 0 # temp_time = time.time() # temp_cost = temp_time - temp_start # print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % ( # end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print("epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % ( idx, epoch_cost, train_num / epoch_cost, total_loss)) print "totalloss:", total_loss if total_loss > 1e8 or str(total_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) # continue speed, acc, p, r, f, _, _ = evaluate(data, wordseq, model, "test") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print("Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % ( dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev torch.save(wordseq.state_dict(), os.path.join(model_file, 'wordseq.pkl')) if opt.self_adv == 'grad': torch.save(wordseq_adv.state_dict(), os.path.join(model_file, 'wordseq_adv.pkl')) elif opt.self_adv == 'label': torch.save(wordseq_adv.state_dict(), os.path.join(model_file, 'wordseq_adv.pkl')) torch.save(model_adv.state_dict(), os.path.join(model_file, 'model_adv.pkl')) model_name = os.path.join(model_file, 'model.pkl') torch.save(model.state_dict(), model_name) best_dev = current_score gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir +".dset" data.save(save_data_name) model = SeqModel(data) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum,weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s"%(data.optimizer)) exit(0) best_dev = -10 # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" %(idx,data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num//batch_size+1 for batch_id in range(total_batch): start = batch_id*batch_size end = (batch_id+1)*batch_size if end >train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label(instance, data.HP_gpu) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss(batch_word,batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end%500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s"%(idx, epoch_cost, train_num/epoch_cost, total_loss)) # continue speed, acc, p, r, f, _,_ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print("Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f"%(dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print("Exceed previous best f score:", best_dev) else: print("Exceed previous best acc score:", best_dev) model_name = data.model_dir +'.'+ str(idx) + ".model" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score # ## decode test speed, acc, p, r, f, _,_ = evaluate(data, model, "test") test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(test_cost, speed, acc, p, r, f)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f"%(test_cost, speed, acc)) gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + "/data.dset" if data.save_model: data.save(save_data_name) batch_size = data.HP_batch_size train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 model = SeqModel(data) pytorch_total_params = sum(p.numel() for p in model.parameters()) print(model) print("pytorch total params: %d" % pytorch_total_params) ## model 1 optimizer lr_detail1 = [ { "params": filter(lambda p: p.requires_grad, model.mcmodel.parameters()), "lr": data.HP_lr }, ] if data.optimizer.lower() == "sgd": optimizer = optim.SGD(lr_detail1, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(lr_detail1, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(lr_detail1, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(lr_detail1, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(lr_detail1, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) ## model 2 optimizer optimizer2 = AdamW(model.get_m2_params(), lr=data.HP_lr2, weight_decay=data.HP_l2) t_total = total_batch * data.HP_iteration warmup_step = int(data.warmup_step * t_total) scheduler2 = WarmupLinearSchedule(optimizer2, warmup_step, t_total) best_dev = -10 best_test = -10 max_test = -10 max_test_epoch = -1 max_dev_epoch = -1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() print("\n ###### Epoch: %s/%s ######" % (idx, data.HP_iteration)) # print (self.train_Ids) if data.optimizer.lower() == "sgd": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) sample_loss = 0 total_loss = 0 random.shuffle(data.train_Ids) model.train() model.zero_grad() for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu, True) loss, tag_seqs = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) sample_loss += loss.item() total_loss += loss.item() if end % 500 == 0: if sample_loss > 1e8 or str(sample_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() clip_grad_norm_(model.parameters(), data.clip_grad) optimizer.step() optimizer2.step() scheduler2.step() model.zero_grad() epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2f s, speed: %.2f doc/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) if total_loss > 1e8 or str(total_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) # dev dev_score, _ = evaluate(data, model, "dev") # test test_score, _ = evaluate(data, model, "test") if max_test < test_score: max_test_epoch = idx max_test = max(test_score, max_test) if dev_score > best_dev: print("Exceed previous best dev score") best_test = test_score best_dev = dev_score max_dev_epoch = idx if data.save_model: model_name = data.model_dir + "/best_model.ckpt" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) print( "Score summary: max dev (%d): %.4f, test: %.4f; max test (%d): %.4f" % (max_dev_epoch, best_dev, best_test, max_test_epoch, max_test)) gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" # 存储data数据 data.save(save_data_name) model = SeqModel(data) # check to load pretrained model if data.use_crf: pretrain_model_path = os.path.join('model_snapshot', 'lan_crf.model') else: pretrain_model_path = os.path.join('model_snapshot', 'lan.model') if data.use_pre_trained_model and os.path.exists(pretrain_model_path): model.load_state_dict(torch.load(pretrain_model_path)) print("load pretrained model success:%s" % pretrain_model_path) pytorch_total_params = sum(p.numel() for p in model.parameters()) print("--------pytorch total params--------") print(pytorch_total_params) optimizer = None if data.optimizer.lower() == "sgd": optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_dev = -10 best_test = -10 no_imprv_epoch = 0 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) # print (self.train_Ids) # every 5 epoch decay learning rate if idx % 5 == 0: optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 total_loss = 0 ## set model in train model model.train() model.zero_grad() start = 0 end = start + data.HP_batch_size train_epochs = [] while end <= len(data.train_Ids): train_epochs.append((start, end)) start = end end = end + data.HP_batch_size if end > len(data.train_Ids) > start: train_epochs.append((start, len(data.train_Ids))) for sample_id, (start, end) in enumerate(train_epochs): instance = data.train_Ids[start: end] sample_loss = 0 batch_word, batch_word_len, _, batch_word_recover, batch_label, mask, input_label_seq_tensor = batchify_with_label( instance, data.HP_gpu, data) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_word_len, batch_label, mask, input_label_seq_tensor) sample_loss += loss.item() total_loss += loss.item() print("Epoch:%s,no_imprv_epoch:%s,Instance: %s" % ( idx, no_imprv_epoch, sample_id)) right, whole = predict_check(tag_seq, batch_label, mask, data.use_crf) print(" loss: %.4f, acc: %s/%s=%.4f" % ( loss.item(), right, whole, (right + 0.) / whole * 100)) if sample_loss > 1e8 or str(sample_loss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) sys.stdout.flush() loss.backward() if data.whether_clip_grad: nn.utils.clip_grad_norm_(model.parameters(), data.clip_grad) optimizer.step() model.zero_grad() # break epoch_finish = time.time() if total_loss > 1e8 or str(total_loss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) speed, acc, report, f_value, \ ner_acc, ner_p, ner_r, ner_f = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f_value # current_score = sent_f1 print("Dev: time: %.2fs, speed: %.2fst/s;\n" "acc: %.4f, f_value: %.4f\n" "ner_acc: %.4f, ner_p: %.4f, ner_r: %.4f, ner_f: %.4f\n" "current f1:%.4f" % ( dev_cost, speed, acc, f_value, ner_acc, ner_p, ner_r, ner_f, current_score )) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % ( dev_cost, speed, acc)) # ## decode test speed, acc, report, f_value, \ ner_acc, ner_p, ner_r, ner_f = evaluate(data, model, "test") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: print("Test: time: %.2fs, speed: %.2fst/s;\n" "acc: %.4f, f_value: %.4f\n" "ner_acc: %.4f, ner_p: %.4f, ner_r: %.4f, ner_f: %.4f\n" "current f1:%.4f" % ( dev_cost, speed, acc, f_value, ner_acc, ner_p, ner_r, ner_f, current_score )) else: print("Test: time: %.2fs speed: %.2fst/s; acc: %.4f" % ( dev_cost, speed, acc)) if current_score > best_dev: if data.seg: best_test = f_value # best_test = sent_f1 print("Exceed previous best avg f score:", best_dev) else: best_test = acc print("Exceed previous best acc score:", best_dev) if data.use_crf: result_file = "result_crf.txt" model_name = data.model_dir + "_crf.model" else: result_file = "result.txt" model_name = data.model_dir + ".model" with open(result_file, 'w', encoding='utf-8') as w: w.write( "Save current best model in file:%s, iteration:%s/%s, best_test_f_score:%.5f\n" "ner:\n" " precision:%.5f, recall:%.5f, f1_score:%.5f\n" "%s\n\n" % ( model_name, idx, data.HP_iteration, best_test, ner_p, ner_r, ner_f, report)) print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score no_imprv_epoch = 0 else: # early stop no_imprv_epoch += 1 if no_imprv_epoch >= 10: print("early stop") print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) break if data.seg: print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) else: print("Current best acc score in dev", best_dev) print("Current best acc score in test", best_test) gc.collect()