def train(data): print "Training model..." data.show_data_summary() save_data_name = data.model_dir +".dset" data.save(save_data_name) model = SeqModel(data) if data.pretrained_model is not None: model_dict = model.state_dict() #We load the weights for the layers that we have pretrained (e.g. for language modeling) pretrained_dict = torch.load(data.pretrained_model) pretrained_dict = {k: v for k, v in pretrained_dict.items() if data.pretrained_part == data.PRETRAINED_ALL or (data.pretrained_part == data.PRETRAINED_LSTMS and "hidden2tagList" not in k)} # We overwrite entries in the existing state dict model_dict.update(pretrained_dict) # We load the new state dict model.load_state_dict(model_dict) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": #optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum) optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum,weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s"%(data.optimizer)) exit(0) best_dev = -10 range_valid_tasks = range(data.HP_tasks) for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" %(idx,data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 sample_loss = {idtask: 0 for idtask in range(data.HP_tasks)} right_token = {idtask: 0 for idtask in range(data.HP_tasks)} whole_token = {idtask: 0 for idtask in range(data.HP_tasks)} random.shuffle(data.train_Ids) #We get the indexes where are the samples of each (shuffled) treebank if data.disjoint: treebank_indexes = {} for idxsample, sample in enumerate(data.train_Ids): if sample[-1] not in treebank_indexes: treebank_indexes[sample[-1]] = [] treebank_indexes[sample[-1]].append(idxsample) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num//batch_size+1 if data.disjoint: tb_idxs = {tb:(0,batch_size) for tb in treebank_indexes} for batch_id in range(total_batch): start = batch_id*batch_size end = (batch_id+1)*batch_size if end >train_num: end = train_num if data.disjoint: eligible_treebanks = [t for t in treebank_indexes if tb_idxs[t][0] < len(treebank_indexes[t]) and idx < data.ignore_after_epoch[t] ] if eligible_treebanks == []: break tb = random.choice(eligible_treebanks) range_valid_tasks = data.dataset_ids[tb] idx_init, idx_end = tb_idxs[tb] train_idxs = treebank_indexes[tb][idx_init:idx_end] instance = [data.train_Ids[idx_ins] for idx_ins in train_idxs] #data.train_Ids[train_idxs] tb_idxs[tb] = (idx_end, idx_end+batch_size) else: instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label(instance, data.HP_gpu, inference=False) instance_count += 1 loss, losses, tag_seq = model.neg_log_likelihood_loss(batch_word,batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, range_valid_tasks, inference=False) log=True for idtask in range_valid_tasks: right, whole = predict_check(tag_seq[idtask], batch_label[idtask], mask) sample_loss[idtask]+= losses[idtask].data[0] right_token[idtask]+=right whole_token[idtask]+=whole if end%500 == 0 and log: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time log = False if sample_loss[idtask] > 1e8 or str(sample_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) sys.stdout.flush() for aux_idtask in range(data.HP_tasks): if whole_token[aux_idtask] == 0: print ("Task %d (no samples found)"%(aux_idtask)) else: if data.inv_dataset_ids[aux_idtask] in eligible_treebanks: print("Task %d %s Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(aux_idtask,data.inv_dataset_ids[aux_idtask],end, temp_cost, sample_loss[aux_idtask], right_token[aux_idtask], whole_token[aux_idtask],(right_token[aux_idtask]+0.)/whole_token[aux_idtask])) else: print("Task %d %s does not contain more samples; loss: %4f"%(aux_idtask,data.inv_dataset_ids[aux_idtask], losses[aux_idtask].data[0])) sample_loss[aux_idtask] = 0 total_loss += loss.data[0] loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start for aux_idtask in range(data.HP_tasks): if whole_token[aux_idtask] == 0: print ("Task %d (no samples found)"%(aux_idtask)) else: name_tb = data.inv_dataset_ids[aux_idtask] print("Task %d %s Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(aux_idtask,name_tb,len(treebank_indexes[name_tb]), temp_cost, sample_loss[aux_idtask], right_token[aux_idtask], whole_token[aux_idtask],(right_token[aux_idtask]+0.)/whole_token[aux_idtask])) sample_loss[aux_idtask] = 0 epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s"%(idx,epoch_cost, train_num/epoch_cost, total_loss)) if total_loss > 1e8 or str(total_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) summary = evaluate(data,model, "dev", False, False) dev_finish = time.time() dev_cost = dev_finish - epoch_finish current_scores = [] for idtask in xrange(0, data.HP_tasks): speed,acc,p,r,f,pred_labels,_,valid_indexes = summary[idtask] if data.seg: current_score = f current_scores.append(f) print("Task %d Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(idtask, dev_cost, speed, acc, p, r, f)) else: current_score = acc current_scores.append(acc) print("Task %d Dev: time: %.2fs speed: %.2fst/s; acc: %.4f"%(idtask, dev_cost, speed, acc)) pred_results_tasks = [] pred_scores_tasks = [] pred_las_tasks = [] valid_indexes = None for idtask in xrange(data.HP_tasks): speed, acc, p, r, f, pred_results, pred_scores, pred_indexes = summary[idtask] pred_results_tasks.append(pred_results) pred_scores_tasks.append(pred_scores_tasks) if idtask in data.task_metric and data.task_metric[idtask] == "LAS": pred_las_tasks.append(pred_results) valid_indexes = pred_indexes with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: # If we are learning multiple task we move it as a sequence labeling if data.HP_main_tasks > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_las_tasks, 'dev', indexes=valid_indexes) split_char = "{}" else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_las_tasks, 'dev', indexes=valid_indexes) split_char = "@" output_nn = open(data.decode_dir) tmp = tempfile.NamedTemporaryFile().name if data.offset: decode_dependencies.decode_combined_tasks(output_nn, tmp, split_char) else: print("decoding single task") decode_dependencies.decode(output_nn, tmp, split_char) current_score = decode_dependencies.evaluate_dependencies(data.gold_dev_dep, tmp) print "Current Score (from LAS)", current_score, "Previous best dev (from LAS)", best_dev if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev model_name = data.model_dir +".model" print "Overwritting model to", model_name torch.save(model.state_dict(), model_name) best_dev = current_score else: print("sofar the best "+repr(best_dev)) if data.HP_tasks_weight_decays: print "Updating the weights using linear weight decay. ", print "The old weights were", data.HP_tasks_weights, data.HP_tasks_weights =[max(weight-decay,0) for weight,decay in zip(data.HP_tasks_weights, data.HP_tasks_weight_decays)] print ". The new weights are", data.HP_tasks_weights model.set_tasks_weights(data.HP_tasks_weights) gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(0) best_dev = -10 # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end % 500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print("Exceed previous best f score:", best_dev) else: print("Exceed previous best acc score:", best_dev) model_name = data.model_dir + '.' + str(idx) + ".model" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score # ## decode test speed, acc, p, r, f, _, _ = evaluate(data, model, "test") test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc, p, r, f)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc)) gc.collect()
batch_word_documents = [train_word_documents[i] for i in batch_ids] batch_label_documents = [ train_label_documents[i] for i in batch_ids ] word_features = get_word_features(batch_word_documents, word_dic, config_dic.get("gpu")) #char_features = get_char_features(batch_word_documents, char_dic, config_dic.get("gpu")) sw_features_list = [] for sp_key, sp in sps.items(): sw_features_list.append( get_sw_features(batch_word_documents, sw_dicts[sp_key], sp, config_dic.get("gpu"))) label_features = get_label_features(batch_label_documents, label_dic, config_dic.get("gpu")) loss, train_tag_seq = seq_model.neg_log_likelihood_loss( word_features, None, sw_features_list, label_features) batch_ave_loss += loss.data total_loss += loss.data loss.backward() optimizer.step() seq_model.zero_grad() rt, tt = predict_check(train_tag_seq, label_features.get("label_ids"), word_features.get("masks")) right_token += rt total_token += tt if batch_i != 0 and batch_i % 50 == 0: if batch_ave_loss > 1e8 or str(loss) == "nan": print("Error: Loss Explosion (>1e8)! EXIT...")
def train(data, decode_data, args): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_score = -10 best_epoch = 0 for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end % 500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) if sample_loss > 1e8 or str(sample_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) print("totalloss:", total_loss) if total_loss > 1e8 or str(total_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) dev_finish = time.time() if data.eval_type == "CONLLU": lookup = processing.dump_into_lookup(data.dev_gold) # save the model model_name = data.model_dir + ".model" torch.save(model.state_dict(), model_name) # decode the saved model decode_data.load(decode_data.dset_dir) decode_data.read_config(args.decode) decode_data.generate_instance('raw') decode_results, pred_scores, probs, _, _ = load_model_decode( decode_data, 'raw') decode_data.write_decoded_results2(decode_results, probs, 'raw') test_finish = time.time() gc.collect() dev_enc = l.Encoding(data.encoding, data.postag_type) dict_encoded, all_sent, text = dev_enc.encode(data.dev_gold) processing.write_to_conllu(dict_encoded, data.dev_enc_dep2label, 0) diction, words = dev_enc.decode(decode_data.output_nn, data.encoding, all_sent) processing.write_to_conllu(diction, "final.conllu", text) #EVALUATE AND SAVE THE MODEL WITH THE HIGHEST UAS SCORE ON THE DEV SET if data.eval_type == "CONLLU": processing.merge_lookup("final.conllu", lookup) score = -10.0 if data.eval_type == 'CONLL': call( "perl eval07.pl -p -q -g " + data.dev_gold + " -s final.conllu | grep Unlabeled | cut -d ' ' -f 12 > score.txt", shell=True) with open('score.txt') as f: score = float(f.read()) elif data.eval_type == 'CONLLU': subparser = argparse.ArgumentParser() subparser.add_argument( "gold_file", type=str, help="Name of the CoNLL-U file with the gold data.") subparser.add_argument( "system_file", type=str, help="Name of the CoNLL-U file with the predicted data.") subparser.add_argument("--verbose", "-v", default=False, action="store_true", help="Print all metrics.") subparser.add_argument( "--counts", "-c", default=False, action="store_true", help= "Print raw counts of correct/gold/system/aligned words instead of prec/rec/F1 for all metrics." ) subargs = subparser.parse_args([data.dev_gold, "final.conllu"]) # Evaluate evaluation = conll18.evaluate_wrapper(subargs) score = 100 * evaluation["UAS"].f1 else: print('Invalid option for --eval-type') exit() if score > best_score: best_score = score print("Exceeds. New best score: " + repr(score)) best_epoch = idx best_model_name = data.model_dir + "_best.model" torch.save(model.state_dict(), best_model_name) print("Best score sofar " + repr(best_score)) print("Best epoch " + repr(best_epoch)) """
def train(data): ''' initialize model and train ''' print('Training model...') data.show_data_summary() # save dset save_data_name = data.model_dir + '.dset' data.save(save_data_name) # save exportable model architecture (for deployment) data.save_export(data.model_dir + '.xpt') model = SeqModel(data) if data.optimizer.lower() == 'sgd': optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'adagrad': optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'adadelta': optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'rmsprop': optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == 'adam': optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print('Optimizer illegal: {}'.format(data.optimizer)) exit(1) best_dev = -10 ## start training for idx in range(data.iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch {}/{}".format(idx, data.iteration)) if data.optimizer.lower() == 'sgd': optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train mode model.train() model.zero_grad() batch_size = data.batch_size #batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] instance_texts = data.train_texts[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu, volatile_flag=False, label_flag=True) #print(batch_char.size()) #print(batch_char.max()) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.item() total_loss += loss.item() if end % 500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print(' Instance {}; Time {:.2}s; loss {:.4}; acc {}/{}={:.4}'. format(end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) if sample_loss > 1e8 or str(sample_loss) == 'nan': print( 'ERROR: LOSS EXPLOSION (>1e8) ! Please set adapted parameters and structure! EXIT ...' ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(' Instance {}; Time {:.2}s; loss {:.4}; acc {}/{}={:.4}'.format( end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( ' Epoch: {} training finished. Time: {:.2}s; speed: {:.2}st/s; total loss: {}' .format(idx, epoch_cost, train_num / epoch_cost, total_loss)) if total_loss > 1e8 or str(sample_loss) == 'nan': print( 'ERROR: LOSS EXPLOSION (>1e8) ! Please set adapted parameters and structure! EXIT ...' ) exit(1) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish # saving dev results json for model analysis dev_res = tuple((speed, acc, p, r, f)) if data.seg: current_score = f print( "Dev: time: {:.2}s, speed {:.2}st/s; acc: {:.4}, p: {:.4}, r: {:.4}, f: {:.4}" .format(dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: {:.2}s, speed {:.2}st/s; acc: {:.4}".format( dev_cost, speed, acc)) # decode test speed, acc, p, r, f, _, _ = evaluate(data, model, 'test') test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: {:.2}s, speed {:.2}st/s; acc: {:.4}, p: {:.4}, r: {:.4}, f: {:.4}" .format(dev_cost, speed, acc, p, r, f)) else: print("Test: time: {:.2}s, speed {:.2}st/s; acc: {:.4}".format( dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print('"Exceed previous best f score:', best_dev) else: print('"Exceed previous best acc score:', best_dev) model_name = data.model_dir + '.' + str(idx) + '.model' print('Save current best model in file:', model_name) torch.save(model.state_dict(), model_name) best_dev = current_score path2info = data.model_dir + '.infos' save_infos(data, dev_res, path2info) gc.collect() print('Training done!') return best_dev
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) pytorch_total_params = sum(p.numel() for p in model.parameters()) print("--------pytorch total params--------") print(pytorch_total_params) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_dev = -10 best_test = -10 best_epoch = -1 no_imprv_epoch = 0 # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) # print (self.train_Ids) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue # label_instance = [[i for i in range(0, data.label_alphabet_size + 1)] for _ in range(len(instance))] batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask, input_label_seq_tensor = batchify_with_label( instance, data.HP_gpu, data.label_alphabet_size) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, input_label_seq_tensor) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end % 500 == 0: # temp_time = time.time() # temp_cost = temp_time - temp_start # temp_start = temp_time # print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) if sample_loss > 1e8 or str(sample_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() if data.whether_clip_grad: from torch.nn.utils import clip_grad_norm clip_grad_norm(model.parameters(), data.clip_grad) optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) print("totalloss:", total_loss) if total_loss > 1e8 or str(total_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) # exit(1) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) # ## decode test speed, acc_test, p, r, f_test, _, _ = evaluate(data, model, "test") test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc_test, p, r, f_test)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc_test)) if current_score > best_dev: if data.seg: best_test = f_test print("Exceed previous best f score:", best_dev) else: best_test = acc_test print("Exceed previous best acc score:", best_dev) best_epoch = idx # model_name = data.model_dir +'.'+ str(idx) + ".model" # print("Save current best model in file:", model_name) # torch.save(model.state_dict(), model_name) best_dev = current_score no_imprv_epoch = 0 else: # early stop no_imprv_epoch += 1 if no_imprv_epoch >= 10: print("early stop") print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) break if data.seg: print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) else: print("Current best acc score in dev", best_dev) print("Current best acc score in test", best_test) gc.collect()
def pipeline(data, ner_dir, re_dir): seq_model = SeqModel(data) seq_wordseq = WordSequence(data, False, True, True, data.use_char) classify_wordseq = WordSequence(data, True, False, True, False) classify_model = ClassifyModel(data) if torch.cuda.is_available(): classify_model = classify_model.cuda(data.HP_gpu) iter_parameter = itertools.chain( *map(list, [seq_wordseq.parameters(), seq_model.parameters()])) seq_optimizer = optim.Adam(iter_parameter, lr=opt.ner_lr, weight_decay=data.HP_l2) iter_parameter = itertools.chain(*map( list, [classify_wordseq.parameters(), classify_model.parameters()])) classify_optimizer = optim.Adam(iter_parameter, lr=opt.re_lr, weight_decay=data.HP_l2) if data.tune_wordemb == False: my_utils.freeze_net(seq_wordseq.wordrep.word_embedding) my_utils.freeze_net(classify_wordseq.wordrep.word_embedding) re_X_positive = [] re_Y_positive = [] re_X_negative = [] re_Y_negative = [] relation_vocab = data.re_feature_alphabets[ data.re_feature_name2id['[RELATION]']] my_collate = my_utils.sorted_collate1 for i in range(len(data.re_train_X)): x = data.re_train_X[i] y = data.re_train_Y[i] if y != relation_vocab.get_index("</unk>"): re_X_positive.append(x) re_Y_positive.append(y) else: re_X_negative.append(x) re_Y_negative.append(y) re_test_loader = DataLoader(my_utils.RelationDataset( data.re_test_X, data.re_test_Y), data.HP_batch_size, shuffle=False, collate_fn=my_collate) best_ner_score = -1 best_re_score = -1 for idx in range(data.HP_iteration): epoch_start = time.time() seq_wordseq.train() seq_wordseq.zero_grad() seq_model.train() seq_model.zero_grad() classify_wordseq.train() classify_wordseq.zero_grad() classify_model.train() classify_model.zero_grad() batch_size = data.HP_batch_size random.shuffle(data.train_Ids) ner_train_num = len(data.train_Ids) ner_total_batch = ner_train_num // batch_size + 1 re_train_loader, re_train_iter = makeRelationDataset( re_X_positive, re_Y_positive, re_X_negative, re_Y_negative, data.unk_ratio, True, my_collate, data.HP_batch_size) re_total_batch = len(re_train_loader) total_batch = max(ner_total_batch, re_total_batch) min_batch = min(ner_total_batch, re_total_batch) for batch_id in range(total_batch): if batch_id < ner_total_batch: start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > ner_train_num: end = ner_train_num instance = data.train_Ids[start:end] batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask, \ batch_permute_label = batchify_with_label(instance, data.HP_gpu) hidden = seq_wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) hidden_adv = None loss, tag_seq = seq_model.neg_log_likelihood_loss( hidden, hidden_adv, batch_label, mask) loss.backward() seq_optimizer.step() seq_wordseq.zero_grad() seq_model.zero_grad() if batch_id < re_total_batch: [batch_word, batch_features, batch_wordlen, batch_wordrecover, \ batch_char, batch_charlen, batch_charrecover, \ position1_seq_tensor, position2_seq_tensor, e1_token, e1_length, e2_token, e2_length, e1_type, e2_type, \ tok_num_betw, et_num], [targets, targets_permute] = my_utils.endless_get_next_batch_without_rebatch1( re_train_loader, re_train_iter) hidden = classify_wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, position1_seq_tensor, position2_seq_tensor) hidden_adv = None loss, pred = classify_model.neg_log_likelihood_loss( hidden, hidden_adv, batch_wordlen, e1_token, e1_length, e2_token, e2_length, e1_type, e2_type, tok_num_betw, et_num, targets) loss.backward() classify_optimizer.step() classify_wordseq.zero_grad() classify_model.zero_grad() epoch_finish = time.time() print("epoch: %s training finished. Time: %.2fs" % (idx, epoch_finish - epoch_start)) # _, _, _, _, f, _, _ = ner.evaluate(data, seq_wordseq, seq_model, "test") ner_score = ner.evaluate1(data, seq_wordseq, seq_model, "test") print("ner evaluate: f: %.4f" % (ner_score)) re_score = relation_extraction.evaluate(classify_wordseq, classify_model, re_test_loader) print("re evaluate: f: %.4f" % (re_score)) if ner_score + re_score > best_ner_score + best_re_score: print("new best score: ner: %.4f , re: %.4f" % (ner_score, re_score)) best_ner_score = ner_score best_re_score = re_score torch.save(seq_wordseq.state_dict(), os.path.join(ner_dir, 'wordseq.pkl')) torch.save(seq_model.state_dict(), os.path.join(ner_dir, 'model.pkl')) torch.save(classify_wordseq.state_dict(), os.path.join(re_dir, 'wordseq.pkl')) torch.save(classify_model.state_dict(), os.path.join(re_dir, 'model.pkl'))
def train(data): print "Training model..." model = SeqModel(data) print "model:{}".format(model) if data.gpu: model.cuda() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.lr, momentum=data.momentum, weight_decay=data.l2) if data.use_mapping: optimizer_wc = optim.SGD(model.word_hidden.wordrep.w.parameters(), lr=data.lr, momentum=data.momentum, weight_decay=data.l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.lr, weight_decay=data.l2) if data.use_mapping: optimizer_wc = optim.Adam(model.word_hidden.wordrep.w.parameters(), lr=data.lr, weight_decay=data.l2) else: print("Optimizer illegal: %s , use sgd or adam." % data.optimizer) exit(0) best_dev = -10 best_dev_epoch = -1 best_test = -10 best_test_epoch = -1 # start training for idx in range(data.iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx + 1, data.iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.lr_decay, data.lr) if data.use_mapping: optimizer_wc = lr_decay(optimizer_wc, idx, data.lr_decay, data.lr) instance_count = 0 sample_id = 0 # sample_loss = 0 sample_mapping_loss = 0 total_loss = 0 total_mapping_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) # set model in train mode model.train() model.zero_grad() batch_size = data.batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, batch_trans, trans_seq_lengths, trans_seq_recover, mask = batchify_with_label( instance, data.gpu) instance_count += 1 loss, tag_seq, wc_loss = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, batch_trans, trans_seq_lengths, trans_seq_recover) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole #sample_loss += loss.data[0] sample_loss += loss.data.item() if data.use_mapping: sample_mapping_loss += wc_loss.data[0] #total_loss += loss.data[0] total_loss += loss.data.item() if data.use_mapping: total_mapping_loss += wc_loss.data[0] if batch_id % data.show_loss_per_batch == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time if data.use_mapping: print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) else: print( " Instance: %s; Time: %.2fs; loss: %.4f;mapping_loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, sample_mapping_loss, right_token, whole_token, (right_token + 0.) / whole_token)) sys.stdout.flush() sample_loss = 0 sample_mapping_loss = 0 if data.use_trans and data.use_mapping: for param in model.word_hidden.wordrep.w.parameters(): param.requires_grad = False loss.backward(retain_graph=True) if data.clip != None: torch.nn.utils.clip_grad_norm(model.parameters(), data.clip) optimizer.step() model.zero_grad() for param in model.word_hidden.wordrep.w.parameters(): param.requires_grad = True wc_loss.backward() optimizer_wc.step() model.zero_grad() else: loss.backward() # torch.nn.utils.clip_grad_norm(model.parameters(), data.clip) optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start if data.use_mapping: print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) else: print( " Instance: %s; Time: %.2fs; loss: %.4f;mapping_loss: %.4f; acc: %s/%s=%.4f" % (batch_id, temp_cost, sample_loss, sample_mapping_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start if data.use_mapping: print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s,total mapping loss: %s" % (idx + 1, epoch_cost, train_num / epoch_cost, total_loss, total_mapping_loss)) else: print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx + 1, epoch_cost, train_num / epoch_cost, total_loss)) # continue speed, acc, p, r, f, _, _ = evaluate(data, model, "dev", data.nbest) dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print( "Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev if data.save_model: model_name = data.model_dir + data.state_training_name + '.' + str( current_score)[2:-1] print "Save current best model in file:", model_name torch.save(model.state_dict(), model_name) best_dev = current_score best_dev_epoch = idx # ## decode test speed, acc, p, r, f, _, _ = evaluate(data, model, "test", data.nbest) if f > best_test: best_test = f best_test_epoch = idx test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print( "Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (test_cost, speed, acc, p, r, f)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f" % (test_cost, speed, acc)) print('best_dev_score: %.4f, best_dev_epoch:%d' % (best_dev, best_dev_epoch)) print('best_test_score: %.4f, best_test_epoch:%d' % (best_test, best_test_epoch)) gc.collect()
def train(data): print "Training model..." data.show_data_summary() save_data_name = data.model_dir + ".dset" data.save(save_data_name) model = SeqModel(data) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(0) best_dev = -10 for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 sample_loss = {idtask: 0 for idtask in range(data.HP_tasks)} right_token = {idtask: 0 for idtask in range(data.HP_tasks)} whole_token = {idtask: 0 for idtask in range(data.HP_tasks)} random.shuffle(data.train_Ids) model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu, inference=False) instance_count += 1 loss, losses, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask, inference=False) for idtask in range(data.HP_tasks): right, whole = predict_check(tag_seq[idtask], batch_label[idtask], mask) sample_loss[idtask] += losses[idtask].data[0] right_token[idtask] += right whole_token[idtask] += whole if end % 500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print( " Instance: %s; Task %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, idtask, temp_cost, sample_loss[idtask], right_token[idtask], whole_token[idtask], (right_token[idtask] + 0.) / whole_token[idtask])) if sample_loss[idtask] > 1e8 or str(sample_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) sys.stdout.flush() sample_loss[idtask] = 0 if end % 500 == 0: print "--------------------------------------------------------------------------" total_loss += loss.data[0] loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start for idtask in range(data.HP_tasks): print( " Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % (end, temp_cost, sample_loss[idtask], right_token[idtask], whole_token[idtask], (right_token[idtask] + 0.) / whole_token[idtask])) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) print "totalloss:", total_loss if total_loss > 1e8 or str(total_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) summary = evaluate(data, model, "dev", False, False) dev_finish = time.time() dev_cost = dev_finish - epoch_finish current_scores = [] for idtask in xrange(0, data.HP_tasks): speed, acc, p, r, f, pred_labels, _ = summary[idtask] if data.seg: current_score = f current_scores.append(f) print( "Task %d Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (idtask, dev_cost, speed, acc, p, r, f)) else: current_score = acc current_scores.append(acc) print("Task %d Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (idtask, dev_cost, speed, acc)) pred_results_tasks = [] pred_scores_tasks = [] for idtask in xrange(data.HP_tasks): speed, acc, p, r, f, pred_results, pred_scores = summary[idtask] pred_results_tasks.append(pred_results) pred_scores_tasks.append(pred_scores_tasks) # EVALUATING ON DEV SET FOR CHOOSING THE BEST MODEL # MULTITASK LEARNING OF BOTH CONSTITUENCY AND DEPENDENCY PARSING if data.dependency_parsing and data.constituency_parsing: # CONSTITUENCY PARSING with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') # transform between @ and {} rebuild.rebuild_tree(data.decode_dir, decoded_st_dir) else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') # evaluate the output comparing to the gold command = [ "PYTHONPATH=" + data.cons2label, "python", data.evaluate, " --input ", decoded_st_dir, " --gold ", data.gold_dev_cons, " --evalb ", data.evalb, ">", f_decode_mt.name ] os.system(" ".join(command)) current_score_cons = float([ l for l in f_decode_mt.read().split("\n") if l.startswith("Bracketing FMeasure") ][0].split("=")[1]) print(current_score_cons) # DEPENDENCY PARSING with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') else: print("else") if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') output_nn = open(data.decode_dir) tmp = tempfile.NamedTemporaryFile().name decodeDependencies.decode(output_nn, tmp, data.language) current_score_depen = float( decodeDependencies.evaluateDependencies( data.gold_dev_dep, tmp)) print(current_score_depen) # SINGLE OR MULTITASK CONSTITUENCY PARSING elif data.constituency_parsing: with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') # transform between @ and {} rebuild.rebuild_tree(data.decode_dir, decoded_st_dir) else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') command = [ "PYTHONPATH=" + data.cons2label, "python", data.evaluate, " --input ", decoded_st_dir, " --gold ", data.gold_dev_cons, " --evalb ", data.evalb, ">", f_decode_mt.name ] os.system(" ".join(command)) current_score = float([ l for l in f_decode_mt.read().split("\n") if l.startswith("Bracketing FMeasure") ][0].split("=")[1]) print "Current Score (from EVALB)", current_score, "Previous best dev (from EVALB)", best_dev # SINGLE OR MULTITASK DEPENDENCY PARSING elif data.dependency_parsing: with tempfile.NamedTemporaryFile() as f_decode_mt: with tempfile.NamedTemporaryFile() as f_decode_st: # If we are learning multiple task we move it as a sequence # labeling if len(data.index_of_main_tasks) > 1: data.decode_dir = f_decode_mt.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') else: if data.decode_dir is None: data.decode_dir = f_decode_st.name decoded_st_dir = f_decode_st.name data.write_decoded_results(pred_results_tasks, 'dev') output_nn = open(data.decode_dir) tmp = tempfile.NamedTemporaryFile().name decodeDependencies.decode(output_nn, tmp, data.language) current_score = decodeDependencies.evaluateDependencies( data.gold_dev_dep, tmp) print "Current Score (from LAS)", current_score, "Previous best dev (from LAS)", best_dev else: current_score = sum(current_scores) / len(current_scores) print "Current Score", current_score, "Previous best dev", best_dev # SAVE THE BEST MODEL # by default save model with highest harmonic mean when parsing both # dependency and constituency trees if data.dependency_parsing and data.constituency_parsing: harmonic_mean = (2 * current_score_cons * current_score_depen) / \ (current_score_cons + current_score_depen) if harmonic_mean > best_dev: print("New harmonic mean " + repr(harmonic_mean)) print("Exceed previous best harmonic mean score: " + repr(best_dev) + " LAS " + repr(current_score_depen) + " F1 " + repr(current_score_cons)) model_name = data.model_dir + ".model" print "Overwritting model to", model_name torch.save(model.state_dict(), model_name) best_dev = harmonic_mean else: print("sofar the best " + repr(best_dev)) else: if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev model_name = data.model_dir + ".model" print "Overwritting model to", model_name torch.save(model.state_dict(), model_name) best_dev = current_score else: print("sofar the best " + repr(best_dev)) summary = evaluate(data, model, "test", False) test_finish = time.time() test_cost = test_finish - dev_finish for idtask in xrange(0, data.HP_tasks): speed, acc, p, r, f, _, _ = summary[idtask] if data.seg: current_score = f print( "Task %d Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % (idtask, test_cost, speed, acc, p, r, f)) else: current_score = acc print("Task %d Test: time: %.2fs speed: %.2fst/s; acc: %.4f" % (idtask, test_cost, speed, acc)) gc.collect()
def train(data, model_file): print "Training model..." model = SeqModel(data) wordseq = WordSequence(data, False, True, data.use_char) if opt.self_adv == 'grad': wordseq_adv = WordSequence(data, False, True, data.use_char) elif opt.self_adv == 'label': wordseq_adv = WordSequence(data, False, True, data.use_char) model_adv = SeqModel(data) else: wordseq_adv = None if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": if opt.self_adv == 'grad': iter_parameter = itertools.chain(*map(list, [wordseq.parameters(), wordseq_adv.parameters(), model.parameters()])) optimizer = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) elif opt.self_adv == 'label': iter_parameter = itertools.chain(*map(list, [wordseq.parameters(), model.parameters()])) optimizer = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) iter_parameter = itertools.chain(*map(list, [wordseq_adv.parameters(), model_adv.parameters()])) optimizer_adv = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) else: iter_parameter = itertools.chain(*map(list, [wordseq.parameters(), model.parameters()])) optimizer = optim.Adam(iter_parameter, lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(0) best_dev = -10 if data.tune_wordemb == False: my_utils.freeze_net(wordseq.wordrep.word_embedding) if opt.self_adv != 'no': my_utils.freeze_net(wordseq_adv.wordrep.word_embedding) # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("epoch: %s/%s" % (idx, data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model wordseq.train() wordseq.zero_grad() if opt.self_adv == 'grad': wordseq_adv.train() wordseq_adv.zero_grad() elif opt.self_adv == 'label': wordseq_adv.train() wordseq_adv.zero_grad() model_adv.train() model_adv.zero_grad() model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask,\ batch_permute_label = batchify_with_label(instance, data.HP_gpu) instance_count += 1 if opt.self_adv == 'grad': hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen,batch_charrecover, None, None) hidden_adv = wordseq_adv.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) loss, tag_seq = model.neg_log_likelihood_loss(hidden, hidden_adv, batch_label, mask) loss.backward() my_utils.reverse_grad(wordseq_adv) optimizer.step() wordseq.zero_grad() wordseq_adv.zero_grad() model.zero_grad() elif opt.self_adv == 'label' : wordseq.unfreeze_net() wordseq_adv.freeze_net() hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen,batch_charrecover, None, None) hidden_adv = wordseq_adv.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) loss, tag_seq = model.neg_log_likelihood_loss(hidden, hidden_adv, batch_label, mask) loss.backward() optimizer.step() wordseq.zero_grad() wordseq_adv.zero_grad() model.zero_grad() wordseq.freeze_net() wordseq_adv.unfreeze_net() hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen,batch_charrecover, None, None) hidden_adv = wordseq_adv.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) loss_adv, _ = model_adv.neg_log_likelihood_loss(hidden, hidden_adv, batch_permute_label, mask) loss_adv.backward() optimizer_adv.step() wordseq.zero_grad() wordseq_adv.zero_grad() model_adv.zero_grad() else: hidden = wordseq.forward(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, None, None) hidden_adv = None loss, tag_seq = model.neg_log_likelihood_loss(hidden, hidden_adv, batch_label, mask) loss.backward() optimizer.step() wordseq.zero_grad() model.zero_grad() # right, whole = predict_check(tag_seq, batch_label, mask) # right_token += right # whole_token += whole sample_loss += loss.data.item() total_loss += loss.data.item() if end % 500 == 0: # temp_time = time.time() # temp_cost = temp_time - temp_start # temp_start = temp_time # print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % ( # end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) if sample_loss > 1e8 or str(sample_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) sys.stdout.flush() sample_loss = 0 # temp_time = time.time() # temp_cost = temp_time - temp_start # print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f" % ( # end, temp_cost, sample_loss, right_token, whole_token, (right_token + 0.) / whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print("epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s" % ( idx, epoch_cost, train_num / epoch_cost, total_loss)) print "totalloss:", total_loss if total_loss > 1e8 or str(total_loss) == "nan": print "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." exit(0) # continue speed, acc, p, r, f, _, _ = evaluate(data, wordseq, model, "test") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print("Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f" % ( dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % (dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print "Exceed previous best f score:", best_dev else: print "Exceed previous best acc score:", best_dev torch.save(wordseq.state_dict(), os.path.join(model_file, 'wordseq.pkl')) if opt.self_adv == 'grad': torch.save(wordseq_adv.state_dict(), os.path.join(model_file, 'wordseq_adv.pkl')) elif opt.self_adv == 'label': torch.save(wordseq_adv.state_dict(), os.path.join(model_file, 'wordseq_adv.pkl')) torch.save(model_adv.state_dict(), os.path.join(model_file, 'model_adv.pkl')) model_name = os.path.join(model_file, 'model.pkl') torch.save(model.state_dict(), model_name) best_dev = current_score gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir +".dset" data.save(save_data_name) model = SeqModel(data) loss_function = nn.NLLLoss() if data.optimizer.lower() == "sgd": optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum,weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s"%(data.optimizer)) exit(0) best_dev = -10 # data.HP_iteration = 1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" %(idx,data.HP_iteration)) if data.optimizer == "SGD": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 sample_id = 0 sample_loss = 0 total_loss = 0 right_token = 0 whole_token = 0 random.shuffle(data.train_Ids) ## set model in train model model.train() model.zero_grad() batch_size = data.HP_batch_size batch_id = 0 train_num = len(data.train_Ids) total_batch = train_num//batch_size+1 for batch_id in range(total_batch): start = batch_id*batch_size end = (batch_id+1)*batch_size if end >train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label(instance, data.HP_gpu) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss(batch_word,batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) right, whole = predict_check(tag_seq, batch_label, mask) right_token += right whole_token += whole sample_loss += loss.data[0] total_loss += loss.data[0] if end%500 == 0: temp_time = time.time() temp_cost = temp_time - temp_start temp_start = temp_time print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) sys.stdout.flush() sample_loss = 0 loss.backward() optimizer.step() model.zero_grad() temp_time = time.time() temp_cost = temp_time - temp_start print(" Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)) epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print("Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s, total loss: %s"%(idx, epoch_cost, train_num/epoch_cost, total_loss)) # continue speed, acc, p, r, f, _,_ = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f print("Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(dev_cost, speed, acc, p, r, f)) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f"%(dev_cost, speed, acc)) if current_score > best_dev: if data.seg: print("Exceed previous best f score:", best_dev) else: print("Exceed previous best acc score:", best_dev) model_name = data.model_dir +'.'+ str(idx) + ".model" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score # ## decode test speed, acc, p, r, f, _,_ = evaluate(data, model, "test") test_finish = time.time() test_cost = test_finish - dev_finish if data.seg: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f"%(test_cost, speed, acc, p, r, f)) else: print("Test: time: %.2fs, speed: %.2fst/s; acc: %.4f"%(test_cost, speed, acc)) gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + "/data.dset" if data.save_model: data.save(save_data_name) batch_size = data.HP_batch_size train_num = len(data.train_Ids) total_batch = train_num // batch_size + 1 model = SeqModel(data) pytorch_total_params = sum(p.numel() for p in model.parameters()) print(model) print("pytorch total params: %d" % pytorch_total_params) ## model 1 optimizer lr_detail1 = [ { "params": filter(lambda p: p.requires_grad, model.mcmodel.parameters()), "lr": data.HP_lr }, ] if data.optimizer.lower() == "sgd": optimizer = optim.SGD(lr_detail1, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(lr_detail1, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(lr_detail1, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(lr_detail1, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(lr_detail1, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) ## model 2 optimizer optimizer2 = AdamW(model.get_m2_params(), lr=data.HP_lr2, weight_decay=data.HP_l2) t_total = total_batch * data.HP_iteration warmup_step = int(data.warmup_step * t_total) scheduler2 = WarmupLinearSchedule(optimizer2, warmup_step, t_total) best_dev = -10 best_test = -10 max_test = -10 max_test_epoch = -1 max_dev_epoch = -1 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() print("\n ###### Epoch: %s/%s ######" % (idx, data.HP_iteration)) # print (self.train_Ids) if data.optimizer.lower() == "sgd": optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) sample_loss = 0 total_loss = 0 random.shuffle(data.train_Ids) model.train() model.zero_grad() for batch_id in range(total_batch): start = batch_id * batch_size end = (batch_id + 1) * batch_size if end > train_num: end = train_num instance = data.train_Ids[start:end] if not instance: continue batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask = batchify_with_label( instance, data.HP_gpu, True) loss, tag_seqs = model.neg_log_likelihood_loss( batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask) sample_loss += loss.item() total_loss += loss.item() if end % 500 == 0: if sample_loss > 1e8 or str(sample_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) sys.stdout.flush() sample_loss = 0 loss.backward() clip_grad_norm_(model.parameters(), data.clip_grad) optimizer.step() optimizer2.step() scheduler2.step() model.zero_grad() epoch_finish = time.time() epoch_cost = epoch_finish - epoch_start print( "Epoch: %s training finished. Time: %.2f s, speed: %.2f doc/s, total loss: %s" % (idx, epoch_cost, train_num / epoch_cost, total_loss)) if total_loss > 1e8 or str(total_loss) == "nan": print( "ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT...." ) exit(1) # dev dev_score, _ = evaluate(data, model, "dev") # test test_score, _ = evaluate(data, model, "test") if max_test < test_score: max_test_epoch = idx max_test = max(test_score, max_test) if dev_score > best_dev: print("Exceed previous best dev score") best_test = test_score best_dev = dev_score max_dev_epoch = idx if data.save_model: model_name = data.model_dir + "/best_model.ckpt" print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) print( "Score summary: max dev (%d): %.4f, test: %.4f; max test (%d): %.4f" % (max_dev_epoch, best_dev, best_test, max_test_epoch, max_test)) gc.collect()
def train(data): print("Training model...") data.show_data_summary() save_data_name = data.model_dir + ".dset" # 存储data数据 data.save(save_data_name) model = SeqModel(data) # check to load pretrained model if data.use_crf: pretrain_model_path = os.path.join('model_snapshot', 'lan_crf.model') else: pretrain_model_path = os.path.join('model_snapshot', 'lan.model') if data.use_pre_trained_model and os.path.exists(pretrain_model_path): model.load_state_dict(torch.load(pretrain_model_path)) print("load pretrained model success:%s" % pretrain_model_path) pytorch_total_params = sum(p.numel() for p in model.parameters()) print("--------pytorch total params--------") print(pytorch_total_params) optimizer = None if data.optimizer.lower() == "sgd": optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adagrad": optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adadelta": optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "rmsprop": optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) elif data.optimizer.lower() == "adam": optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2) else: print("Optimizer illegal: %s" % (data.optimizer)) exit(1) best_dev = -10 best_test = -10 no_imprv_epoch = 0 ## start training for idx in range(data.HP_iteration): epoch_start = time.time() temp_start = epoch_start print("Epoch: %s/%s" % (idx, data.HP_iteration)) # print (self.train_Ids) # every 5 epoch decay learning rate if idx % 5 == 0: optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr) instance_count = 0 total_loss = 0 ## set model in train model model.train() model.zero_grad() start = 0 end = start + data.HP_batch_size train_epochs = [] while end <= len(data.train_Ids): train_epochs.append((start, end)) start = end end = end + data.HP_batch_size if end > len(data.train_Ids) > start: train_epochs.append((start, len(data.train_Ids))) for sample_id, (start, end) in enumerate(train_epochs): instance = data.train_Ids[start: end] sample_loss = 0 batch_word, batch_word_len, _, batch_word_recover, batch_label, mask, input_label_seq_tensor = batchify_with_label( instance, data.HP_gpu, data) instance_count += 1 loss, tag_seq = model.neg_log_likelihood_loss( batch_word, batch_word_len, batch_label, mask, input_label_seq_tensor) sample_loss += loss.item() total_loss += loss.item() print("Epoch:%s,no_imprv_epoch:%s,Instance: %s" % ( idx, no_imprv_epoch, sample_id)) right, whole = predict_check(tag_seq, batch_label, mask, data.use_crf) print(" loss: %.4f, acc: %s/%s=%.4f" % ( loss.item(), right, whole, (right + 0.) / whole * 100)) if sample_loss > 1e8 or str(sample_loss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) sys.stdout.flush() loss.backward() if data.whether_clip_grad: nn.utils.clip_grad_norm_(model.parameters(), data.clip_grad) optimizer.step() model.zero_grad() # break epoch_finish = time.time() if total_loss > 1e8 or str(total_loss) == "nan": print("ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....") exit(1) speed, acc, report, f_value, \ ner_acc, ner_p, ner_r, ner_f = evaluate(data, model, "dev") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: current_score = f_value # current_score = sent_f1 print("Dev: time: %.2fs, speed: %.2fst/s;\n" "acc: %.4f, f_value: %.4f\n" "ner_acc: %.4f, ner_p: %.4f, ner_r: %.4f, ner_f: %.4f\n" "current f1:%.4f" % ( dev_cost, speed, acc, f_value, ner_acc, ner_p, ner_r, ner_f, current_score )) else: current_score = acc print("Dev: time: %.2fs speed: %.2fst/s; acc: %.4f" % ( dev_cost, speed, acc)) # ## decode test speed, acc, report, f_value, \ ner_acc, ner_p, ner_r, ner_f = evaluate(data, model, "test") dev_finish = time.time() dev_cost = dev_finish - epoch_finish if data.seg: print("Test: time: %.2fs, speed: %.2fst/s;\n" "acc: %.4f, f_value: %.4f\n" "ner_acc: %.4f, ner_p: %.4f, ner_r: %.4f, ner_f: %.4f\n" "current f1:%.4f" % ( dev_cost, speed, acc, f_value, ner_acc, ner_p, ner_r, ner_f, current_score )) else: print("Test: time: %.2fs speed: %.2fst/s; acc: %.4f" % ( dev_cost, speed, acc)) if current_score > best_dev: if data.seg: best_test = f_value # best_test = sent_f1 print("Exceed previous best avg f score:", best_dev) else: best_test = acc print("Exceed previous best acc score:", best_dev) if data.use_crf: result_file = "result_crf.txt" model_name = data.model_dir + "_crf.model" else: result_file = "result.txt" model_name = data.model_dir + ".model" with open(result_file, 'w', encoding='utf-8') as w: w.write( "Save current best model in file:%s, iteration:%s/%s, best_test_f_score:%.5f\n" "ner:\n" " precision:%.5f, recall:%.5f, f1_score:%.5f\n" "%s\n\n" % ( model_name, idx, data.HP_iteration, best_test, ner_p, ner_r, ner_f, report)) print("Save current best model in file:", model_name) torch.save(model.state_dict(), model_name) best_dev = current_score no_imprv_epoch = 0 else: # early stop no_imprv_epoch += 1 if no_imprv_epoch >= 10: print("early stop") print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) break if data.seg: print("Current best f score in dev", best_dev) print("Current best f score in test", best_test) else: print("Current best acc score in dev", best_dev) print("Current best acc score in test", best_test) gc.collect()