def keywords(self): ''' Keywords SelfAttention ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) output_dir = '../nats_results/' + self.args.keywords_output_dir if not os.path.exists(output_dir): os.mkdir(output_dir) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): print('Begin Testing: {}'.format(self.args.file_test)) test_batch = len(self.test_data) print('The number of batches (testing): {}'.format(test_batch)) pred_data = [] true_data = [] keywords_data = [] if self.args.debug: test_batch = 3 for test_id in range(test_batch): self.build_batch(self.test_data[test_id]) ratePred, rateTrue = self.test_worker() output = self.keywords_worker() keywords_data += output pred_data += ratePred true_data += rateTrue show_progress(test_id + 1, test_batch) print() for k in range(len(keywords_data)): keywords_data[k]['pred_label'] = pred_data[k] keywords_data[k]['gold_label'] = true_data[k] fout = open( os.path.join( output_dir, '{}_{}.pickled'.format(self.args.file_test, self.args.best_epoch)), 'wb') pickle.dump(keywords_data, fout) fout.close()
def app(self): ''' Visualization ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() self.app_data = create_batch_memory( path_=self.args.data_dir, file_=self.args.file_app, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower ) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): for epoch in range(self.args.n_epoch): self.init_app_model_params(epoch+1) print('Begin Testing') n_batch = len(self.app_data) print('The number of batches (App): {}'.format(n_batch)) self.pred_data = [] self.true_data = [] for batch_id in range(n_batch): self.build_batch(self.app_data[batch_id]) ratePred, rateTrue = self.test_worker() self.pred_data += ratePred self.true_data += rateTrue show_progress(batch_id+1, n_batch) print() self.pred_data = np.array(self.pred_data).astype(int) np.savetxt( os.path.join('..', 'nats_results', 'app_pred_{}.txt'.format(epoch+1)), self.pred_data, fmt='%d') self.true_data = np.array(self.true_data).astype(int) np.savetxt( os.path.join('..', 'nats_results', 'app_true_{}.txt'.format(epoch+1)), self.true_data, fmt='%d')
def keyword_extraction(self): ''' Visualization ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.vis_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_vis, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) key_dir = '../nats_results/attn_keywords' if not os.path.exists(key_dir): os.mkdir(key_dir) else: shutil.rmtree(key_dir) os.mkdir(key_dir) with torch.no_grad(): print('Begin Generate Keywords') n_batch = len(self.vis_data) print('The number of batches (keywords): {}'.format(n_batch)) for batch_id in range(n_batch): self.build_batch(self.vis_data[batch_id]) self.keyword_worker(batch_id, key_dir) show_progress(batch_id + 1, n_batch) print() for k in range(self.args.n_tasks): key_arr = [[ wd, 100 * self.keywords1[k][wd] / (self.wd_freq[wd] + 100) ] for wd in self.keywords1[k]] key_arr = sorted(key_arr, key=lambda k: k[1])[::-1] key_arr = [[itm[0]] * int(round(itm[1])) for itm in key_arr if (itm[0] not in stop_words) and ( len(itm[0]) > 3) and (itm[0] != '<unk>')] key_arr = key_arr[:100] key_arr = list(itertools.chain(*key_arr)) fout = open(os.path.join(key_dir, str(k) + '.txt'), 'w') fout.write(' '.join(key_arr) + '\n') fout.close()
def test_penultimate(self): ''' Testing ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) output_dir = '../nats_results/' + \ self.args.test_output_dir + '_penultimate' if not os.path.exists(output_dir): os.mkdir(output_dir) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): print('Begin Testing: {}'.format(self.args.file_test)) test_batch = len(self.test_data) print('The number of batches (testing): {}'.format(test_batch)) self.pred_data = [] if self.args.debug: test_batch = 3 for test_id in range(test_batch): self.build_batch(self.test_data[test_id]) logits = self.test_penultimate_worker() self.pred_data += logits.data.cpu().numpy().tolist() show_progress(test_id + 1, test_batch) print() # save testing data. outfile = os.path.join( output_dir, '{}_pred_{}.pickled'.format(self.args.file_test, self.args.best_epoch)) fout = open(outfile, 'wb') pickle.dump(self.pred_data, fout) fout.close()
def visualization(self): ''' Visualization ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.vis_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_vis, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) vis_dir = '../nats_results/attn_vis' if not os.path.exists(vis_dir): os.mkdir(vis_dir) else: shutil.rmtree(vis_dir) os.mkdir(vis_dir) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): print('Begin Visualization') n_batch = len(self.vis_data) print('The number of batches (visualization): {}'.format(n_batch)) for batch_id in range(n_batch): self.build_batch(self.vis_data[batch_id]) self.visualization_worker(batch_id, vis_dir) show_progress(batch_id + 1, n_batch) print()
def visualization(self): ''' Visualization ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() output_dir = '../nats_results/visualization_{}'.format( self.args.file_test) if not os.path.exists(output_dir): os.mkdir(output_dir) aspect_label = self.args.aspect_label.split(',') vis_label = [ int(wd) - 1 for wd in self.args.visualization_aspect.split(',') ] data_aspect = [aspect_label[idx] for idx in vis_label] print('You will visualize Aspects: {}'.format(', '.join(data_aspect))) with torch.no_grad(): print('Begin Testing: {}'.format(self.args.file_test)) test_batch = len(self.test_data) print('The number of batches (testing): {}'.format(test_batch)) pred_data = [] true_data = [] keywords_data = [] if self.args.debug: test_batch = 3 for test_id in range(test_batch): self.build_batch(self.test_data[test_id]) ratePred, rateTrue = self.test_worker() output = self.visualization_worker() keywords_data += output pred_data += ratePred true_data += rateTrue show_progress(test_id + 1, test_batch) print() for k in range(len(keywords_data)): keywords_data[k]['pred_label'] = [ pred_data[k][idx] for idx in vis_label ] keywords_data[k]['gold_label'] = [ true_data[k][idx] for idx in vis_label ] len_txt = len(keywords_data[k]['text'][0].split()) diff = [] for j in range(len(pred_data[k])): if pred_data[k][j] == true_data[k][ j] and true_data[k][j] > 0: diff.append(0) else: diff.append(1) diff = np.sum(diff) ftxt = '_'.join( map(str, [k, len_txt, diff] + true_data[k] + pred_data[k])) file_output = os.path.join(output_dir, '{}.html'.format(ftxt)) keywords_data[k]['text'] = [ keywords_data[k]['text'][idx] for idx in vis_label ] keywords_data[k]['weights'] = [ keywords_data[k]['weights'][idx] for idx in vis_label ] createHTML(data_aspect, keywords_data[k], file_output)
def test_uncertainty(self): ''' Testing Uncertainty ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) output_dir = '../nats_results/{}_uncertainty_{}_{}'.format( self.args.test_output_dir, self.args.drop_option, self.args.drop_rate) if not os.path.exists(output_dir): os.mkdir(output_dir) with torch.no_grad(): for k_unc in range(self.args.uncertainty_total_samples): print('Begin Testing: {}, {}'.format(self.args.file_test, k_unc)) test_batch = len(self.test_data) print('The number of batches (testing): {}'.format(test_batch)) self.pred_data = [] self.true_data = [] self.vector_data = [] if self.args.debug: test_batch = 3 for test_id in range(test_batch): self.build_batch(self.test_data[test_id]) logits = self.test_penultimate_worker() ratePred, rateTrue = self.test_worker() self.vector_data += logits.data.cpu().numpy().tolist() self.pred_data += ratePred self.true_data += rateTrue show_progress(test_id + 1, test_batch) print() # save testing data. outfile = os.path.join( output_dir, '{}_vector_{}_{}.pickled'.format(self.args.file_test, self.args.best_epoch, k_unc)) fout = open(outfile, 'wb') pickle.dump(self.vector_data, fout) fout.close() try: self.pred_data = np.array(self.pred_data).astype(int) np.savetxt(os.path.join( output_dir, '{}_pred_{}_unc_{}.txt'.format(self.args.file_test, self.args.best_epoch, k_unc)), self.pred_data, fmt='%d') self.true_data = np.array(self.true_data).astype(int) np.savetxt(os.path.join( output_dir, '{}_true_{}_unc_{}.txt'.format(self.args.file_test, self.args.best_epoch, k_unc)), self.true_data, fmt='%d') except: fout = open( os.path.join( output_dir, '{}_pred_{}.pickled'.format( self.args.file_best, self.args.best_epoch)), 'wb') pickle.dump(self.pred_data, fout) fout.close() fout = open( os.path.join( output_dir, '{}_true_{}.pickled'.format( self.args.file_test, self.args.best_epoch)), 'wb') pickle.dump(self.true_data, fout) fout.close()
def test(self): ''' Testing ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() if len(self.train_models) > 0: self.init_train_model_params() self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) output_dir = '../nats_results/' + self.args.test_output_dir if not os.path.exists(output_dir): os.mkdir(output_dir) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): print('Begin Testing: {}'.format(self.args.file_test)) test_batch = len(self.test_data) print('The number of batches (testing): {}'.format(test_batch)) self.pred_data = [] self.true_data = [] if self.args.debug: test_batch = 3 for test_id in range(test_batch): self.build_batch(self.test_data[test_id]) ratePred, rateTrue = self.test_worker() self.pred_data += ratePred self.true_data += rateTrue show_progress(test_id + 1, test_batch) print() # save testing data. try: self.pred_data = np.array(self.pred_data).astype(int) np.savetxt(os.path.join( output_dir, '{}_pred_{}.txt'.format(self.args.file_test, self.args.best_epoch)), self.pred_data, fmt='%d') self.true_data = np.array(self.true_data).astype(int) np.savetxt(os.path.join( output_dir, '{}_true_{}.txt'.format(self.args.file_test, self.args.best_epoch)), self.true_data, fmt='%d') except: fout = open( os.path.join( output_dir, '{}_pred_{}.pickled'.format(self.args.file_best, self.args.best_epoch)), 'wb') pickle.dump(self.pred_data, fout) fout.close() fout = open( os.path.join( output_dir, '{}_true_{}.pickled'.format(self.args.file_test, self.args.best_epoch)), 'wb') pickle.dump(self.true_data, fout) fout.close()
def train(self): ''' training here. Don't overwrite. ''' self.build_vocabulary() self.build_models() print(self.base_models) print(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() # here it is necessary to put list. Instead of directly append. for model_name in self.train_models: try: params += list(self.train_models[model_name].parameters()) except: params = list(self.train_models[model_name].parameters()) if self.args.train_base_model: for model_name in self.base_models: try: params += list(self.base_models[model_name].parameters()) except: params = list(self.base_models[model_name].parameters()) # define optimizer optimizer = self.build_optimizer(params) try: scheduler = self.build_scheduler(optimizer) except: pass # load checkpoint cc_model = 0 out_dir = os.path.join('..', 'nats_results') if not os.path.exists(out_dir): os.mkdir(out_dir) if self.args.continue_training: model_para_files = glob.glob(os.path.join(out_dir, '*.model')) if len(model_para_files) > 0: uf_model = [] for fl_ in model_para_files: arr = re.split('\/', fl_)[-1] arr = re.split('\_|\.', arr) if arr not in uf_model: uf_model.append(int(arr[-2])) cc_model = sorted(uf_model)[-1] try: print("Try *_{}.model".format(cc_model)) for model_name in self.train_models: fl_ = os.path.join( out_dir, model_name + '_' + str(cc_model) + '.model') self.train_models[model_name].load_state_dict( torch.load( fl_, map_location=lambda storage, loc: storage)) except: cc_model = sorted(uf_model)[-2] print("Try *_{}.model".format(cc_model)) for model_name in self.train_models: fl_ = os.path.join( out_dir, model_name + '_' + str(cc_model) + '.model') self.train_models[model_name].load_state_dict( torch.load( fl_, map_location=lambda storage, loc: storage)) print('Continue training with *_{}.model'.format(cc_model)) else: shutil.rmtree(out_dir) os.mkdir(out_dir) self.val_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_val, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) # train models if cc_model > 0: cc_model -= 1 for epoch in range(cc_model, self.args.n_epoch): ''' Train ''' for model_name in self.base_models: self.base_models[model_name].train() for model_name in self.train_models: self.train_models[model_name].train() print('====================================') print('Training Epoch: {}'.format(epoch + 1)) self.train_data = create_batch_memory( path_=self.args.data_dir, file_=self.args.file_train, is_shuffle=True, batch_size=self.args.batch_size, is_lower=self.args.is_lower) n_batch = len(self.train_data) print('The number of batches (training): {}'.format(n_batch)) self.global_steps = max(0, epoch) * n_batch try: scheduler.step() except: pass if self.args.debug: n_batch = 1 loss_arr = [] for batch_id in range(n_batch): self.global_steps += 1 self.build_batch(self.train_data[batch_id]) loss = self.build_pipelines() optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(params, self.args.grad_clip) optimizer.step() if batch_id % self.args.checkpoint == 0: for model_name in self.train_models: fmodel = open( os.path.join( out_dir, model_name + '_' + str(epoch + 1) + '.model'), 'wb') torch.save(self.train_models[model_name].state_dict(), fmodel) fmodel.close() show_progress(batch_id + 1, n_batch) loss_arr.append(loss.data.cpu().numpy()) print() # write models print('Training Loss = {}.'.format(np.average(loss_arr))) for model_name in self.train_models: fmodel = open( os.path.join(out_dir, model_name + '_' + str(epoch + 1) + '.model'), 'wb') torch.save(self.train_models[model_name].state_dict(), fmodel) fmodel.close() for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): ''' Validate ''' print('Begin Validation') n_batch = len(self.val_data) print('The number of batches (validation): {}'.format(n_batch)) self.pred_data = [] self.true_data = [] if self.args.debug: n_batch = 1 for batch_id in range(n_batch): self.build_batch(self.val_data[batch_id]) ratePred, rateTrue = self.test_worker() self.pred_data += ratePred self.true_data += rateTrue show_progress(batch_id + 1, n_batch) print() self.pred_data = np.array(self.pred_data).astype(int) np.savetxt(os.path.join( '..', 'nats_results', 'validate_pred_{}.txt'.format(epoch + 1)), self.pred_data, fmt='%d') self.true_data = np.array(self.true_data).astype(int) np.savetxt(os.path.join( '..', 'nats_results', 'validate_true_{}.txt'.format(epoch + 1)), self.true_data, fmt='%d') self.run_evaluation() ''' Testing ''' print('Begin Testing') n_batch = len(self.test_data) print('The number of batches (testing): {}'.format(n_batch)) self.pred_data = [] self.true_data = [] if self.args.debug: n_batch = 1 for batch_id in range(n_batch): self.build_batch(self.test_data[batch_id]) ratePred, rateTrue = self.test_worker() self.pred_data += ratePred self.true_data += rateTrue show_progress(batch_id + 1, n_batch) print() self.pred_data = np.array(self.pred_data).astype(int) np.savetxt(os.path.join('..', 'nats_results', 'test_pred_{}.txt'.format(epoch + 1)), self.pred_data, fmt='%d') self.true_data = np.array(self.true_data).astype(int) np.savetxt(os.path.join('..', 'nats_results', 'test_true_{}.txt'.format(epoch + 1)), self.true_data, fmt='%d') self.run_evaluation()
def test(self): ''' testing Don't overwrite. ''' self.args.batch_size = 1 self.build_vocabulary() self.build_models() pprint(self.base_models) pprint(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() _nbatch = create_batch_file(path_data=self.args.data_dir, path_work=os.path.join( '..', 'nats_results'), is_shuffle=False, fkey_=self.args.task, file_=self.args.file_test, batch_size=self.args.batch_size) print('The number of samples (test): {}'.format(_nbatch)) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): if self.args.use_optimal_model: model_valid_file = os.path.join('..', 'nats_results', 'model_validate.txt') fp = open(model_valid_file, 'r') for line in fp: arr = re.split(r'\s', line[:-1]) model_optimal_key = ''.join( ['_', arr[1], '_', arr[2], '.model']) break fp.close() else: arr = re.split(r'\D', self.args.model_optimal_key) model_optimal_key = ''.join( ['_', arr[0], '_', arr[1], '.model']) print("You choose to use *{} for decoding.".format( model_optimal_key)) for model_name in self.train_models: model_optimal_file = os.path.join( '..', 'nats_results', model_name + model_optimal_key) self.train_models[model_name].load_state_dict( torch.load(model_optimal_file, map_location=lambda storage, loc: storage)) start_time = time.time() output_file = os.path.join('..', 'nats_results', self.args.file_output) data_check = [] try: self.args.continue_decoding except: self.args.continue_decoding = True if os.path.exists(output_file) and self.args.continue_decoding: fchk = open(output_file, 'r') for line in fchk: data_check.append(line) fchk.close() data_check = data_check[:-1] fchk = open(output_file, 'w') for line in data_check: fchk.write(line) fchk.close() else: fout = open(output_file, 'w') fout.close() try: fout = open(output_file, 'a') except: fout = open(output_file, 'w') for batch_id in range(_nbatch): if batch_id < len(data_check): continue self.build_batch(batch_id) self.test_worker() json.dump(self.test_data, fout) fout.write('\n') end_time = time.time() show_progress(batch_id, _nbatch, str((end_time - start_time) / 3600)[:8] + "h") fout.close() print()
def validate(self): ''' Validation here. Don't overwrite. ''' self.build_vocabulary() self.build_models() pprint(self.base_models) pprint(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() best_arr = [] val_file = os.path.join('..', 'nats_results', 'model_validate.txt') if os.path.exists(val_file): fp = open(val_file, 'r') for line in fp: arr = re.split(r'\s', line[:-1]) best_arr.append( [arr[0], arr[1], arr[2], float(arr[3]), float(arr[4])]) fp.close() for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): while 1: model_para_files = [] model_para_files = glob.glob( os.path.join( '..', 'nats_results', sorted(list(self.train_models))[0] + '*.model')) for j in range(len(model_para_files)): arr = re.split(r'\_|\.', model_para_files[j]) arr = [int(arr[-3]), int(arr[-2]), model_para_files[j]] model_para_files[j] = arr model_para_files = sorted(model_para_files) for fl_ in model_para_files: best_model = {itm[0]: itm[3] for itm in best_arr} if fl_[-1] in best_model: continue print('Validate *_{}_{}.model'.format(fl_[0], fl_[1])) losses = [] start_time = time.time() if os.path.exists(fl_[-1]): time.sleep(3) try: for model_name in self.train_models: fl_tmp = os.path.join( '..', 'nats_results', model_name + '_' + str(fl_[0]) + '_' + str(fl_[1]) + '.model') self.train_models[model_name].load_state_dict( torch.load(fl_tmp, map_location=lambda storage, loc: storage)) except: continue else: continue val_batch = create_batch_file( path_data=self.args.data_dir, path_work=os.path.join('..', 'nats_results'), is_shuffle=True, fkey_=self.args.task, file_=self.args.file_val, batch_size=self.args.batch_size) print('The number of batches (test): {}'.format(val_batch)) if self.args.val_num_batch > val_batch: self.args.val_num_batch = val_batch for batch_id in range(self.args.val_num_batch): self.build_batch(batch_id) loss = self.build_pipelines() losses.append(loss.data.cpu().numpy()) show_progress(batch_id + 1, self.args.val_num_batch) print() losses = np.array(losses) end_time = time.time() if self.args.use_move_avg: try: losses_out = 0.9*losses_out + \ 0.1*np.average(losses) except: losses_out = np.average(losses) else: losses_out = np.average(losses) best_arr.append([ fl_[2], fl_[0], fl_[1], losses_out, end_time - start_time ]) best_arr = sorted(best_arr, key=lambda bb: bb[3]) if best_arr[0][0] == fl_[2]: out_dir = os.path.join('..', 'nats_results', 'model') try: shutil.rmtree(out_dir) except: pass os.mkdir(out_dir) for model_name in self.base_models: fmodel = open( os.path.join(out_dir, model_name + '.model'), 'wb') torch.save( self.base_models[model_name].state_dict(), fmodel) fmodel.close() for model_name in self.train_models: fmodel = open( os.path.join(out_dir, model_name + '.model'), 'wb') torch.save( self.train_models[model_name].state_dict(), fmodel) fmodel.close() try: shutil.copy2( os.path.join(self.args.data_dir, self.args.file_vocab), out_dir) except: pass for itm in best_arr[:self.args.nbestmodel]: print('model={}_{}, loss={}, time={}'.format( itm[1], itm[2], np.round(float(itm[3]), 4), np.round(float(itm[4]), 4))) for itm in best_arr[self.args.nbestmodel:]: tarr = re.split(r'_|\.', itm[0]) if tarr[-2] == '0': continue if os.path.exists(itm[0]): for model_name in self.train_models: fl_tmp = os.path.join( '..', 'nats_results', model_name + '_' + str(itm[1]) + '_' + str(itm[2]) + '.model') os.unlink(fl_tmp) fout = open(val_file, 'w') for itm in best_arr: if len(itm) == 0: continue fout.write(' '.join([ itm[0], str(itm[1]), str(itm[2]), str(itm[3]), str(itm[4]) ]) + '\n') fout.close()
def validate(self): ''' Validation here. Don't overwrite. ''' self.build_vocabulary() self.build_models() pprint(self.base_models) pprint(self.train_models) self.init_base_model_params() for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): model_para_files = [] model_para_files = glob.glob( os.path.join('../nats_results', sorted(list(self.train_models))[0] + '*.model')) for j in range(len(model_para_files)): arr = re.split(r'\_|\.', model_para_files[j]) arr = [int(arr[-3]), int(arr[-2]), model_para_files[j]] model_para_files[j] = arr model_para_files = sorted(model_para_files) if not os.path.exists(self.args.optimal_model_dir): os.mkdir(self.args.optimal_model_dir) best_f1 = 0 for fl_ in model_para_files: print('Validate *_{}_{}.model'.format(fl_[0], fl_[1])) try: for model_name in self.train_models: fl_tmp = os.path.join( '../nats_results', model_name + '_' + str(fl_[0]) + '_' + str(fl_[1]) + '.model') self.train_models[model_name].load_state_dict( torch.load( fl_tmp, map_location=lambda storage, loc: storage)) except: print('Models cannot be load!!!') continue val_batch = create_batch_file(path_data=self.args.data_dir, path_work='../nats_results', is_shuffle=False, fkey_=self.args.task, file_=self.args.file_dev, batch_size=self.args.batch_size) print('The number of batches (Dev): {}'.format(val_batch)) val_results = [] for batch_id in range(val_batch): start_time = time.time() self.build_batch(batch_id) self.test_worker() val_results += self.test_data self.test_data = [] end_time = time.time() show_progress(batch_id + 1, val_batch, str((end_time - start_time))[:8] + "s") print() f1 = self.evaluate_worker(val_results) print('Best f1: {}; Current f1: {}.'.format(best_f1, f1)) if f1 > best_f1: for model_name in self.train_models: fmodel = open( os.path.join(self.args.optimal_model_dir, '{}.model'.format(model_name)), 'wb') torch.save(self.train_models[model_name].state_dict(), fmodel) fmodel.close() best_f1 = f1
def test(self): ''' testing Don't overwrite. ''' self.build_vocabulary() self.build_models() pprint(self.base_models) pprint(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() _nbatch = create_batch_file(path_data=self.args.data_dir, path_work=os.path.join( '..', 'nats_results'), is_shuffle=False, fkey_=self.args.task, file_=self.args.file_test, batch_size=self.args.batch_size) print('The number of samples (test): {}'.format(_nbatch)) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): if self.args.use_optimal_model: for model_name in self.train_models: fl_ = os.path.join(self.args.optimal_model_dir, '{}.model'.format(model_name)) self.train_models[model_name].load_state_dict( torch.load(fl_, map_location=lambda storage, loc: storage)) else: arr = re.split(r'\D', self.args.model_optimal_key) model_optimal_key = ''.join( ['_', arr[0], '_', arr[1], '.model']) print("You choose to use *{} for decoding.".format( model_optimal_key)) for model_name in self.train_models: model_optimal_file = os.path.join( '../nats_results', model_name + model_optimal_key) self.train_models[model_name].load_state_dict( torch.load(model_optimal_file, map_location=lambda storage, loc: storage)) start_time = time.time() output_file = os.path.join('../nats_results', self.args.file_output) fout = open(output_file, 'w') self.aspect_worker() for batch_id in range(_nbatch): self.build_batch(batch_id) self.test_worker() for itm in self.test_data: json.dump(itm, fout) fout.write('\n') self.test_data = [] end_time = time.time() show_progress(batch_id + 1, _nbatch, str((end_time - start_time) / 3600)[:8] + "h") fout.close() print()
def train(self): ''' training here. Don't overwrite. ''' self.build_vocabulary() self.build_models() pprint(self.base_models) pprint(self.train_models) if len(self.base_models) > 0: self.init_base_model_params() # here it is necessary to put list. Instead of directly append. params = [] for model_name in self.train_models: params += list(self.train_models[model_name].parameters()) if self.args.train_base_model: for model_name in self.base_models: params += list(self.base_models[model_name].parameters()) print('Total number of parameters: {}.'.format( sum([para.numel() for para in params]))) # define optimizer optimizer = self.build_optimizer(params) if self.args.lr_schedule == 'build-in': scheduler = self.build_scheduler(optimizer) # load checkpoint cc_model = 0 out_dir = os.path.join('..', 'nats_results') if not os.path.exists(out_dir): os.mkdir(out_dir) if self.args.continue_training: model_para_files = glob.glob(os.path.join(out_dir, '*.model')) if len(model_para_files) > 0: uf_model = [] for fl_ in model_para_files: arr = re.split(r'\/', fl_)[-1] arr = re.split(r'\_|\.', arr) if arr not in uf_model: uf_model.append(int(arr[-2])) cc_model = sorted(uf_model)[-1] try: print("Try *_{}.model".format(cc_model)) for model_name in self.train_models: fl_ = os.path.join( out_dir, model_name + '_' + str(cc_model) + '.model') self.train_models[model_name].load_state_dict( torch.load( fl_, map_location=lambda storage, loc: storage)) except: cc_model = sorted(uf_model)[-2] print("Try *_{}.model".format(cc_model)) for model_name in self.train_models: fl_ = os.path.join( out_dir, model_name + '_' + str(cc_model) + '.model') self.train_models[model_name].load_state_dict( torch.load( fl_, map_location=lambda storage, loc: storage)) print('Continue training with *_{}.model'.format(cc_model)) else: shutil.rmtree(out_dir) os.mkdir(out_dir) self.val_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_val, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) self.test_data = create_batch_memory(path_=self.args.data_dir, file_=self.args.file_test, is_shuffle=False, batch_size=self.args.batch_size, is_lower=self.args.is_lower) # train models fout = open('../nats_results/args.pickled', 'wb') pickle.dump(self.args, fout) fout.close() if cc_model > 0: cc_model -= 1 for epoch in range(cc_model, self.args.n_epoch): # Training print('====================================') print('Training Epoch: {}'.format(epoch + 1)) self.train_data = create_batch_memory( path_=self.args.data_dir, file_=self.args.file_train, is_shuffle=True, batch_size=self.args.batch_size, is_lower=self.args.is_lower) n_batch = len(self.train_data) print('The number of batches (training): {}'.format(n_batch)) self.global_steps = max(0, epoch) * n_batch if self.args.debug: n_batch = 3 loss_arr = [] accu_best = 0 for batch_id in range(n_batch): self.global_steps += 1 learning_rate = self.args.learning_rate if self.args.lr_schedule == 'warm-up': learning_rate = 2.0 * \ (self.args.model_size ** (-0.5) * min(self.global_steps ** (-0.5), self.global_steps * self.args.warmup_step**(-1.5))) for p in optimizer.param_groups: p['lr'] = learning_rate elif self.args.lr_schedule == 'build-in': for p in optimizer.param_groups: learning_rate = p['lr'] break # print(learning_rate) ccnt = batch_id % self.args.checkpoint if batch_id > 0 and batch_id % self.args.checkpoint == 0: ccnt = self.args.checkpoint self.build_batch(self.train_data[batch_id]) loss = self.build_pipelines() if loss != loss: raise ValueError optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(params, self.args.grad_clip) optimizer.step() loss_arr.append(loss.data.cpu().numpy().tolist()) message = 'lr={}, Loss={}, AvgLoss={}{}'.format( np.around(learning_rate, 6), np.around(loss.data.cpu().numpy().tolist(), 4), np.around(np.average(loss_arr), 4), ' ' * 10) show_progress(ccnt, min(n_batch, self.args.checkpoint), message) if (batch_id % self.args.checkpoint == 0 and batch_id != 0) or batch_id == n_batch - 1: print() print('Training Loss = {}.'.format(np.average(loss_arr))) for model_name in self.base_models: self.base_models[model_name].eval() for model_name in self.train_models: self.train_models[model_name].eval() with torch.no_grad(): # validate print('Begin Validation') val_batch = len(self.val_data) print('The number of batches (validation): {}'.format( val_batch)) self.pred_data = [] self.true_data = [] if self.args.debug: val_batch = 3 for val_id in range(val_batch): self.build_batch(self.val_data[val_id]) ratePred, rateTrue = self.test_worker() self.pred_data += ratePred self.true_data += rateTrue show_progress(val_id + 1, val_batch) print() # evaluate development accu = self.run_evaluation() print('Best Results: {}'.format(np.round(accu_best, 4))) if accu >= accu_best: accu_best = accu # save results. try: self.pred_data = np.array( self.pred_data).astype(int) np.savetxt(os.path.join( '..', 'nats_results', 'validate_pred_{}.txt'.format(epoch + 1)), self.pred_data, fmt='%d') self.true_data = np.array( self.true_data).astype(int) np.savetxt(os.path.join( '..', 'nats_results', 'validate_true_{}.txt'.format(epoch + 1)), self.true_data, fmt='%d') except: fout = open( os.path.join( '..', 'nats_results', 'validate_pred_{}.pickled'.format( epoch + 1)), 'wb') pickle.dump(self.pred_data, fout) fout.close() fout = open( os.path.join( '..', 'nats_results', 'validate_true_{}.pickled'.format( epoch + 1)), 'wb') pickle.dump(self.true_data, fout) fout.close() # save models for model_name in self.train_models: fmodel = open( os.path.join( out_dir, model_name + '_' + str(epoch + 1) + '.model'), 'wb') torch.save( self.train_models[model_name].state_dict(), fmodel) fmodel.close() # testing print('Begin Testing') test_batch = len(self.test_data) print('The number of batches (testing): {}'.format( test_batch)) self.pred_data = [] self.true_data = [] if self.args.debug: test_batch = 3 for test_id in range(test_batch): self.build_batch(self.test_data[test_id]) ratePred, rateTrue = self.test_worker() self.pred_data += ratePred self.true_data += rateTrue show_progress(test_id + 1, test_batch) print() # save testing data. try: self.pred_data = np.array( self.pred_data).astype(int) np.savetxt(os.path.join( '..', 'nats_results', 'test_pred_{}.txt'.format(epoch + 1)), self.pred_data, fmt='%d') self.true_data = np.array( self.true_data).astype(int) np.savetxt(os.path.join( '..', 'nats_results', 'test_true_{}.txt'.format(epoch + 1)), self.true_data, fmt='%d') except: fout = open( os.path.join( '..', 'nats_results', 'test_pred_{}.pickled'.format(epoch + 1)), 'wb') pickle.dump(self.pred_data, fout) fout.close() fout = open( os.path.join( '..', 'nats_results', 'test_true_{}.pickled'.format(epoch + 1)), 'wb') pickle.dump(self.true_data, fout) fout.close() # evaluate testing self.run_evaluation() print('====================================') for model_name in self.base_models: self.base_models[model_name].train() for model_name in self.train_models: self.train_models[model_name].train() if self.args.lr_schedule == 'build-in': scheduler.step()