def load(self, conf, problem, emb_matrix): # load dictionary when (not finetune) and (cache valid) if not conf.pretrained_model_path and not self.dictionary_invalid: problem.load_problem(conf.problem_path) if not self.embedding_invalid: emb_matrix = np.array(load_from_pkl(conf.emb_pkl_path)) logging.info('[Cache] loading dictionary successfully') if not self.encoding_invalid: pass return problem, emb_matrix
def load_problem(self, problem_path): info_dict = load_from_pkl(problem_path) for name in info_dict: if isinstance(getattr(self, name), CellDict): getattr(self, name).load_cell_dict(info_dict[name]) else: setattr(self, name, info_dict[name]) # the type of input_dicts is dict # elif name == 'input_dicts' and isinstance(getattr(self, name), type(info_dict[name])): # setattr(self, name, info_dict[name]) logging.debug("Problem loaded")
def test(self, loss_fn, test_data_path=None, predict_output_path=None): if test_data_path is None: # test_data_path in the parameter is prior to self.conf.test_data_path test_data_path = self.conf.test_data_path if not test_data_path.endswith('.pkl'): test_data, test_length, test_target = self.problem.encode(test_data_path, self.conf.file_columns, self.conf.input_types, self.conf.file_with_col_header, self.conf.object_inputs, self.conf.answer_column_name, max_lengths=self.conf.max_lengths, min_sentence_len = self.conf.min_sentence_len, extra_feature = self.conf.extra_feature,fixed_lengths=self.conf.fixed_lengths, file_format='tsv', show_progress=True if self.conf.mode == 'normal' else False, cpu_num_workers=self.conf.cpu_num_workers) else: test_pkl_data = load_from_pkl(test_data_path) test_data, test_length, test_target = test_pkl_data['data'], test_pkl_data['length'], test_pkl_data['target'] if not predict_output_path: self.evaluate(test_data, test_length, test_target, self.conf.input_types, self.evaluator, loss_fn, pad_ids=None, phase="test") else: self.evaluate(test_data, test_length, test_target, self.conf.input_types, self.evaluator, loss_fn, pad_ids=None, phase="test", origin_data_path=test_data_path, predict_output_path=predict_output_path)
def main(params): conf = ModelConf("train", params.conf_path, version, params, mode=params.mode) shutil.copy(params.conf_path, conf.save_base_dir) logging.info('Configuration file is backed up to %s' % (conf.save_base_dir)) if ProblemTypes[conf.problem_type] == ProblemTypes.sequence_tagging: problem = Problem(conf.problem_type, conf.input_types, conf.answer_column_name, source_with_start=True, source_with_end=True, source_with_unk=True, source_with_pad=True, target_with_start=True, target_with_end=True, target_with_unk=True, target_with_pad=True, same_length=True, with_bos_eos=conf.add_start_end_for_seq, tagging_scheme=conf.tagging_scheme, remove_stopwords=conf.remove_stopwords, DBC2SBC=conf.DBC2SBC, unicode_fix=conf.unicode_fix) elif ProblemTypes[conf.problem_type] == ProblemTypes.classification \ or ProblemTypes[conf.problem_type] == ProblemTypes.regression: problem = Problem(conf.problem_type, conf.input_types, conf.answer_column_name, source_with_start=True, source_with_end=True, source_with_unk=True, source_with_pad=True, target_with_start=False, target_with_end=False, target_with_unk=False, target_with_pad=False, same_length=False, with_bos_eos=conf.add_start_end_for_seq, remove_stopwords=conf.remove_stopwords, DBC2SBC=conf.DBC2SBC, unicode_fix=conf.unicode_fix) elif ProblemTypes[conf.problem_type] == ProblemTypes.mrc: problem = Problem(conf.problem_type, conf.input_types, conf.answer_column_name, source_with_start=True, source_with_end=True, source_with_unk=True, source_with_pad=True, target_with_start=False, target_with_end=False, target_with_unk=False, target_with_pad=False, same_length=False, with_bos_eos=False, remove_stopwords=conf.remove_stopwords, DBC2SBC=conf.DBC2SBC, unicode_fix=conf.unicode_fix) cache_load_flag = False if not conf.pretrained_model_path: # first time training, load cache if appliable if conf.use_cache: cache_conf_path = os.path.join(conf.cache_dir, 'conf_cache.json') if os.path.isfile(cache_conf_path): params_cache = copy.deepcopy(params) ''' for key in vars(params_cache): setattr(params_cache, key, None) params_cache.mode = params.mode ''' try: cache_conf = ModelConf('cache', cache_conf_path, version, params_cache) except Exception as e: cache_conf = None if cache_conf is None or verify_cache(cache_conf, conf) is not True: logging.info('Found cache that is ineffective') if params.mode == 'philly' or params.force is True: renew_option = 'yes' else: renew_option = input('There exists ineffective cache %s for old models. Input "yes" to renew cache and "no" to exit. (default:no): ' % os.path.abspath(conf.cache_dir)) if renew_option.lower() != 'yes': exit(0) else: shutil.rmtree(conf.cache_dir) time.sleep(2) # sleep 2 seconds since the deleting is asynchronous logging.info('Old cache is deleted') else: logging.info('Found cache that is appliable to current configuration...') elif os.path.isdir(conf.cache_dir): renew_option = input('There exists ineffective cache %s for old models. Input "yes" to renew cache and "no" to exit. (default:no): ' % os.path.abspath(conf.cache_dir)) if renew_option.lower() != 'yes': exit(0) else: shutil.rmtree(conf.cache_dir) time.sleep(2) # Sleep 2 seconds since the deleting is asynchronous logging.info('Old cache is deleted') if not os.path.exists(conf.cache_dir): os.makedirs(conf.cache_dir) shutil.copy(params.conf_path, os.path.join(conf.cache_dir, 'conf_cache.json')) # first time training, load problem from cache, and then backup the cache to model_save_dir/.necessary_cache/ if conf.use_cache and os.path.isfile(conf.problem_path): problem.load_problem(conf.problem_path) if conf.emb_pkl_path is not None: if os.path.isfile(conf.emb_pkl_path): emb_matrix = np.array(load_from_pkl(conf.emb_pkl_path)) cache_load_flag = True else: if params.mode == 'normal': renew_option = input('The cache is invalid because the embedding matrix does not exist in the cache directory. Input "yes" to renew cache and "no" to exit. (default:no): ') if renew_option.lower() != 'yes': exit(0) else: # by default, renew cache renew_option = 'yes' else: emb_matrix = None cache_load_flag = True if cache_load_flag: logging.info("Cache loaded!") if cache_load_flag is False: logging.info("Preprocessing... Depending on your corpus size, this step may take a while.") if conf.pretrained_emb_path: emb_matrix = problem.build(conf.train_data_path, conf.file_columns, conf.input_types, conf.file_with_col_header, conf.answer_column_name, word2vec_path=conf.pretrained_emb_path, word_emb_dim=conf.pretrained_emb_dim, format=conf.pretrained_emb_type, file_type=conf.pretrained_emb_binary_or_text, involve_all_words=conf.involve_all_words_in_pretrained_emb, show_progress=True if params.mode == 'normal' else False, max_vocabulary=conf.max_vocabulary, word_frequency=conf.min_word_frequency) else: emb_matrix = problem.build(conf.train_data_path, conf.file_columns, conf.input_types, conf.file_with_col_header, conf.answer_column_name, word2vec_path=None, word_emb_dim=None, format=None, file_type=None, involve_all_words=conf.involve_all_words_in_pretrained_emb, show_progress=True if params.mode == 'normal' else False, max_vocabulary=conf.max_vocabulary, word_frequency=conf.min_word_frequency) if conf.mode == 'philly' and conf.emb_pkl_path.startswith('/hdfs/'): with HDFSDirectTransferer(conf.problem_path, with_hdfs_command=True) as transferer: transferer.pkl_dump(problem.export_problem(conf.problem_path, ret_without_save=True)) else: problem.export_problem(conf.problem_path) if conf.use_cache: logging.info("Cache saved to %s" % conf.problem_path) if emb_matrix is not None and conf.emb_pkl_path is not None: if conf.mode == 'philly' and conf.emb_pkl_path.startswith('/hdfs/'): with HDFSDirectTransferer(conf.emb_pkl_path, with_hdfs_command=True) as transferer: transferer.pkl_dump(emb_matrix) else: dump_to_pkl(emb_matrix, conf.emb_pkl_path) logging.info("Embedding matrix saved to %s" % conf.emb_pkl_path) else: logging.debug("Cache saved to %s" % conf.problem_path) # Back up the problem.pkl to save_base_dir/.necessary_cache. During test phase, we would load cache from save_base_dir/.necessary_cache/problem.pkl cache_bakup_path = os.path.join(conf.save_base_dir, 'necessary_cache/') logging.debug('Prepare dir: %s' % cache_bakup_path) prepare_dir(cache_bakup_path, True, allow_overwrite=True, clear_dir_if_exist=True) shutil.copy(conf.problem_path, cache_bakup_path) logging.debug("Problem %s is backed up to %s" % (conf.problem_path, cache_bakup_path)) if problem.output_dict: logging.debug("Problem target cell dict: %s" % (problem.output_dict.cell_id_map)) if params.make_cache_only: logging.info("Finish building cache!") return vocab_info = dict() # include input_type's vocab_size & init_emd_matrix vocab_sizes = problem.get_vocab_sizes() for input_cluster in vocab_sizes: vocab_info[input_cluster] = dict() vocab_info[input_cluster]['vocab_size'] = vocab_sizes[input_cluster] # add extra info for char_emb if input_cluster.lower() == 'char': for key, value in conf.input_types[input_cluster].items(): if key != 'cols': vocab_info[input_cluster][key] = value if input_cluster == 'word' and emb_matrix is not None: vocab_info[input_cluster]['init_weights'] = emb_matrix else: vocab_info[input_cluster]['init_weights'] = None lm = LearningMachine('train', conf, problem, vocab_info=vocab_info, initialize=True, use_gpu=conf.use_gpu) else: # when finetuning, load previous saved problem problem.load_problem(conf.saved_problem_path) lm = LearningMachine('train', conf, problem, vocab_info=None, initialize=False, use_gpu=conf.use_gpu) if len(conf.metrics_post_check) > 0: for metric_to_chk in conf.metrics_post_check: metric, target = metric_to_chk.split('@') if not problem.output_dict.has_cell(target): raise Exception("The target %s of %s does not exist in the training data." % (target, metric_to_chk)) if conf.pretrained_model_path: logging.info('Loading the pretrained model: %s...' % conf.pretrained_model_path) lm.load_model(conf.pretrained_model_path) loss_conf = conf.loss loss_conf['output_layer_id'] = conf.output_layer_id loss_conf['answer_column_name'] = conf.answer_column_name # loss_fn = eval(loss_conf['type'])(**loss_conf['conf']) loss_fn = Loss(**loss_conf) if conf.use_gpu is True: loss_fn.cuda() optimizer = eval(conf.optimizer_name)(lm.model.parameters(), **conf.optimizer_params) lm.train(optimizer, loss_fn) # test the best model with the best model saved lm.load_model(conf.model_save_path) if conf.test_data_path is not None: test_path = conf.test_data_path elif conf.valid_data_path is not None: test_path = conf.valid_data_path logging.info('Testing the best model saved at %s, with %s' % (conf.model_save_path, test_path)) if not test_path.endswith('pkl'): lm.test(loss_fn, test_path, predict_output_path=conf.predict_output_path) else: lm.test(loss_fn, test_path)
def _load_encoding_cache_generator(cache_dir, file_index): for index in file_index: file_path = os.path.join(cache_dir, index[0]) yield load_from_pkl(file_path)
def train(self, optimizer, loss_fn): self.model.train() if not self.conf.train_data_path.endswith('.pkl'): train_data, train_length, train_target = self.problem.encode(self.conf.train_data_path, self.conf.file_columns, self.conf.input_types, self.conf.file_with_col_header, self.conf.object_inputs, self.conf.answer_column_name, max_lengths=self.conf.max_lengths, min_sentence_len = self.conf.min_sentence_len, extra_feature=self.conf.extra_feature,fixed_lengths=self.conf.fixed_lengths, file_format='tsv', show_progress=True if self.conf.mode == 'normal' else False, cpu_num_workers=self.conf.cpu_num_workers) else: train_pkl_data = load_from_pkl(self.conf.train_data_path) train_data, train_length, train_target = train_pkl_data['data'], train_pkl_data['length'], train_pkl_data['target'] if not self.conf.valid_data_path.endswith('.pkl'): valid_data, valid_length, valid_target = self.problem.encode(self.conf.valid_data_path, self.conf.file_columns, self.conf.input_types, self.conf.file_with_col_header, self.conf.object_inputs, self.conf.answer_column_name, max_lengths=self.conf.max_lengths, min_sentence_len = self.conf.min_sentence_len, extra_feature = self.conf.extra_feature,fixed_lengths=self.conf.fixed_lengths, file_format='tsv', show_progress=True if self.conf.mode == 'normal' else False, cpu_num_workers=self.conf.cpu_num_workers) else: valid_pkl_data = load_from_pkl(self.conf.valid_data_path) valid_data, valid_length, valid_target = valid_pkl_data['data'], valid_pkl_data['length'], valid_pkl_data['target'] if self.conf.test_data_path is not None: if not self.conf.test_data_path.endswith('.pkl'): test_data, test_length, test_target = self.problem.encode(self.conf.test_data_path, self.conf.file_columns, self.conf.input_types, self.conf.file_with_col_header, self.conf.object_inputs, self.conf.answer_column_name, max_lengths=self.conf.max_lengths, min_sentence_len = self.conf.min_sentence_len, extra_feature = self.conf.extra_feature,fixed_lengths=self.conf.fixed_lengths, file_format='tsv', show_progress=True if self.conf.mode == 'normal' else False, cpu_num_workers=self.conf.cpu_num_workers) else: test_pkl_data = load_from_pkl(self.conf.test_data_path) test_data, test_length, test_target = test_pkl_data['data'], test_pkl_data['length'], test_pkl_data['target'] stop_training = False epoch = 1 best_result = None show_result_cnt = 0 lr_scheduler = LRScheduler(optimizer, self.conf.lr_decay, self.conf.minimum_lr, self.conf.epoch_start_lr_decay) if ProblemTypes[self.problem.problem_type] == ProblemTypes.classification: streaming_recoder = StreamingRecorder(['prediction', 'pred_scores', 'pred_scores_all', 'target']) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.sequence_tagging: streaming_recoder = StreamingRecorder(['prediction', 'pred_scores', 'target']) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.regression: streaming_recoder = StreamingRecorder(['prediction', 'target']) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.mrc: streaming_recoder = StreamingRecorder(['prediction', 'answer_text']) while not stop_training and epoch <= self.conf.max_epoch: logging.info('Training: Epoch ' + str(epoch)) data_batches, length_batches, target_batches = \ get_batches(self.problem, train_data, train_length, train_target, self.conf.batch_size_total, self.conf.input_types, None, permutate=True, transform_tensor=True) whole_batch_num = len(target_batches) valid_batch_num = max(len(target_batches) // self.conf.valid_times_per_epoch, 1) if torch.cuda.device_count() > 1: small_batch_num = whole_batch_num * torch.cuda.device_count() # total batch num over all the gpus valid_batch_num_show = valid_batch_num * torch.cuda.device_count() # total batch num over all the gpus to do validation else: small_batch_num = whole_batch_num valid_batch_num_show = valid_batch_num streaming_recoder.clear_records() all_costs = [] logging.info('There are %d batches during an epoch; validation are conducted every %d batch' % (small_batch_num, valid_batch_num_show)) if self.conf.mode == 'normal': progress = tqdm(range(len(target_batches))) elif self.conf.mode == 'philly': progress = range(len(target_batches)) for i in progress: # the result shape: for classification: [batch_size, # of classes]; for sequence tagging: [batch_size, seq_len, # of tags] param_list, inputs_desc, length_desc = transform_params2tensors(data_batches[i], length_batches[i]) logits_softmax = self.model(inputs_desc, length_desc, *param_list) # check the output if ProblemTypes[self.problem.problem_type] == ProblemTypes.classification: logits_softmax = list(logits_softmax.values())[0] assert len(logits_softmax.shape) == 2, 'The dimension of your output is %s, but we need [batch_size*GPUs, class num]' % (str(list(logits_softmax.shape))) assert logits_softmax.shape[1] == self.problem.output_target_num(), 'The dimension of your output layer %d is inconsistent with your type number %d!' % (logits_softmax.shape[1], self.problem.output_target_num()) # for auc metric prediction_scores = logits_softmax[:, self.conf.pos_label].cpu().data.numpy() if self.evaluator.has_auc_type_specific: prediction_scores_all = logits_softmax.cpu().data.numpy() else: prediction_scores_all = None elif ProblemTypes[self.problem.problem_type] == ProblemTypes.sequence_tagging: logits_softmax = list(logits_softmax.values())[0] assert len(logits_softmax.shape) == 3, 'The dimension of your output is %s, but we need [batch_size*GPUs, sequence length, representation dim]' % (str(list(logits_softmax.shape)), ) prediction_scores = None prediction_scores_all = None elif ProblemTypes[self.problem.problem_type] == ProblemTypes.regression: logits_softmax = list(logits_softmax.values())[0] assert len(logits_softmax.shape) == 2 and logits_softmax.shape[1] == 1, 'The dimension of your output is %s, but we need [batch_size*GPUs, 1]' % (str(list(logits_softmax.shape))) prediction_scores = None prediction_scores_all = None elif ProblemTypes[self.problem.problem_type] == ProblemTypes.mrc: for single_value in logits_softmax.values(): assert len(single_value.shape) == 3, 'The dimension of your output is %s, but we need [batch_size*GPUs, sequence_len, 1]' % (str(list(single_value.shape))) prediction_scores = None prediction_scores_all = None logits_softmax_flat = dict() if ProblemTypes[self.problem.problem_type] == ProblemTypes.sequence_tagging: # Transform output shapes for metric evaluation # for seq_tag_f1 metric prediction_indices = logits_softmax.data.max(2)[1].cpu().numpy() # [batch_size, seq_len] streaming_recoder.record_one_row([self.problem.decode(prediction_indices, length_batches[i]['target'][self.conf.answer_column_name[0]].numpy()), prediction_scores, self.problem.decode(target_batches[i][self.conf.answer_column_name[0]], length_batches[i]['target'][self.conf.answer_column_name[0]].numpy())], keep_dim=False) # pytorch's CrossEntropyLoss only support this logits_softmax_flat[self.conf.output_layer_id[0]] = logits_softmax.view(-1, logits_softmax.size(2)) # [batch_size * seq_len, # of tags] #target_batches[i] = target_batches[i].view(-1) # [batch_size * seq_len] # [batch_size * seq_len] target_batches[i][self.conf.answer_column_name[0]] = target_batches[i][self.conf.answer_column_name[0]].reshape(-1) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.classification: prediction_indices = logits_softmax.detach().max(1)[1].cpu().numpy() # Should not decode! streaming_recoder.record_one_row([prediction_indices, prediction_scores, prediction_scores_all, target_batches[i][self.conf.answer_column_name[0]].numpy()]) logits_softmax_flat[self.conf.output_layer_id[0]] = logits_softmax elif ProblemTypes[self.problem.problem_type] == ProblemTypes.regression: temp_logits_softmax_flat = logits_softmax.squeeze(1) prediction_scores = temp_logits_softmax_flat.detach().cpu().numpy() streaming_recoder.record_one_row([prediction_scores, target_batches[i][self.conf.answer_column_name[0]].numpy()]) logits_softmax_flat[self.conf.output_layer_id[0]] = temp_logits_softmax_flat elif ProblemTypes[self.problem.problem_type] == ProblemTypes.mrc: for key, value in logits_softmax.items(): logits_softmax[key] = value.squeeze() passage_identify = None for type_key in data_batches[i].keys(): if 'p' in type_key.lower(): passage_identify = type_key break if not passage_identify: raise Exception('MRC task need passage information.') prediction = self.problem.decode(logits_softmax, lengths=length_batches[i][passage_identify], batch_data=data_batches[i][passage_identify]) logits_softmax_flat = logits_softmax mrc_answer_target = None for single_target in target_batches[i]: if isinstance(target_batches[i][single_target][0], str): mrc_answer_target = target_batches[i][single_target] streaming_recoder.record_one_row([prediction, mrc_answer_target]) if self.use_gpu: for single_target in self.conf.answer_column_name: if isinstance(target_batches[i][single_target], torch.Tensor): target_batches[i][single_target] = transfer_to_gpu(target_batches[i][single_target]) loss = loss_fn(logits_softmax_flat, target_batches[i]) all_costs.append(loss.item()) optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.conf.clip_grad_norm_max_norm) optimizer.step() del loss, logits_softmax, logits_softmax_flat del prediction_scores if ProblemTypes[self.problem.problem_type] == ProblemTypes.sequence_tagging \ or ProblemTypes[self.problem.problem_type] == ProblemTypes.classification: del prediction_indices if show_result_cnt == self.conf.batch_num_to_show_results: if ProblemTypes[self.problem.problem_type] == ProblemTypes.classification: result = self.evaluator.evaluate(streaming_recoder.get('target'), streaming_recoder.get('prediction'), y_pred_pos_score=streaming_recoder.get('pred_scores'), y_pred_scores_all=streaming_recoder.get('pred_scores_all'), formatting=True) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.sequence_tagging: result = self.evaluator.evaluate(streaming_recoder.get('target'), streaming_recoder.get('prediction'), y_pred_pos_score=streaming_recoder.get('pred_scores'), formatting=True) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.regression: result = self.evaluator.evaluate(streaming_recoder.get('target'), streaming_recoder.get('prediction'), y_pred_pos_score=None, y_pred_scores_all=None, formatting=True) elif ProblemTypes[self.problem.problem_type] == ProblemTypes.mrc: result = self.evaluator.evaluate(streaming_recoder.get('answer_text'), streaming_recoder.get('prediction'), y_pred_pos_score=None, y_pred_scores_all=None, formatting=True) if torch.cuda.device_count() > 1: logging.info("Epoch %d batch idx: %d; lr: %f; since last log, loss=%f; %s" % \ (epoch, i * torch.cuda.device_count(), lr_scheduler.get_lr(), np.mean(all_costs), result)) else: logging.info("Epoch %d batch idx: %d; lr: %f; since last log, loss=%f; %s" % \ (epoch, i, lr_scheduler.get_lr(), np.mean(all_costs), result)) show_result_cnt = 0 # The loss and other metrics printed during a training epoch are just the result of part of the training data. all_costs = [] streaming_recoder.clear_records() if (i != 0 and i % valid_batch_num == 0) or i == len(target_batches) - 1: torch.cuda.empty_cache() # actually useless logging.info('Valid & Test : Epoch ' + str(epoch)) new_result = self.evaluate(valid_data, valid_length, valid_target, self.conf.input_types, self.evaluator, loss_fn, pad_ids=None, cur_best_result=best_result, model_save_path=self.conf.model_save_path, phase="valid", epoch=epoch) renew_flag = best_result != new_result best_result = new_result if renew_flag and self.conf.test_data_path is not None: self.evaluate(test_data, test_length, test_target, self.conf.input_types, self.evaluator, loss_fn, pad_ids=None, phase="test", epoch=epoch) self.model.train() show_result_cnt += 1 del data_batches, length_batches, target_batches lr_scheduler.step() epoch += 1