def test(sentence, model): chars = [] # print(sentence) for sen in tqdm(sentence.split()): idx = data__.to_index(sen) if idx == 1: idx = data__.to_index(sen.lower()) # print(idx) chars.append(idx) seq_len = len(chars) target = [0] * seq_len target = torch.Tensor(np.array([target])) target = target.type(torch.LongTensor) seq_len = torch.Tensor(np.array([seq_len])) seq_len = seq_len.type(torch.LongTensor) chars = torch.Tensor(np.array([chars])) chars = chars.type(torch.LongTensor) z = dict({'target': target, 'seq_len': seq_len, 'chars': chars}) x = _build_args(model.predict, **z) result = model.predict(**x) nx = result['pred'][0].numpy() result = [] for i, j in zip(sentence.split(), nx): tmp = {} tmp['text'] = i tmp['value'] = dict_labels[j] result.append(tmp) print(result) return result
def get_predictions(pred_model, input_data, batch_size, num_workers=4): texts = list(list(map(lambda x: vocabs['char'].to_word(x), sample['chars'])) for sample in input_data) seq_lens = [sample['seq_len'] for sample in input_data] pred_model.to(device) sampler = SequentialSampler() data_iterator = DataSetIter(dataset=input_data, batch_size=batch_size, sampler=sampler, num_workers=num_workers) with torch.no_grad(): preds, golds = [], [] pred_model.eval() for batch_x, batch_y in data_iterator: _move_dict_value_to_device(batch_x, batch_y, device=device) x = _build_args(pred_model.forward, **batch_x) with torch.no_grad(): y = pred_model.forward(**x) preds.extend(list(map(list, y['pred'].cpu().numpy()))) golds.extend(list(map(list, batch_y['target'].cpu().numpy()))) pred_seqs = list(list(map(lambda _y: vocabs['label'].to_word(_y), pred)) for pred in preds) gold_seqs = list(list(map(lambda _y: vocabs['label'].to_word(_y), pred)) for pred in golds) case_result = [] for pred_seq, gold_seq, word_seq, seq_len in zip(pred_seqs, gold_seqs, texts, seq_lens): pred_seq = pred_seq[:seq_len] gold_seq = gold_seq[:seq_len] case_result.append((''.join(word_seq), extract_kvpairs_in_bmoes(gold_seq, word_seq), extract_kvpairs_in_bmoes(pred_seq, word_seq))) # output for case study os.makedirs(f'../output/case_study/{args.dataset}', exist_ok=True) fout = open(f'../output/case_study/{args.dataset}/{args.dataset}_bert{args.use_bert}_scheme{args.new_tag_scheme}_ple{args.ple_channel_num}_plstm{int(args.use_ple_lstm)}_trainrate{args.train_dataset_rate}.casestudy', 'w', encoding='utf8') for word_seq, gold_pair, pred_pair in case_result: fout.write(word_seq + '\n' + str(gold_pair) + '\n' + str(pred_pair) + '\n\n')
def test(self): # turn on the testing mode; clean up the history network = self._model self.mode(network, is_test=True) self.eval_history.clear() output, truths = defaultdict(list), defaultdict(list) data_iterator = Batch(self.data, self.batch_size, sampler=RandomSampler(), as_numpy=False) with torch.no_grad(): for batch_x, batch_y in data_iterator: prediction = self.data_forward(network, batch_x) assert isinstance(prediction, dict) for k, v in prediction.items(): output[k].append(v) for k, v in batch_y.items(): truths[k].append(v) for k, v in output.items(): output[k] = itertools.chain(*v) for k, v in truths.items(): truths[k] = itertools.chain(*v) args = _build_args(self._evaluator, **output, **truths) eval_results = self._evaluator(**args) print("[tester] {}".format(self.print_eval_results(eval_results))) self.mode(network, is_test=False) return eval_results
def process(self, dataset): self.model.eval() assert isinstance(dataset, DataSet), "Only Dataset class is allowed, not {}.".format(type(dataset)) data_iterator = Batch(dataset, batch_size=self.batch_size, sampler=SequentialSampler()) batch_output = defaultdict(list) if hasattr(self.model, "predict"): predict_func = self.model.predict else: predict_func = self.model.forward with torch.no_grad(): for batch_x, _ in data_iterator: refined_batch_x = _build_args(predict_func, **batch_x) prediction = predict_func(**refined_batch_x) seq_lens = batch_x[self.seq_len_field_name].tolist() for key, value in prediction.items(): tmp_batch = [] value = value.cpu().numpy() if len(value.shape) == 1 or (len(value.shape) == 2 and value.shape[1] == 1): batch_output[key].extend(value.tolist()) else: for idx, seq_len in enumerate(seq_lens): tmp_batch.append(value[idx, :seq_len]) batch_output[key].extend(tmp_batch) if not self.seq_len_field_name in prediction: batch_output[self.seq_len_field_name].extend(seq_lens) # TODO 当前的实现会导致之后的processor需要知道model输出的output的key是什么 for field_name, fields in batch_output.items(): dataset.add_field(field_name, fields, is_input=True, is_target=False) return dataset
def _data_forward(self, network, x): x = _build_args(network.forward, **x) y = network(**x) if not isinstance(y, dict): raise TypeError( f"The return value of {get_func_signature(network.forward)} should be dict, got {type(y)}." ) return y
def get_loss(self, predict, truth): """Compute loss given prediction and ground truth. :param predict: prediction label vector :param truth: ground truth label vector :return: a scalar """ assert isinstance(predict, dict) and isinstance(truth, dict) args = _build_args(self.loss_func, **predict, **truth) return self.loss_func(**args)
def predict(self, data: DataSet, seq_len_field_name=None): r"""用已经训练好的模型进行inference. :param fastNLP.DataSet data: 待预测的数据集 :param str seq_len_field_name: 表示序列长度信息的field名字 :return: dict dict里面的内容为模型预测的结果 """ if not isinstance(data, DataSet): raise ValueError("Only Dataset class is allowed, not {}.".format( type(data))) if seq_len_field_name is not None and seq_len_field_name not in data.field_arrays: raise ValueError("Field name {} not found in DataSet {}.".format( seq_len_field_name, data)) prev_training = self.network.training self.network.eval() network_device = _get_model_device(self.network) batch_output = defaultdict(list) data_iterator = DataSetIter(data, batch_size=self.batch_size, sampler=SequentialSampler(), as_numpy=False) if hasattr(self.network, "predict"): predict_func = self.network.predict else: predict_func = self.network.forward with torch.no_grad(): for batch_x, _ in data_iterator: _move_dict_value_to_device(batch_x, _, device=network_device) refined_batch_x = _build_args(predict_func, **batch_x) prediction = predict_func(**refined_batch_x) if seq_len_field_name is not None: seq_lens = batch_x[seq_len_field_name].tolist() for key, value in prediction.items(): value = value.cpu().numpy() if len(value.shape) == 1 or (len(value.shape) == 2 and value.shape[1] == 1): batch_output[key].extend(value.tolist()) else: if seq_len_field_name is not None: tmp_batch = [] for idx, seq_len in enumerate(seq_lens): tmp_batch.append(value[idx, :seq_len]) batch_output[key].extend(tmp_batch) else: batch_output[key].append(value) self.network.train(prev_training) return batch_output
def predict(self, data, seq_len_field_name=None): """Perform inference using the trained model. :param data: a DataSet object. :param str seq_len_field_name: field name indicating sequence lengths :return: list of batch outputs """ if not isinstance(data, DataSet): raise ValueError("Only Dataset class is allowed, not {}.".format( type(data))) if seq_len_field_name is not None and seq_len_field_name not in data.field_arrays: raise ValueError("Field name {} not found in DataSet {}.".format( seq_len_field_name, data)) self.network.eval() batch_output = defaultdict(list) data_iterator = Batch(data, batch_size=self.batch_size, sampler=SequentialSampler(), as_numpy=False, prefetch=False) if hasattr(self.network, "predict"): predict_func = self.network.predict else: predict_func = self.network.forward with torch.no_grad(): for batch_x, _ in data_iterator: refined_batch_x = _build_args(predict_func, **batch_x) prediction = predict_func(**refined_batch_x) if seq_len_field_name is not None: seq_lens = batch_x[seq_len_field_name].tolist() for key, value in prediction.items(): value = value.cpu().numpy() if len(value.shape) == 1 or (len(value.shape) == 2 and value.shape[1] == 1): batch_output[key].extend(value.tolist()) else: if seq_len_field_name is not None: tmp_batch = [] for idx, seq_len in enumerate(seq_lens): tmp_batch.append(value[idx, :seq_len]) batch_output[key].extend(tmp_batch) else: batch_output[key].append(value) return batch_output
def predict(self, data: DataSet, seq_len_field_name=None): r""" """ if not isinstance(data, DataSet): raise ValueError( "Only Dataset class is allowed, not {}.".format(type(data))) if seq_len_field_name is not None and seq_len_field_name not in data.field_arrays: raise ValueError("Field name {} not found in DataSet {}.".format( seq_len_field_name, data)) self.network.eval() # self.network.module for multi-GPU network_device = _get_model_device(self.network) batch_output = defaultdict(list) data_iterator = DataSetIter( data, batch_size=self.batch_size, sampler=SequentialSampler(), as_numpy=False) # predict_func = self.network.module.predict # self.network.module for # multi-GPU try: predict_func = self.network.predict except ModuleAttributeError: predict_func = self.network.module.predict with torch.no_grad(): # for batch_x, _ in tqdm(data_iterator): for batch_x, _ in tqdm(data_iterator, total=len(data_iterator)): _move_dict_value_to_device(batch_x, _, device=network_device) refined_batch_x = _build_args(predict_func, **batch_x) prediction = predict_func(**refined_batch_x) if seq_len_field_name is not None: seq_lens = batch_x[seq_len_field_name].tolist() for key, value in prediction.items(): value = value.cpu().numpy() if len(value.shape) == 1 or ( len(value.shape) == 2 and value.shape[1] == 1): batch_output[key].extend(value.tolist()) else: if seq_len_field_name is not None: tmp_batch = [] for idx, seq_len in enumerate(seq_lens): tmp_batch.append(value[idx, :seq_len]) batch_output[key].extend(tmp_batch) else: batch_output[key].append(value) return batch_output
def get_loss(self, inputs, targets, hidden, dags): """Computes the loss for the same batch for M models. This amounts to an estimate of the loss, which is turned into an estimate for the gradients of the shared model. """ if not isinstance(dags, list): dags = [dags] loss = 0 for dag in dags: self.shared.setDAG(dag) inputs = _build_args(self.shared.forward, **inputs) inputs['hidden'] = hidden result = self.shared(**inputs) output, hidden, extra_out = result['pred'], result[ 'hidden'], result['extra_out'] self.callback_manager.on_loss_begin(targets, result) sample_loss = self._compute_loss(result, targets) loss += sample_loss assert len(dags) == 1, 'there are multiple `hidden` for multple `dags`' return loss, hidden, extra_out
def __call__(self, pred_dict, target_dict): """ This method will call self.evaluate method. Before calling self.evaluate, it will first check the validity of output_dict, target_dict (1) whether self.evaluate has varargs, which is not supported. (2) whether params needed by self.evaluate is not included in output_dict,target_dict. (3) whether params needed by self.evaluate duplicate in pred_dict, target_dict (4) whether params in output_dict, target_dict are not used by evaluate.(Might cause warning) Besides, before passing params into self.evaluate, this function will filter out params from output_dict and target_dict which are not used in self.evaluate. (but if **kwargs presented in self.evaluate, no filtering will be conducted.) This function also support _fast_param_map. :param pred_dict: usually the output of forward or prediction function :param target_dict: usually features set as target.. :return: """ if not callable(self.evaluate): raise TypeError( f"{self.__class__.__name__}.evaluate has to be callable, not {type(self.evaluate)}." ) fast_param = self._fast_param_map(pred_dict=pred_dict, target_dict=target_dict) if fast_param: self.evaluate(**fast_param) return if not self._checked: # 1. check consistence between signature and param_map func_spect = inspect.getfullargspec(self.evaluate) func_args = set([arg for arg in func_spect.args if arg != 'self']) for func_arg, input_arg in self.param_map.items(): if func_arg not in func_args: raise NameError( f"`{func_arg}` not in {get_func_signature(self.evaluate)}." ) # 2. only part of the param_map are passed, left are not for arg in func_args: if arg not in self.param_map: self.param_map[ arg] = arg # This param does not need mapping. self._evaluate_args = func_args self._reverse_param_map = { input_arg: func_arg for func_arg, input_arg in self.param_map.items() } # need to wrap inputs in dict. mapped_pred_dict = {} mapped_target_dict = {} duplicated = [] for input_arg in set( list(pred_dict.keys()) + list(target_dict.keys())): not_duplicate_flag = 0 if input_arg in self._reverse_param_map: mapped_arg = self._reverse_param_map[input_arg] not_duplicate_flag += 1 else: mapped_arg = input_arg if input_arg in pred_dict: mapped_pred_dict[mapped_arg] = pred_dict[input_arg] not_duplicate_flag += 1 if input_arg in target_dict: mapped_target_dict[mapped_arg] = target_dict[input_arg] not_duplicate_flag += 1 if not_duplicate_flag == 3: duplicated.append(input_arg) # missing if not self._checked: check_res = _check_arg_dict_list( self.evaluate, [mapped_pred_dict, mapped_target_dict]) # only check missing. # replace missing. missing = check_res.missing replaced_missing = list(missing) for idx, func_arg in enumerate(missing): # Don't delete `` in this information, nor add `` replaced_missing[idx] = f"{self.param_map[func_arg]}" + f"(assign to `{func_arg}` " \ f"in `{self.__class__.__name__}`)" check_res = CheckRes(missing=replaced_missing, unused=check_res.unused, duplicated=duplicated, required=check_res.required, all_needed=check_res.all_needed, varargs=check_res.varargs) if check_res.missing or check_res.duplicated or check_res.varargs: raise CheckError(check_res=check_res, func_signature=get_func_signature( self.evaluate)) refined_args = _build_args(self.evaluate, **mapped_pred_dict, **mapped_target_dict) self.evaluate(**refined_args) self._checked = True return
def _check_code(dataset, model, batch_size=DEFAULT_CHECK_BATCH_SIZE, dev_data=None, check_level=WARNING_CHECK_LEVEL): # check get_loss 方法 model_name = model.__class__.__name__ if not hasattr(model, 'get_loss'): raise AttributeError( "{} has to have a 'get_loss' function.".format(model_name)) batch = Batch(dataset=dataset, batch_size=batch_size, sampler=SequentialSampler()) for batch_count, (batch_x, batch_y) in enumerate(batch): _syn_model_data(model, batch_x, batch_y) # forward check if batch_count == 0: _check_forward_error(model_func=model.forward, check_level=check_level, batch_x=batch_x) refined_batch_x = _build_args(model.forward, **batch_x) output = model(**refined_batch_x) func_signature = get_func_signature(model.forward) assert isinstance( output, dict), "The return value of {} should be dict.".format( func_signature) # loss check if batch_count == 0: _check_loss_evaluate(prev_func=model.forward, func=model.get_loss, check_level=check_level, output=output, batch_y=batch_y) loss_input = _build_args(model.get_loss, **output, **batch_y) loss = model.get_loss(**loss_input) # check loss output if batch_count == 0: if not isinstance(loss, torch.Tensor): raise ValueError( "The return value of {}.get_loss() should be torch.Tensor, but {} got." .format(model_name, type(loss))) if len(loss.size()) != 0: raise ValueError( "The size of return value of {}.get_loss() is {}, should be torch.size([])" .format(model_name, loss.size())) loss.backward() model.zero_grad() if batch_count + 1 >= DEFAULT_CHECK_NUM_BATCH: break if dev_data is not None: if not hasattr(model, 'evaluate'): raise AttributeError( "{} has to have a 'evaluate' function to do evaluation. Or set" "dev_data to 'None'.".format(model_name)) outputs, truths = defaultdict(list), defaultdict(list) dev_batch = Batch(dataset=dataset, batch_size=batch_size, sampler=SequentialSampler()) with torch.no_grad(): for batch_count, (batch_x, batch_y) in enumerate(dev_batch): _syn_model_data(model, batch_x, batch_y) if hasattr(model, 'predict'): refined_batch_x = _build_args(model.predict, **batch_x) prev_func = model.predict output = prev_func(**refined_batch_x) func_signature = get_func_signature(model.predict) assert isinstance( output, dict), "The return value of {} should be dict.".format( func_signature) else: refined_batch_x = _build_args(model.forward, **batch_x) prev_func = model.forward output = prev_func(**refined_batch_x) for k, v in output.items(): outputs[k].append(v) for k, v in batch_y.items(): truths[k].append(v) if batch_count + 1 > DEFAULT_CHECK_NUM_BATCH: break for k, v in outputs.items(): outputs[k] = itertools.chain(*v) for k, v in truths.items(): truths[k] = itertools.chain(*v) _check_loss_evaluate(prev_func=prev_func, func=model.evaluate, check_level=check_level, output=outputs, batch_y=truths) refined_input = _build_args(model.evaluate, **outputs, **truths) metrics = model.evaluate(**refined_input) func_signature = get_func_signature(model.evaluate) assert isinstance(metrics, dict), "The return value of {} should be dict.". \ format(func_signature)
def data_forward(self, network, x): x = _build_args(network.forward, **x) y = network(**x) return y
def _check_code(dataset, model, losser, metrics, batch_size=DEFAULT_CHECK_BATCH_SIZE, dev_data=None, metric_key=None, check_level=0): # check get_loss 方法 model_devcie = model.parameters().__next__().device batch = Batch(dataset=dataset, batch_size=batch_size, sampler=SequentialSampler()) for batch_count, (batch_x, batch_y) in enumerate(batch): _move_dict_value_to_device(batch_x, batch_y, device=model_devcie) # forward check if batch_count == 0: info_str = "" input_fields = _get_value_info(batch_x) target_fields = _get_value_info(batch_y) if len(input_fields) > 0: info_str += "input fields after batch(if batch size is {}):\n".format( batch_size) info_str += "\n".join(input_fields) info_str += '\n' else: raise RuntimeError("There is no input field.") if len(target_fields) > 0: info_str += "target fields after batch(if batch size is {}):\n".format( batch_size) info_str += "\n".join(target_fields) info_str += '\n' else: info_str += 'There is no target field.' print(info_str) _check_forward_error(forward_func=model.forward, dataset=dataset, batch_x=batch_x, check_level=check_level) refined_batch_x = _build_args(model.forward, **batch_x) pred_dict = model(**refined_batch_x) func_signature = get_func_signature(model.forward) if not isinstance(pred_dict, dict): raise TypeError( f"The return value of {func_signature} should be `dict`, not `{type(pred_dict)}`." ) # loss check try: loss = losser(pred_dict, batch_y) # check loss output if batch_count == 0: if not isinstance(loss, torch.Tensor): raise TypeError( f"The return value of {get_func_signature(losser.get_loss)} should be `torch.Tensor`, " f"but got `{type(loss)}`.") if len(loss.size()) != 0: raise ValueError( f"The size of return value of {get_func_signature(losser.get_loss)} is {loss.size()}, " f"should be torch.size([])") loss.backward() except CheckError as e: # TODO: another error raised if CheckError caught pre_func_signature = get_func_signature(model.forward) _check_loss_evaluate(prev_func_signature=pre_func_signature, func_signature=e.func_signature, check_res=e.check_res, pred_dict=pred_dict, target_dict=batch_y, dataset=dataset, check_level=check_level) model.zero_grad() if batch_count + 1 >= DEFAULT_CHECK_NUM_BATCH: break if dev_data is not None: tester = Tester(data=dataset[:batch_size * DEFAULT_CHECK_NUM_BATCH], model=model, metrics=metrics, batch_size=batch_size, verbose=-1) evaluate_results = tester.test() _check_eval_results(metrics=evaluate_results, metric_key=metric_key, metric_list=metrics)
def __call__(self, pred_dict, target_dict, check=False): """ :param pred_dict: A dict from forward function of the network. :param target_dict: A dict from DataSet.batch_y. :param check: Boolean. Force to check the mapping functions when it is running. :return: """ fast_param = self._fast_param_map(pred_dict, target_dict) if fast_param: loss = self.get_loss(**fast_param) return loss if not self._checked: # 1. check consistence between signature and param_map func_spect = inspect.getfullargspec(self.get_loss) func_args = set([arg for arg in func_spect.args if arg != 'self']) for func_arg, input_arg in self.param_map.items(): if func_arg not in func_args: raise NameError( f"`{func_arg}` not in {get_func_signature(self.get_loss)}." ) # 2. only part of the param_map are passed, left are not for arg in func_args: if arg not in self.param_map: self.param_map[ arg] = arg # This param does not need mapping. self._evaluate_args = func_args self._reverse_param_map = { input_arg: func_arg for func_arg, input_arg in self.param_map.items() } # need to wrap inputs in dict. mapped_pred_dict = {} mapped_target_dict = {} duplicated = [] for input_arg in set( list(pred_dict.keys()) + list(target_dict.keys())): not_duplicate_flag = 0 if input_arg in self._reverse_param_map: mapped_arg = self._reverse_param_map[input_arg] not_duplicate_flag += 1 else: mapped_arg = input_arg if input_arg in pred_dict: mapped_pred_dict[mapped_arg] = pred_dict[input_arg] not_duplicate_flag += 1 if input_arg in target_dict: mapped_target_dict[mapped_arg] = target_dict[input_arg] not_duplicate_flag += 1 if not_duplicate_flag == 3: duplicated.append(input_arg) # missing if not self._checked: check_res = _check_arg_dict_list( self.get_loss, [mapped_pred_dict, mapped_target_dict]) # replace missing. missing = check_res.missing replaced_missing = list(missing) for idx, func_arg in enumerate(missing): # Don't delete `` in this information, nor add `` replaced_missing[idx] = f"{self.param_map[func_arg]}" + f"(assign to `{func_arg}` " \ f"in `{self.__class__.__name__}`)" check_res = CheckRes(missing=replaced_missing, unused=check_res.unused, duplicated=duplicated, required=check_res.required, all_needed=check_res.all_needed, varargs=check_res.varargs) if check_res.missing or check_res.duplicated: raise CheckError(check_res=check_res, func_signature=get_func_signature( self.get_loss)) refined_args = _build_args(self.get_loss, **mapped_pred_dict, **mapped_target_dict) loss = self.get_loss(**refined_args) self._checked = True return loss
def data_forward(self, network, x): """A forward pass of the model. """ x = _build_args(network.forward, **x) y = self._predict_func(**x) return y
def _data_forward(self, func, x): """A forward pass of the model. """ x = _build_args(func, **x) y = func(**x) return y