def score(self, qa_pairs, model, data, task=None): input_ids_lst = [] input_mask_lst = [] segment_ids_lst = [] label_ids_lst = [] for (q, a, label) in qa_pairs: cache_id = '{}-{}'.format(id(q), id(a)) example_features = self._cache.get(cache_id) if example_features is None: example = InputExample(q.metadata['id'], q.text, a.text, label=label) example_features = convert_examples_to_features([example], self.length_question + self.length_answer, model.tokenizer, self.logger)[0] self._cache[cache_id] = example_features input_ids_lst.append(example_features.input_ids) input_mask_lst.append(example_features.input_mask) segment_ids_lst.append(example_features.segment_ids) label_ids_lst.append(example_features.label_id) input_ids = torch.tensor(input_ids_lst, dtype=torch.long).to(model.device) input_mask = torch.tensor(input_mask_lst, dtype=torch.long).to(model.device) segment_ids = torch.tensor(segment_ids_lst, dtype=torch.long).to(model.device) label_ids = torch.tensor(label_ids_lst, dtype=torch.float).to(model.device) with torch.no_grad(): scores = model.bert(input_ids, segment_ids, input_mask, label_ids, tasks=task) # loss_fct = nn.BCELoss() # loss = loss_fct(scores, label_ids.view(-1, 1)) return scores.squeeze(dim=1).cpu().numpy(), None # loss.cpu().numpy()
def get_next_batch(self, model, data): self.cur_dataset = (self.cur_dataset + 1) % len(self.train_questions) n_positive = self.batchsize // self.n_train_answers example_index = self.get_batch_indices(self.cur_dataset, n_positive) batch_examples = [] for i in example_index: unrelated_answers = [ InputExample( self.train_questions[self.cur_dataset][i].metadata['id'], self.train_questions[self.cur_dataset][i].text, a.text, label=0.0) for a in self.get_random_answers( self.cur_dataset, self.n_train_answers - 1) ] unrelated_answers_features = convert_examples_to_features( unrelated_answers, self.length_question + self.length_answer, model.tokenizer, self.logger, show_example=False) batch_examples += [ self.train_positive_instances[self.cur_dataset][i] ] + unrelated_answers_features self.batch_i += n_positive return batch_examples
def prepare_training(self, model, data): super(BertTrainingPointwiseAllQSameBatch, self).prepare_training(model, data) # create features for all training examples for i, dataset in enumerate(data.datas_train): dataset_examples = [] for pool in dataset.archive.train.qa: for gt in pool.ground_truth: self.train_questions[i].append(pool.question) dataset_examples.append( InputExample(pool.question.metadata['id'], pool.question.text, gt.text, label=1.0)) self.train_positive_instances[i] = convert_examples_to_features( dataset_examples, self.length_question + self.length_answer, model.tokenizer, self.logger) # calculate number of batches min_examples = min([len(v) for v in self.train_questions.values()]) n_datasets = len(self.train_questions) if self.epoch_max_examples is None or min_examples * n_datasets < self.epoch_max_examples: examples_per_epoch = min_examples * n_datasets else: examples_per_epoch = self.epoch_max_examples self.n_batches = examples_per_epoch
def prepare_training(self, model, data): super(BertTrainingPointwiseWithNegativePoolsAllQSameBatch, self).prepare_training(model, data) # create features for all training examples for i, dataset in enumerate(data.datas_train): dataset_batches = [] for pool in dataset.archive.train.qa: batch = [] for a in pool.pooled_answers: batch.append( InputExample( pool.question.metadata['id'], pool.question.text, a.text, label=1.0 if a in pool.ground_truth else 0.0)) dataset_batches.append( convert_examples_to_features(batch, self.length_question + self.length_answer, model.tokenizer, self.logger, show_example=False)) self.batches[i] = dataset_batches # calculate number of batches min_batches = min([len(v) for v in self.batches.values()]) self.n_batches = min_batches * len(self.batches)
def prepare_training(self, model, data): super(BertTrainingPointwiseWithNegativePools, self).prepare_training(model, data) # create features for all training examples for i, dataset in enumerate(data.datas_train): dataset_examples = [] for pool in dataset.archive.train.qa: for a in pool.ground_truth: input_a, input_b = self.potentially_flip_inputs( pool.question.text, a.text) dataset_examples.append( InputExample(pool.question.metadata['id'], input_a, input_b, label=1.0)) neg = [] pot_neg_answer_list = pool.pooled_answers if self.negative_sample_from_top_n is not False: pot_neg_answer_list = list( pool.pooled_answers[:self.negative_sample_from_top_n]) random.shuffle(pot_neg_answer_list) for a in pot_neg_answer_list: if a in pool.ground_truth: continue input_a, input_b = self.potentially_flip_inputs( pool.question.text, a.text) neg.append( InputExample(pool.question.metadata['id'], input_a, input_b, label=0.0)) if len(neg) >= self.max_negative_per_answer * len( pool.ground_truth): break dataset_examples += neg self.train_instances[i] = convert_examples_to_features( dataset_examples, self.length_question + self.length_answer, model.tokenizer, self.logger) # calculate number of batches min_examples = min([len(v) for v in self.train_instances.values()]) n_datasets = len(self.train_instances) if self.epoch_max_examples is None or min_examples * n_datasets < self.epoch_max_examples: examples_per_epoch = min_examples * n_datasets else: examples_per_epoch = self.epoch_max_examples self.n_batches = math.ceil(examples_per_epoch / float(self.batchsize))
def prepare_training(self, model, data): examples = [] for i, dataset in enumerate(data.datas_train): for pool in dataset.archive.train.qa: for gt in pool.ground_truth: self.train_questions.append(pool.question) self.train_dataset_ids.append(i) examples.append( InputExample(pool.question.metadata['id'], pool.question.text, gt.text, label=1.0) ) self.train_positive_instances = convert_examples_to_features( examples, self.length_question + self.length_answer, model.tokenizer, self.logger )
def get_next_batch(self, model, data): batchsize_half = int(self.batchsize / 2) indices = self.epoch_random_indices[self.batch_i * batchsize_half: (self.batch_i + 1) * batchsize_half] batch_examples = [] prediction_examples = [] for i in indices: batch_examples.append(self.train_positive_instances[i]) prediction_examples += [InputExample(self.train_questions[i].metadata['id'], self.train_questions[i].text, next(self.random_pools[self.train_dataset_ids[i]]).text, label=0.0) for _ in range(self.n_train_answers)] prediction_examples = convert_examples_to_features( prediction_examples, self.length_question + self.length_answer, model.tokenizer, self.logger ) # we only execute all this negative sampling using the network IF we choose between more than one neg. answer if self.n_train_answers > 1: prediction_results = [] model.bert.eval() for predict_batch in range(int(math.ceil(len(prediction_examples) / float(self.batchsize_neg_ranking)))): batch_start_idx = predict_batch * self.batchsize_neg_ranking predict_batch_examples = prediction_examples[ batch_start_idx: batch_start_idx + self.batchsize_neg_ranking] input_ids = torch.tensor([f.input_ids for f in predict_batch_examples], dtype=torch.long).to( model.device) input_mask = torch.tensor([f.input_mask for f in predict_batch_examples], dtype=torch.long).to( model.device) segment_ids = torch.tensor([f.segment_ids for f in predict_batch_examples], dtype=torch.long).to( model.device) with torch.no_grad(): scores = model.bert(input_ids, segment_ids, input_mask) prediction_results += scores.squeeze(dim=1).tolist() for count, i in enumerate(indices): predictions = prediction_results[self.n_train_answers * count:self.n_train_answers * (count + 1)] incorrect_example = prediction_examples[np.argmax(predictions) + self.n_train_answers * count] batch_examples.append(incorrect_example) else: batch_examples += prediction_examples self.batch_i += 1 return batch_examples
def _get_mean_adapter(self, model, pool): question = [InputExample("", pool.question.text, label=1.0)] features = convert_examples_to_features( question, self.length_question + self.length_answer, model.tokenizer, self.logger) input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long).to(model.device) input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long).to(model.device) segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long).to(model.device) with torch.no_grad(): q_vector = model.bert.average_standard_bert_output( input_ids, segment_ids, input_mask).cpu().numpy() q_vector = np.squeeze(q_vector / norm(q_vector)) if self.mean_adapter_similarity == "cosine": similarity = 1 - np.dot(self.vectors, q_vector) task = self.tasks[np.argmin(similarity)] return task
def score(self, qa_pairs, model, data): eval_examples = [ InputExample(q.metadata['id'], q.text, a.text, label=label) for (q, a, label) in qa_pairs ] eval_features = convert_examples_to_features( eval_examples, self.length_question + self.length_answer, model.tokenizer, self.logger) input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long).to(model.device) input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long).to(model.device) segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long).to(model.device) label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.float).to(model.device) with torch.no_grad(): scores, loss = model.bert(input_ids, segment_ids, input_mask, label_ids) return scores.squeeze(dim=1).cpu().numpy(), loss.cpu().numpy()