def test_masked_softmax(): """ 测试 masked softmax :return: """ vector = torch.FloatTensor([[1., 2., 3.], [4., 5., 6.]]) mask = torch.ByteTensor([[1, 1, 0], [1, 1, 1]]) result = masked_softmax(vector=vector, mask=mask) expect1 = np.exp(np.array([1., 2.])) expect1 = expect1 / np.sum(expect1) expect1 = np.concatenate([expect1, np.array([0.])], axis=-1).tolist() result1 = result[0].tolist() ASSERT.assertEqual(len(expect1), len(result1)) for expect_data, result_data in zip(expect1, result1): ASSERT.assertAlmostEqual(expect_data, result_data) expect2 = np.exp(np.array([4., 5., 6.])) expect2 = expect2 / np.sum(expect2) expect2 = expect2.tolist() result2 = result[1].tolist() ASSERT.assertEqual(len(expect2), len(result2)) for expect_data, result_data in zip(expect2, result2): ASSERT.assertAlmostEqual(expect_data, result_data)
def test_multi_input_lstm_cell(): """ 测试 MultiInputLSTMCell """ input_size = 2 hidden_size = 3 cell = MultiInputLSTMCell(input_size=input_size, hidden_size=hidden_size, bias=True) with torch.no_grad(): weight_ih_value = list() for i in range(input_size): weight_ih_value.append([ j * 0.37 for j in range(i * hidden_size * 3, (i + 1) * hidden_size * 3) ]) cell.weight_ih.copy_(torch.tensor(weight_ih_value, dtype=torch.float)) alpha_weight_ih_value = list() for i in range(input_size): alpha_weight_ih_value.append([ j * 0.23 for j in range(i * hidden_size, (i + 1) * hidden_size) ]) cell.alpha_weight_ih.copy_( torch.tensor(alpha_weight_ih_value, dtype=torch.float)) torch.nn.init.constant(cell.bias, val=1.0) torch.nn.init.constant(cell.alpha_bias, val=0.5) char_input = torch.tensor([[0.2, 0.4]], dtype=torch.float) h = torch.tensor([[0.2, 0.11, 0.15]], dtype=torch.float) c = torch.tensor([[0.5, 0.6, 0.7]], dtype=torch.float) word_c_input = [ torch.tensor([[0.7, 0.5, 0.2]], dtype=torch.float), torch.tensor([[0.3, 0.4, 1.5]], dtype=torch.float) ] output_hc = cell(input_=char_input, c_input=word_c_input, hx=(h, c)) expect_size = (1, hidden_size) ASSERT.assertEqual(expect_size, output_hc[0].size()) ASSERT.assertEqual(expect_size, output_hc[1].size()) expects = [[0.5728, 0.5523, 0.7130], [0.6873, 0.6506, 0.9345]] for expect, hc in zip(expects, output_hc): for e_i, hc_i in zip(expect, hc[0].tolist()): ASSERT.assertAlmostEqual(e_i, hc_i, places=4)
def test_acc_metric(): # 对应的label是 [1, 1, 0, 1] logits = torch.tensor([[1., 2.], [3., 4.], [5, 4.], [3., 7.]], dtype=torch.float) prediction_labels = torch.argmax(logits, dim=-1) golden_labels = torch.tensor([0, 1, 1, 0], dtype=torch.long) acc_metric = AccMetric() expect = 1/4 acc = acc_metric(prediction_labels=prediction_labels, gold_labels=golden_labels, mask=None) ASSERT.assertAlmostEqual(expect, acc[acc_metric.ACC]) ASSERT.assertAlmostEqual(expect, acc_metric.metric[acc_metric.ACC]) # 对应的label是 [0, 1, 0, 1] logits = torch.tensor([[3., 2.], [4., 6.], [5, 4.], [3., 7.]], dtype=torch.float) prediction_labels = torch.argmax(logits, dim=-1) golden_labels = torch.tensor([0, 1, 1, 0], dtype=torch.long) acc = acc_metric(prediction_labels=prediction_labels, gold_labels=golden_labels, mask=None) expect = 2 / 4 ASSERT.assertAlmostEqual(expect, acc[acc_metric.ACC]) # 下面的会计算将两次结果综合起来 expect = (1+2)/(4+4) ASSERT.assertAlmostEqual(expect, acc_metric.metric[acc_metric.ACC])
def test_span_f1_measure_with_mask(): # [[O, B, I], [B, B, I], [B, I, I], [B, I, O]] batch_sequence_logits = torch.tensor([[[0.2, 0.3, 0.4], [0.7, 0.2, 0.3], [0.2, 0.3, 0.1]], [[0.8, 0.3, 0.4], [0.7, 0.2, 0.3], [0.2, 0.3, 0.1]], [[0.8, 0.3, 0.4], [0.1, 0.7, 0.3], [0.2, 0.3, 0.1]], [[0.8, 0.3, 0.4], [0.1, 0.7, 0.3], [0.2, 0.3, 0.5]]], dtype=torch.float) batch_sequence_labels = [["O", "B-T", "I-T"], ["B-T", "B-T", "I-T"], ["B-T", "I-T", "I-T"], ["B-T", "I-T", "O"]] sequence_label_indices = list() for sequence_label in batch_sequence_labels: sequence_label_indices.append([VOCAB.index(label) for label in sequence_label]) sequence_label_indices = torch.tensor(sequence_label_indices, dtype=torch.long) gold = torch.tensor([ [2, 0, 1], [0, 0, 1], [0, 1, 1], [0, 1, 2] ]) f1 = SpanF1Metric(label_vocabulary=VOCAB) mask = torch.tensor([ [1, 1, 0], [1, 1, 1], [1, 0, 0], [1, 1, 1] ], dtype=torch.long) f1(prediction_labels=sequence_label_indices, gold_labels=gold, mask=mask) metrics = f1.metric print(f"metrics: {json.dumps(metrics)}") expect = {f"{SpanF1Metric.PRECISION}-T": 1., f"{SpanF1Metric.RECALL}-T": 1., f"{SpanF1Metric.F1}-T": 1., f"{SpanF1Metric.PRECISION_OVERALL}": 1., f"{SpanF1Metric.RECALL_OVERALL}": 1., f"{SpanF1Metric.F1_OVERALL}": 1.} for key, _ in expect.items(): ASSERT.assertAlmostEqual(expect[key], metrics[key])
def test_cnn_seq2vec(): """ 测试 cnn seq2vec :return: """ encoder = CnnSeq2Vec(embedding_dim=2, num_filters=1, kernel_sizes=(1, 2)) for name, parameter in encoder.named_parameters(): parameter.data.fill_(1.) tokens = torch.FloatTensor([[[0.7, 0.8], [0.1, 1.5]]]) vector = encoder(sequence=tokens, mask=None) vector = vector.view(-1).tolist() expect = torch.tensor([[0.1 + 1.5 + 1., 0.7 + 0.8 + 0.1 + 1.5 + 1.]]).view(-1).tolist() ASSERT.assertEqual(len(expect), len(vector)) for i in range(len(vector)): ASSERT.assertAlmostEqual(expect[i], vector[i])
def test_label_f1_metric_with_mask(): """ 测试 label f1 metric """ predictions = torch.tensor([0, 1, 2, 3]) gold_labels = torch.tensor([0, 0, 0, 2]) mask = torch.tensor([1, 1, 1, 0], dtype=torch.long) labels = [0, 1, 2, 3] f1_metric = LabelF1Metric(labels=labels, label_vocabulary=None) metrics = f1_metric(prediction_labels=predictions, gold_labels=gold_labels, mask=mask) logging.debug(json2str(metrics)) ASSERT.assertEqual((len(labels) + 1) * 3, len(metrics)) precision_0 = metrics[f"{LabelF1Metric.PRECISION}-0"] recall_0 = metrics[f"{LabelF1Metric.RECALL}-0"] f1_0 = metrics[f"{LabelF1Metric.F1}-0"] expect_precision_0 = 1. / 1. ASSERT.assertAlmostEqual(expect_precision_0, precision_0) expect_recall_0 = 1. / 3. ASSERT.assertAlmostEqual(expect_recall_0, recall_0) expect_f1_0 = 2. * expect_precision_0 * expect_recall_0 / ( expect_precision_0 + expect_recall_0) ASSERT.assertAlmostEqual(expect_f1_0, f1_0) expect_precision_overall = 1. / 3. expect_recall_overall = 1. / 3. precision_overall = metrics[LabelF1Metric.PRECISION_OVERALL] recall_overall = metrics[LabelF1Metric.RECALL_OVERALL] ASSERT.assertAlmostEqual(expect_precision_overall, precision_overall) ASSERT.assertAlmostEqual(expect_recall_overall, recall_overall)
def test_attention_seq2vec_no_mask(inputs): """ 测试 attention seq2vec :return: """ sequence, mask = inputs encoder = AttentionSeq2Vec(input_size=2, query_hidden_size=3, value_hidden_size=None) encoder.wk.weight = Parameter(FloatTensor([ [0.1, 0.2], [0.3, 0.4], [0.5, 0.6] ])) encoder.wk.bias = Parameter(FloatTensor([0.2, 0.4, 0.6])) encoder.attention.weight = Parameter(FloatTensor( [ [0.6, 0.2, 7] ] )) vec = encoder(sequence=sequence, mask=None) print(vec) ASSERT.assertEqual((2, 2), vec.size()) expect = torch.tensor([[4.8455, 5.2867], [5.7232, 3.6037]]) vec1d = vec.view(-1).tolist() expect1d = expect.view(-1).tolist() for expect_data, vec_data in zip(expect1d, vec1d): ASSERT.assertAlmostEqual(expect_data, vec_data, delta=1e-4)
def test_word_lstm_cell_with_bias(): """ 测试 WordLSTMCell :return: """ input_size = 2 hidden_size = 3 word_lstm_cell = WordLSTMCell(input_size=input_size, hidden_size=hidden_size, bias=True) value = list() for i in range(input_size): value.append([ j * 0.37 for j in range(i * hidden_size * 3, (i + 1) * hidden_size * 3) ]) with torch.no_grad(): word_lstm_cell.weight_ih.copy_(torch.tensor(value, dtype=torch.float)) torch.nn.init.constant(word_lstm_cell.bias, val=1.0) word_input = torch.tensor([[0.2, 0.4]], dtype=torch.float) h = torch.tensor([[0.2, 0.11, 0.15]], dtype=torch.float) c = torch.tensor([[0.5, 0.6, 0.7]], dtype=torch.float) output_c = word_lstm_cell(input_=word_input, hx=(h, c)) expect_size = (1, hidden_size) ASSERT.assertEqual(expect_size, output_c.size()) expect_output_c = [1.4231, 1.5257, 1.6372] for e_i, i in zip(expect_output_c, output_c[0].tolist()): ASSERT.assertAlmostEqual(e_i, i, places=3)
def test_synchronized_data(): """ 测试 from_synchronized_data 和 to_synchronized_data :return: """ acc_metric = AccMetric() sync_data, op = acc_metric.to_synchronized_data() ASSERT.assertEqual((2,), sync_data.size()) ASSERT.assertEqual(0, sync_data[0].item()) ASSERT.assertEqual(0, sync_data[1].item()) # 对应的label是 [1, 1, 0, 1] logits = torch.tensor([[1., 2.], [3., 4.], [5, 4.], [3., 7.]], dtype=torch.float) prediction_labels = torch.argmax(logits, dim=-1) golden_labels = torch.tensor([0, 1, 1, 0], dtype=torch.long) acc_metric(prediction_labels=prediction_labels, gold_labels=golden_labels, mask=None) # acc = 1/4 sync_data, op = acc_metric.to_synchronized_data() ASSERT.assertListEqual([1, 4], sync_data.tolist()) acc_metric.from_synchronized_data(sync_data=sync_data, reduce_op=op) acc = acc_metric.metric expect = 1/4 ASSERT.assertAlmostEqual(expect, acc[AccMetric.ACC]) new_sync_data, op = acc_metric.to_synchronized_data() ASSERT.assertListEqual(sync_data.tolist(), new_sync_data.tolist())
def test_event_f1_metric(event_type_vocabulary): """ 测试 event f1 metric """ f1_metric = EventF1MetricAdapter( event_type_vocabulary=event_type_vocabulary) # label: [1, 0, 1, 1, 0, 0, 1, 1, 0] logits = torch.tensor([0.6, 0.2, 0.7, 0.8, 0.1, 0.2, 0.8, 0.9, 0.3], dtype=torch.float) # [1, 0, 1, 1, 0, 0, 1, 1, 0] golden_labels = torch.tensor([1, 0, 0, 1, 1, 0, 0, 1, 1], dtype=torch.long) event_type = torch.tensor([ event_type_vocabulary.index("A"), event_type_vocabulary.index("A"), event_type_vocabulary.index("A"), event_type_vocabulary.index("B"), event_type_vocabulary.index("B"), event_type_vocabulary.index("C"), event_type_vocabulary.index("C"), event_type_vocabulary.index(event_type_vocabulary.unk), event_type_vocabulary.index(event_type_vocabulary.unk) ], dtype=torch.long) model_outputs = EventModelOutputs(logits=logits, event_type=event_type) metric, target_metric = f1_metric(model_outputs=model_outputs, golden_labels=golden_labels) expect_precision_A_1 = 1 / 2 expect_recall_A_1 = 1 / 1 expect_f1_A_1 = 2 * expect_precision_A_1 * expect_recall_A_1 / ( expect_precision_A_1 + expect_recall_A_1) ASSERT.assertAlmostEqual(expect_precision_A_1, metric[f"{F1Metric.PRECISION}-A"]) ASSERT.assertAlmostEqual(expect_recall_A_1, metric[f"{F1Metric.RECALL}-A"]) ASSERT.assertAlmostEqual(expect_f1_A_1, metric[f"{F1Metric.F1}-A"]) expect_precision_overall = 2 / 4 expect_recall_overall = 2 / 3 expect_f1_overall = 2 * expect_precision_overall * expect_recall_overall / ( expect_precision_overall + expect_recall_overall) ASSERT.assertAlmostEqual(expect_precision_overall, metric[F1Metric.PRECISION_OVERALL]) ASSERT.assertAlmostEqual(expect_recall_overall, metric[F1Metric.RECALL_OVERALL]) ASSERT.assertAlmostEqual(expect_f1_overall, metric[F1Metric.F1_OVERALL]) # 在增加一个数据,因为实际是多个batch的 # label: [1, 1, 0] logits = torch.tensor([0.6, 0.8, 0.2], dtype=torch.float) golden_labels = torch.tensor([1, 1, 1], dtype=torch.long) event_type = torch.tensor([ event_type_vocabulary.index("A"), event_type_vocabulary.index("A"), event_type_vocabulary.index("A") ], dtype=torch.long) model_outputs = EventModelOutputs(logits=logits, event_type=event_type) metric, target_metric = f1_metric(model_outputs=model_outputs, golden_labels=golden_labels) expect_final_precision_A_1 = (1 + 2) / (2 + 2) expect_final_recall_A_1 = (1 + 2) / (1 + 3) expect_final_f1_A_1 = 2 * expect_final_precision_A_1 * expect_final_recall_A_1 / ( expect_final_precision_A_1 + expect_final_recall_A_1) ASSERT.assertAlmostEqual(expect_final_precision_A_1, f1_metric.metric[0][f"{F1Metric.PRECISION}-A"]) ASSERT.assertAlmostEqual(expect_final_recall_A_1, f1_metric.metric[0][f"{F1Metric.RECALL}-A"]) ASSERT.assertAlmostEqual(expect_final_f1_A_1, f1_metric.metric[0][f"{F1Metric.F1}-A"]) expect_final_precision_overall = (2 + 2) / (4 + 2) expect_final_recall_overall = (2 + 2) / (3 + 3) expect_final_f1_overall = 2 * expect_final_precision_overall * expect_final_recall_overall / ( expect_final_precision_overall + expect_final_recall_overall) ASSERT.assertAlmostEqual(expect_final_precision_overall, f1_metric.metric[0][F1Metric.PRECISION_OVERALL]) ASSERT.assertAlmostEqual(expect_final_recall_overall, f1_metric.metric[0][F1Metric.RECALL_OVERALL]) ASSERT.assertAlmostEqual(expect_final_f1_overall, f1_metric.metric[0][F1Metric.F1_OVERALL]) ASSERT.assertEqual(F1Metric.F1_OVERALL, f1_metric.metric[1].name) ASSERT.assertAlmostEqual(f1_metric.metric[1].value, expect_final_f1_overall)
def test_label_f1_metric(): """ 测试 label f1 metric """ predictions = torch.tensor([0, 0, 1, 1, 2, 2, 3, 3, 4, 4]) gold_labels = torch.tensor([0, 1, 1, 2, 2, 3, 3, 4, 4, 1]) labels = [0, 1, 2, 3, 4] f1_metric = LabelF1Metric(labels=labels, label_vocabulary=None) metrics = f1_metric(prediction_labels=predictions, gold_labels=gold_labels, mask=None) logging.debug(json2str(metrics)) ASSERT.assertEqual((len(labels) + 1) * 3, len(metrics)) precision_0 = metrics[f"{LabelF1Metric.PRECISION}-0"] recall_0 = metrics[f"{LabelF1Metric.RECALL}-0"] f1_0 = metrics[f"{LabelF1Metric.F1}-0"] expect_precision_0 = 1. / 2. ASSERT.assertAlmostEqual(expect_precision_0, precision_0) expect_recall_0 = 1. / 1. ASSERT.assertAlmostEqual(expect_recall_0, recall_0) expect_f1_0 = 2. * expect_precision_0 * expect_recall_0 / ( expect_precision_0 + expect_recall_0) ASSERT.assertAlmostEqual(expect_f1_0, f1_0) expect_precision_overall = 5. / 10. expect_recall_overall = 5. / 10 precision_overall = metrics[LabelF1Metric.PRECISION_OVERALL] recall_overall = metrics[LabelF1Metric.RECALL_OVERALL] ASSERT.assertAlmostEqual(expect_precision_overall, precision_overall) ASSERT.assertAlmostEqual(expect_recall_overall, recall_overall) predictions = torch.tensor([0, 2]) gold_labels = torch.tensor([0, 1]) f1_metric(prediction_labels=predictions, gold_labels=gold_labels, mask=None) precision_0 = f1_metric.metric[f"{LabelF1Metric.PRECISION}-0"] recall_0 = f1_metric.metric[f"{LabelF1Metric.RECALL}-0"] f1_0 = f1_metric.metric[f"{LabelF1Metric.F1}-0"] expect_precision_0 = (1. + 1.) / (2. + 1.) ASSERT.assertAlmostEqual(expect_precision_0, precision_0) expect_recall_0 = (1. + 1.) / (1. + 1.) ASSERT.assertAlmostEqual(expect_recall_0, recall_0) expect_f1_0 = 2. * expect_precision_0 * expect_recall_0 / ( expect_precision_0 + expect_recall_0) ASSERT.assertAlmostEqual(expect_f1_0, f1_0)