def test_creating_buffered_context_check_labels_shape(self):
     with self.assertRaises(ValueError):
         BufferedPathContext.create_from_lists(
             ([[], [], []], ConvertParameters(0, False, {})),
             {
                 FROM_TOKEN: ([[], []], ConvertParameters(0, False, {})),
                 PATH_TYPES: ([[], []], ConvertParameters(0, False, {})),
                 TO_TOKEN: ([[], []], ConvertParameters(0, False, {})),
             },
         )
def _convert_raw_buffer(convert_args: Tuple[List[str], PreprocessingConfig, Vocabulary, str, int]):
    lines, config, vocab, output_folder, buffer_id = convert_args
    labels, from_tokens, path_types, to_tokens = [], [], [], []
    for line in lines:
        label, *path_contexts = line.split()
        label = _parse_token(label, config.split_target)
        labels.append([vocab.label_to_id.get(_l, vocab.label_to_id[UNK]) for _l in label])
        converted_context = [_convert_path_context_to_ids(config.split_names, pc, vocab) for pc in path_contexts]
        from_tokens.append([cc[0] for cc in converted_context])
        path_types.append([cc[1] for cc in converted_context])
        to_tokens.append([cc[2] for cc in converted_context])

    bpc = BufferedPathContext.create_from_lists(
        (labels, ConvertParameters(config.max_target_parts, config.wrap_target, vocab.label_to_id)),
        {
            FROM_TOKEN: (from_tokens, ConvertParameters(config.max_name_parts, config.wrap_name, vocab.token_to_id),),
            PATH_TYPES: (path_types, ConvertParameters(config.max_path_length, config.wrap_path, vocab.type_to_id)),
            TO_TOKEN: (to_tokens, ConvertParameters(config.max_name_parts, config.wrap_name, vocab.token_to_id)),
        },
    )

    with open(path.join(output_folder, DESCRIPTION_FILE), "a") as desc_file:
        n_samples = len(bpc.contexts_per_label)
        n_paths = sum(bpc.contexts_per_label)
        desc_file.write(f"{buffer_id},{BUFFERED_PATH_TEMPLATE.format(buffer_id)},{n_samples},{n_paths}\n")
    bpc.dump(path.join(output_folder, BUFFERED_PATH_TEMPLATE.format(buffer_id)))
    def test_creating_standard_path_context(self):
        token_to_id = {SOS: 0, EOS: 1, PAD: 2}
        type_to_id = {SOS: 1, EOS: 2, PAD: 0}
        label_to_id = {SOS: 2, EOS: 0, PAD: 1}
        labels = [[4], [], [4, 5, 6]]
        from_tokens = [
            [[4], [5, 6]],
            [[], [], []],
            [[6, 5, 4]],
        ]
        path_types = [
            [[4, 5], [6]],
            [[], [], []],
            [[6, 5, 4]],
        ]
        to_tokens = [
            [[6], [4, 5]],
            [[], [], []],
            [[4, 6, 4]],
        ]

        buffered_path_context = BufferedPathContext.create_from_lists(
            (labels, ConvertParameters(3, True, label_to_id)),
            {
                FROM_TOKEN:
                (from_tokens, ConvertParameters(3, False, token_to_id)),
                PATH_TYPES:
                (path_types, ConvertParameters(3, True, type_to_id)),
                TO_TOKEN:
                (to_tokens, ConvertParameters(3, False, token_to_id)),
            },
        )

        true_labels = numpy.array([[2, 2, 2], [4, 0, 4], [0, 1, 5], [1, 1, 6]])
        true_from_tokens = numpy.array([[4, 5, 2, 2, 2, 6], [2, 6, 2, 2, 2, 5],
                                        [2, 2, 2, 2, 2, 4]])
        true_path_types = numpy.array([[1, 1, 1, 1, 1, 1], [4, 6, 2, 2, 2, 6],
                                       [5, 2, 0, 0, 0, 5], [2, 0, 0, 0, 0, 4]])
        true_to_tokens = numpy.array([[6, 4, 2, 2, 2, 4], [2, 5, 2, 2, 2, 6],
                                      [2, 2, 2, 2, 2, 4]])

        self.assertListEqual([2, 3, 1],
                             buffered_path_context.contexts_per_label)
        numpy.testing.assert_array_equal(true_labels,
                                         buffered_path_context.labels)
        numpy.testing.assert_array_equal(
            true_from_tokens, buffered_path_context.contexts[FROM_TOKEN])
        numpy.testing.assert_array_equal(
            true_path_types, buffered_path_context.contexts[PATH_TYPES])
        numpy.testing.assert_array_equal(
            true_to_tokens, buffered_path_context.contexts[TO_TOKEN])