def convolved_1d(self, stride, kernel_size) -> Iterable['DataContainer']: """ Returns an iterator that iterates through batches of the DataContainer. :param stride: step size for the convolution operation :param kernel_size: :return: an iterator of DataContainer :rtype: Iterable[DataContainer] .. seealso:: `<https://github.com/guillaume-chevalier/python-conv-lib>`_ """ conv_current_ids = convolved_1d(stride=stride, iterable=self.current_ids, kernel_size=kernel_size, include_incomplete_pass=True) conv_data_inputs = convolved_1d(stride=stride, iterable=self.data_inputs, kernel_size=kernel_size, include_incomplete_pass=True) conv_expected_outputs = convolved_1d(stride=stride, iterable=self.expected_outputs, kernel_size=kernel_size, include_incomplete_pass=True) for current_ids, data_inputs, expected_outputs in zip(conv_current_ids, conv_data_inputs, conv_expected_outputs): for i, (ci, di, eo) in enumerate(zip(current_ids, data_inputs, expected_outputs)): if di is None: current_ids = current_ids[:i] data_inputs = data_inputs[:i] expected_outputs = expected_outputs[:i] break yield DataContainer( data_inputs=data_inputs, current_ids=current_ids, summary_id=self.summary_id, expected_outputs=expected_outputs, sub_data_containers=self.sub_data_containers )
def test_trivial_loop(self): expected = tuple(range(7)) result = [] for kernel_hover in convolved_1d(expected, kernel_size=1, padding=0, stride=1): result.append(*kernel_hover) result = tuple(result) self.assertEqual(expected, result)
def test_seven_kernel_of_two_and_stride_two(self): expected = ([0, 1], [2, 3], [4, 5], [6, None]) result = [] for kernel_hover in convolved_1d(list(range(7)), kernel_size=2, padding=0, stride=2): result.append(kernel_hover) result = tuple(result) self.assertEqual(expected, result)
def test_seven_for_with_stride_two(self): expected = tuple(range(0, 7, 2)) result = [] for kernel_hover in convolved_1d(list(range(7)), kernel_size=1, padding=0, stride=2): result.append(*kernel_hover) result = tuple(result) self.assertEqual(expected, result)
def test_seven_kernel_of_two(self): expected = tuple([a, b] for a, b in zip(list(range(0, 6)), list(range(1, 7)))) result = [] for kernel_hover in convolved_1d(list(range(7)), kernel_size=2, padding=0, stride=1): result.append(kernel_hover) result = tuple(result) self.assertEqual(expected, result)
def test_seven_for_with_padding_one(self): expected = tuple([42] + list(range(0, 7)) + [42]) result = [] for kernel_hover in convolved_1d(list(range(7)), kernel_size=1, padding=1, stride=1, default_value=42): result.append(*kernel_hover) result = tuple(result) self.assertEqual(expected, result)
def test_big_kernel_loops_never(self): expected = [] result = [] for kernel_hover in convolved_1d(expected, kernel_size=7, padding=0, stride=7, default_value=None, include_incomplete_pass=True): result.append(kernel_hover) self.assertEqual(len(result), 0)
def test_seven_kernel_of_three_and_stride_three_and_padding_three(self): expected = ([None, None, None], [0, 1, 2], [3, 4, 5], [6, None, None], [None, None, None]) result = [] for kernel_hover in convolved_1d(list(range(7)), kernel_size=3, padding=3, stride=3): result.append(kernel_hover) result = tuple(result) self.assertEqual(expected, result)
def convolve_test_set(X_test, y_test, kernel_size, stride=None): """ Split chunks of text into chunks of maximum length of "kernel_size" chars each. """ if stride is None: stride = kernel_size convolved_X_test = [] convolved_y_test = [] for x_full, y in zip(X_test, y_test): for x in convolved_1d(x_full, kernel_size=kernel_size, stride=stride, padding='VALID', default_value=' '): convolved_X_test.append("".join(x)) convolved_y_test.append(y) return convolved_X_test, convolved_y_test
def transform(self, x, y=None): """ Would convert ["test!", "hey"] to [["tes", "est", "st!"], ["hey"]]. """ x = [ [ "".join(ngram) # for ngram in convolved_1d( # example: list(convolved_1d("test!", kernel_size=3, stride=1, padding='VALID')) # output : [['t', 'e', 's'], ['e', 's', 't'], ['s', 't', '!']] sentence, kernel_size=self.ngram, stride=self.stride, padding='VALID') ] for sentence in x ] return x