def convolved_1d(self, stride, kernel_size) -> Iterable['DataContainer']:
"""
Returns an iterator that iterates through batches of the DataContainer.
:param stride: step size for the convolution operation
:param kernel_size:
:return: an iterator of DataContainer
:rtype: Iterable[DataContainer]
.. seealso::
`<https://github.com/guillaume-chevalier/python-conv-lib>`_
"""
conv_current_ids = convolved_1d(stride=stride, iterable=self.current_ids, kernel_size=kernel_size,
include_incomplete_pass=True)
conv_data_inputs = convolved_1d(stride=stride, iterable=self.data_inputs, kernel_size=kernel_size,
include_incomplete_pass=True)
conv_expected_outputs = convolved_1d(stride=stride, iterable=self.expected_outputs, kernel_size=kernel_size,
include_incomplete_pass=True)
for current_ids, data_inputs, expected_outputs in zip(conv_current_ids, conv_data_inputs,
conv_expected_outputs):
for i, (ci, di, eo) in enumerate(zip(current_ids, data_inputs, expected_outputs)):
if di is None:
current_ids = current_ids[:i]
data_inputs = data_inputs[:i]
expected_outputs = expected_outputs[:i]
break
yield DataContainer(
data_inputs=data_inputs,
current_ids=current_ids,
summary_id=self.summary_id,
expected_outputs=expected_outputs,
sub_data_containers=self.sub_data_containers
)
def test_trivial_loop(self):
expected = tuple(range(7))
result = []
for kernel_hover in convolved_1d(expected,
kernel_size=1,
padding=0,
stride=1):
result.append(*kernel_hover)
result = tuple(result)
self.assertEqual(expected, result)
def test_seven_kernel_of_two_and_stride_two(self):
expected = ([0, 1], [2, 3], [4, 5], [6, None])
result = []
for kernel_hover in convolved_1d(list(range(7)),
kernel_size=2,
padding=0,
stride=2):
result.append(kernel_hover)
result = tuple(result)
self.assertEqual(expected, result)
def test_seven_for_with_stride_two(self):
expected = tuple(range(0, 7, 2))
result = []
for kernel_hover in convolved_1d(list(range(7)),
kernel_size=1,
padding=0,
stride=2):
result.append(*kernel_hover)
result = tuple(result)
self.assertEqual(expected, result)
def test_seven_kernel_of_two(self):
expected = tuple([a, b]
for a, b in zip(list(range(0, 6)), list(range(1, 7))))
result = []
for kernel_hover in convolved_1d(list(range(7)),
kernel_size=2,
padding=0,
stride=1):
result.append(kernel_hover)
result = tuple(result)
self.assertEqual(expected, result)
def test_seven_for_with_padding_one(self):
expected = tuple([42] + list(range(0, 7)) + [42])
result = []
for kernel_hover in convolved_1d(list(range(7)),
kernel_size=1,
padding=1,
stride=1,
default_value=42):
result.append(*kernel_hover)
result = tuple(result)
self.assertEqual(expected, result)
def test_big_kernel_loops_never(self):
expected = []
result = []
for kernel_hover in convolved_1d(expected,
kernel_size=7,
padding=0,
stride=7,
default_value=None,
include_incomplete_pass=True):
result.append(kernel_hover)
self.assertEqual(len(result), 0)
def test_seven_kernel_of_three_and_stride_three_and_padding_three(self):
expected = ([None, None,
None], [0, 1, 2], [3, 4, 5], [6, None,
None], [None, None, None])
result = []
for kernel_hover in convolved_1d(list(range(7)),
kernel_size=3,
padding=3,
stride=3):
result.append(kernel_hover)
result = tuple(result)
self.assertEqual(expected, result)
def convolve_test_set(X_test, y_test, kernel_size, stride=None):
"""
Split chunks of text into chunks of maximum length of "kernel_size" chars each.
"""
if stride is None:
stride = kernel_size
convolved_X_test = []
convolved_y_test = []
for x_full, y in zip(X_test, y_test):
for x in convolved_1d(x_full, kernel_size=kernel_size, stride=stride, padding='VALID', default_value=' '):
convolved_X_test.append("".join(x))
convolved_y_test.append(y)
return convolved_X_test, convolved_y_test
def transform(self, x, y=None):
"""
Would convert ["test!", "hey"] to [["tes", "est", "st!"], ["hey"]].
"""
x = [
[
"".join(ngram) #
for ngram in convolved_1d(
# example: list(convolved_1d("test!", kernel_size=3, stride=1, padding='VALID'))
# output : [['t', 'e', 's'], ['e', 's', 't'], ['s', 't', '!']]
sentence,
kernel_size=self.ngram,
stride=self.stride,
padding='VALID')
] for sentence in x
]
return x
