def test_simple_calculate_columns(self):
query = np.arange(13)
series = np.arange(23)
m = 4
distance_matrix1 = np.arange(200.).reshape((10, 20))
distance_matrix2 = np.ones((10, 20), dtype=np.float)
distance_matrix3 = np.full((10, 20), np.nan, dtype=np.float)
distance_matrix4 = np.full((10, 20), 5., dtype=np.float)
calc = AnytimeCalculator(m, series, query)
calc.add_generator(0, MockGenerator(distance_matrix1))
calc.add_generator(0, MockGenerator(distance_matrix2))
calc.add_generator(0, MockGenerator(distance_matrix3))
calc.add_generator(0, MockGenerator(distance_matrix4))
consumer1 = DistanceMatrix()
consumer2 = DistanceMatrix()
consumer3 = SummingConsumer()
calc.add_consumer([0], consumer1)
calc.add_consumer([3], consumer2)
calc.add_consumer([0, 1, 3], consumer3)
calc.calculate_columns()
npt.assert_equal(consumer1.distance_matrix, distance_matrix1)
npt.assert_equal(consumer2.distance_matrix, distance_matrix4)
npt.assert_equal(
consumer3.distance_matrix,
distance_matrix1 + distance_matrix2 + distance_matrix4)
def test_simple_calculate_diagonals_partial(self):
query = np.arange(13)
series = np.arange(23)
m = 4
distance_matrix1 = np.arange(200.).reshape((10, 20))
distance_matrix2 = np.ones((10, 20), dtype=np.float)
distance_matrix3 = np.full((10, 20), np.nan, dtype=np.float)
distance_matrix4 = np.full((10, 20), 5., dtype=np.float)
summed_matrix = distance_matrix1 + distance_matrix2 + distance_matrix4
max_diag = min(len(query) - m + 1,
len(series) - m + 1) # Maximum length of a diagonal
calc = AnytimeCalculator(m, series, query)
calc.add_generator(0, MockGenerator(distance_matrix1))
calc.add_generator(0, MockGenerator(distance_matrix2))
calc.add_generator(0, MockGenerator(distance_matrix3))
calc.add_generator(0, MockGenerator(distance_matrix4))
consumer1 = DistanceMatrix()
consumer2 = DistanceMatrix()
consumer3 = SummingConsumer()
calc.add_consumer([0], consumer1)
calc.add_consumer([3], consumer2)
calc.add_consumer([0, 1, 3], consumer3)
calc.calculate_diagonals(partial=20)
npt.assert_(20 <= np.count_nonzero(
~np.isnan(consumer1.distance_matrix) < 20 + max_diag))
npt.assert_(20 <= np.count_nonzero(
~np.isnan(consumer2.distance_matrix) < 20 + max_diag))
npt.assert_(20 <= np.count_nonzero(
~np.isnan(consumer3.distance_matrix) < 20 + max_diag))
# For 20 items, at least 3 diagonals are calculated
for diagonal in calc._diagonal_calc_order[:3]:
diag_indices = diag_indices_of(distance_matrix1, diagonal)
npt.assert_equal(consumer1.distance_matrix[diag_indices],
distance_matrix1[diag_indices])
calc.calculate_diagonals(partial=.8)
npt.assert_(160 <= np.count_nonzero(
~np.isnan(consumer1.distance_matrix) < 160 + max_diag))
npt.assert_(160 <= np.count_nonzero(
~np.isnan(consumer2.distance_matrix) < 160 + max_diag))
npt.assert_(160 <= np.count_nonzero(
~np.isnan(consumer3.distance_matrix) < 160 + max_diag))
calc.calculate_diagonals()
npt.assert_(
200 == np.count_nonzero(~np.isnan(consumer1.distance_matrix)))
npt.assert_(
200 == np.count_nonzero(~np.isnan(consumer2.distance_matrix)))
npt.assert_(
200 == np.count_nonzero(~np.isnan(consumer3.distance_matrix)))
npt.assert_equal(consumer1.distance_matrix, distance_matrix1)
npt.assert_equal(consumer2.distance_matrix, distance_matrix4)
npt.assert_equal(consumer3.distance_matrix, summed_matrix)
def test_simple_calculate_columns_partial(self):
query = np.arange(13)
series = np.arange(23)
m = 4
distance_matrix1 = np.arange(200.).reshape((10, 20))
distance_matrix2 = np.ones((10, 20), dtype=np.float)
distance_matrix3 = np.full((10, 20), np.nan, dtype=np.float)
distance_matrix4 = np.full((10, 20), 5., dtype=np.float)
summed_matrix = distance_matrix1 + distance_matrix2 + distance_matrix4
calc = AnytimeCalculator(m, series, query)
calc.add_generator(0, MockGenerator(distance_matrix1))
calc.add_generator(0, MockGenerator(distance_matrix2))
calc.add_generator(0, MockGenerator(distance_matrix3))
calc.add_generator(0, MockGenerator(distance_matrix4))
consumer1 = DistanceMatrix()
consumer2 = DistanceMatrix()
consumer3 = SummingConsumer()
calc.add_consumer([0], consumer1)
calc.add_consumer([3], consumer2)
calc.add_consumer([0, 1, 3], consumer3)
# Calculates [3..12[
calc.calculate_columns(start=3, upto=0.6)
npt.assert_equal(consumer1.distance_matrix,
copy_columns(distance_matrix1, range(3, 12)))
npt.assert_equal(consumer2.distance_matrix,
copy_columns(distance_matrix4, range(3, 12)))
npt.assert_equal(consumer3.distance_matrix,
copy_columns(summed_matrix, range(3, 12)))
# Calculates [12..15[
calc.calculate_columns(upto=15)
npt.assert_equal(consumer1.distance_matrix,
copy_columns(distance_matrix1, range(3, 15)))
npt.assert_equal(consumer2.distance_matrix,
copy_columns(distance_matrix4, range(3, 15)))
npt.assert_equal(consumer3.distance_matrix,
copy_columns(summed_matrix, range(3, 15)))
# Calculates [15..20[
calc.calculate_columns()
npt.assert_equal(consumer1.distance_matrix,
copy_columns(distance_matrix1, range(3, 20)))
npt.assert_equal(consumer2.distance_matrix,
copy_columns(distance_matrix4, range(3, 20)))
npt.assert_equal(consumer3.distance_matrix,
copy_columns(summed_matrix, range(3, 20)))
# Calculates [0..3[
calc.calculate_columns(start=0., upto=3)
npt.assert_equal(consumer1.distance_matrix, distance_matrix1)
npt.assert_equal(consumer2.distance_matrix, distance_matrix4)
npt.assert_equal(consumer3.distance_matrix, summed_matrix)
def test_streaming_data_points_are_filtered_for_different_query_and_series(
self):
mock_gen = MockGenerator(np.arange(12).reshape((3, 4)))
filter_gen = FilterGenerator(
mock_gen,
invalid_data_function=is_not_finite).prepare_streaming(3, 6, 5)
npt.assert_equal(mock_gen.bound_gen.appended_series, [])
npt.assert_equal(mock_gen.bound_gen.appended_query, [])
filter_gen.append_series(np.array([0, np.inf, 1, 2]))
filter_gen.append_query(np.array([np.inf]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 0, 1, 2])
npt.assert_equal(mock_gen.bound_gen.appended_query, [0])
filter_gen.append_series(np.array([3, 4, 5]))
filter_gen.append_query(np.array([0, 1, 2, 3, 4, 5]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 0, 1, 2, 3, 4, 5])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 0, 1, 2, 3, 4, 5])
filter_gen.append_series(np.array([6, 7, np.nan]))
filter_gen.append_query(np.array([6, 7]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 0, 1, 2, 3, 4, 5, 6, 7, 0])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 0, 1, 2, 3, 4, 5, 6, 7])
def test_streaming_self_join_calc_diag_with_invalid_data(self):
mock_gen = MockGenerator(
np.arange(100, dtype=np.float).reshape((10, 10)))
filter_gen = FilterGenerator(
mock_gen,
invalid_data_function=is_not_finite).prepare_streaming(3, 6)
filter_gen.append_series(np.array([np.Inf, 1, 2]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 1, 2])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.inf])
filter_gen.append_series(np.array([3, 4, 5]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 4, 5])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.Inf, 11, 22, 33])
npt.assert_equal(filter_gen.calc_diagonal(-1), [np.Inf, 21, 32])
npt.assert_equal(filter_gen.calc_diagonal(-2), [np.Inf, 31])
npt.assert_equal(filter_gen.calc_diagonal(-3), [np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(1), [np.Inf, 12, 23])
npt.assert_equal(filter_gen.calc_diagonal(2), [np.Inf, 13])
npt.assert_equal(filter_gen.calc_diagonal(3), [np.Inf])
filter_gen.append_series(np.array([6, 7, np.nan]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 4, 5, 6, 7, 0])
npt.assert_equal(filter_gen.calc_diagonal(0), [33, 44, 55, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(-1), [43, 54, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(-2), [53, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(-3), [np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(1), [34, 45, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(2), [35, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(3), [np.Inf])
def test_data_points_are_filtered_for_different_query_and_series(self):
mock_gen = MockGenerator(np.arange(12).reshape((3, 4)))
filter_gen = FilterGenerator(mock_gen,
invalid_data_function=is_not_finite)
filter_gen.prepare(3, np.array([1, np.inf, 3, 4, 5, np.inf]),
np.array([np.inf, 2, 3, 4, np.inf]))
npt.assert_equal(mock_gen.series, [1, 0, 3, 4, 5, 0])
npt.assert_equal(mock_gen.query, [0, 2, 3, 4, 0])
def test_data_points_are_filtered_for_self_join(self):
mock_gen = MockGenerator(np.arange(9).reshape((3, 3)))
filter_gen = FilterGenerator(mock_gen,
invalid_data_function=is_not_finite)
data = np.array([np.inf, 2, 3, 4, np.inf])
filter_gen.prepare(3, data)
npt.assert_equal(mock_gen.series, [0, 2, 3, 4, 0])
self.assertIsNone(mock_gen.query)
def test_streaming_calc_column_with_invalid_data(self):
mock_gen = MockGenerator(
np.arange(100, dtype=np.float).reshape((10, 10)))
filter_gen = FilterGenerator(
mock_gen,
invalid_data_function=is_not_finite).prepare_streaming(3, 6, 5)
filter_gen.append_series(np.array([0, 1, 2, 3, np.Inf]))
filter_gen.append_query(np.array([0, 1, 2]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 1, 2, 3, 0])
npt.assert_equal(mock_gen.bound_gen.appended_query, [0, 1, 2])
npt.assert_equal(filter_gen.calc_column(0), [0])
npt.assert_equal(filter_gen.calc_column(1), [1])
npt.assert_equal(filter_gen.calc_column(2), [np.Inf])
filter_gen.append_series(np.array([4, 5, 6]))
filter_gen.append_query(np.array([3]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6])
npt.assert_equal(mock_gen.bound_gen.appended_query, [0, 1, 2, 3])
npt.assert_equal(filter_gen.calc_column(0), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(1), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(2), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(3), [5, 15])
filter_gen.append_series(np.array([7]))
filter_gen.append_query(np.array([np.Inf, 4]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6, 7])
npt.assert_equal(mock_gen.bound_gen.appended_query, [0, 1, 2, 3, 0, 4])
npt.assert_equal(filter_gen.calc_column(0), [np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(1), [np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(2), [15, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(3), [16, np.Inf, np.Inf])
filter_gen.append_series(np.array([8]))
filter_gen.append_query(np.array([5, 6]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6, 7, 8])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 1, 2, 3, 0, 4, 5, 6])
npt.assert_equal(filter_gen.calc_column(0), [np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(1), [np.Inf, np.Inf, 55])
npt.assert_equal(filter_gen.calc_column(2), [np.Inf, np.Inf, 56])
npt.assert_equal(filter_gen.calc_column(3), [np.Inf, np.Inf, 57])
filter_gen.append_query(np.array([7]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6, 7, 8])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 1, 2, 3, 0, 4, 5, 6, 7])
npt.assert_equal(filter_gen.calc_column(0), [np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(1), [np.Inf, 55, 65])
npt.assert_equal(filter_gen.calc_column(2), [np.Inf, 56, 66])
npt.assert_equal(filter_gen.calc_column(3), [np.Inf, 57, 67])
def test_calc_column_with_invalid_data(self):
mock_gen = MockGenerator(np.arange(12, dtype=np.float).reshape((3, 4)))
filter_gen = FilterGenerator(
mock_gen, invalid_data_function=is_not_finite).prepare(
3, np.array([1, np.inf, 3, 4, 5, 6], dtype=np.float),
np.array([1, 2, 3, 4, np.inf], dtype=np.float))
npt.assert_equal(filter_gen.calc_column(0), [np.inf, np.inf, np.inf])
npt.assert_equal(filter_gen.calc_column(1), [np.inf, np.inf, np.inf])
npt.assert_equal(filter_gen.calc_column(2), [2, 6, np.inf])
npt.assert_equal(filter_gen.calc_column(3), [3, 7, np.inf])
def test_calc_diag_with_invalid_data(self):
mock_gen = MockGenerator(np.arange(12, dtype=np.float).reshape((3, 4)))
filter_gen = FilterGenerator(
mock_gen, invalid_data_function=is_not_finite).prepare(
3, np.array([1, np.inf, 3, 4, 5, 6], dtype=np.float),
np.array([1, 2, 3, 4, np.inf], dtype=np.float))
# i i 2 3
# i i 6 7
# i i i i
npt.assert_equal(filter_gen.calc_diagonal(-2), [np.inf])
npt.assert_equal(filter_gen.calc_diagonal(-1), [np.inf, np.inf])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.inf, np.inf, np.inf])
npt.assert_equal(filter_gen.calc_diagonal(1), [np.inf, 6, np.inf])
npt.assert_equal(filter_gen.calc_diagonal(2), [2, 7])
npt.assert_equal(filter_gen.calc_diagonal(3), [3])
def test_streaming_self_join_calc_column_with_invalid_data(self):
mock_gen = MockGenerator(
np.arange(100, dtype=np.float).reshape((10, 10)))
filter_gen = FilterGenerator(
mock_gen,
invalid_data_function=is_not_finite).prepare_streaming(3, 6)
filter_gen.append_series(np.array([0, 1, 2, 3, np.Inf]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 1, 2, 3, 0])
npt.assert_equal(filter_gen.calc_column(0), [0, 10, np.Inf])
npt.assert_equal(filter_gen.calc_column(1), [1, 11, np.Inf])
npt.assert_equal(filter_gen.calc_column(2), [np.Inf, np.Inf, np.Inf])
filter_gen.append_series(np.array([4, 5, 6]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6])
npt.assert_equal(filter_gen.calc_column(0),
[np.Inf, np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(1),
[np.Inf, np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(2),
[np.Inf, np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(3),
[np.Inf, np.Inf, np.Inf, 55])
filter_gen.append_series(np.array([7]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6, 7])
npt.assert_equal(filter_gen.calc_column(0),
[np.Inf, np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(1),
[np.Inf, np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_column(2), [np.Inf, np.Inf, 55, 65])
npt.assert_equal(filter_gen.calc_column(3), [np.Inf, np.Inf, 56, 66])
filter_gen.append_series(np.array([8, 9]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 4, 5, 6, 7, 8, 9])
npt.assert_equal(filter_gen.calc_column(0), [55, 65, 75, 85])
npt.assert_equal(filter_gen.calc_column(1), [56, 66, 76, 86])
npt.assert_equal(filter_gen.calc_column(2), [57, 67, 77, 87])
npt.assert_equal(filter_gen.calc_column(3), [58, 68, 78, 88])
def test_simple_calculate_self_join_columns(self):
series = np.arange(23)
m = 4
buffer = 2
distance_matrix = np.arange(1., 401.).reshape((20, 20))
calc = AnytimeCalculator(m, series, trivial_match_buffer=buffer)
calc.add_generator(0, MockGenerator(distance_matrix))
consumer = DistanceMatrix()
calc.add_consumer([0], consumer)
calc.calculate_columns()
expected = distance_matrix.copy()
for diag in range(-buffer, buffer + 1):
expected[diag_indices_of(expected, diag)] = np.inf
npt.assert_equal(consumer.distance_matrix, expected)
def test_simple_calculate_self_join_diagonals(self):
series = np.arange(23)
m = 4
buffer = 2
# This test verifies that no values below the main diagonal are used.
distance_matrix = np.triu(
np.arange(1., 401.).reshape((20, 20)), buffer + 1)
calc = AnytimeCalculator(m, series, trivial_match_buffer=buffer)
calc.add_generator(0, MockGenerator(distance_matrix))
consumer = DistanceMatrix()
calc.add_consumer([0], consumer)
npt.assert_array_less(np.full(17, buffer), calc._diagonal_calc_order)
calc.calculate_diagonals()
expected = distance_matrix + distance_matrix.T
expected[expected == 0] = np.nan
npt.assert_equal(consumer.distance_matrix, expected)
def test_streaming_data_points_are_filtered_for_self_join(self):
mock_gen = MockGenerator(np.arange(9).reshape((3, 3)))
filter_gen = FilterGenerator(
mock_gen,
invalid_data_function=is_not_finite).prepare_streaming(3, 6)
npt.assert_equal(mock_gen.bound_gen.appended_series, [])
filter_gen.append_series(np.array([0, 1, 2, 3]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 1, 2, 3])
npt.assert_equal(mock_gen.bound_gen.appended_query, [])
filter_gen.append_series(np.array([np.nan, np.inf, 4, 5, 6]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 0, 4, 5, 6])
npt.assert_equal(mock_gen.bound_gen.appended_query, [])
filter_gen.append_series(np.array([7, 8, 9]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 0, 0, 4, 5, 6, 7, 8, 9])
npt.assert_equal(mock_gen.bound_gen.appended_query, [])
def test_streaming_calculate_columns(self):
dist_matrix = np.arange(10, 100).astype(dtype=np.float).reshape(
(9, 10))
calc = StreamingCalculator(
101, 106, 105) # Distance Matrix in consumer has shape (5, 6)
calc.add_generator(0, MockGenerator(dist_matrix))
consumer = DistanceMatrix()
calc.add_consumer([0], consumer)
nn = np.nan
calc.calculate_columns()
expected = np.array([[nn, nn, nn, nn, nn,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_series(np.zeros((1, 102)))
calc.append_query(np.zeros((1, 103)))
calc.calculate_columns()
expected = np.array([[10, 11, nn, nn, nn,
nn], [20, 21, nn, nn, nn, nn],
[30, 31, nn, nn, nn,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_series(np.zeros((1, 3)))
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns()
expected = np.array([[10, 11, 12, 13, 14,
nn], [20, 21, 22, 23, 24, nn],
[30, 31, 32, 33, 34,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_query(np.zeros((1, 2)))
npt.assert_equal(consumer.distance_matrix, expected)
# Expect same result after full calculation, because query shift does not change last column calculated
calc.calculate_columns()
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns(start=2)
expected = np.array([[10, 11, 12, 13, 14,
nn], [20, 21, 22, 23, 24, nn],
[30, 31, 32, 33, 34,
nn], [nn, nn, 42, 43, 44, nn],
[nn, nn, 52, 53, 54, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_series(np.zeros((1, 3)))
calc.append_query(np.zeros((1, 2)))
expected = np.array([[32, 33, 34, nn, nn,
nn], [42, 43, 44, nn, nn, nn],
[52, 53, 54, nn, nn,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns()
expected = np.array([[32, 33, 34, 35, 36,
37], [42, 43, 44, 45, 46, 47],
[52, 53, 54, 55, 56,
57], [nn, nn, nn, 65, 66, 67],
[nn, nn, nn, 75, 76, 77]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns(0, 1.)
expected = np.array([[32, 33, 34, 35, 36,
37], [42, 43, 44, 45, 46, 47],
[52, 53, 54, 55, 56,
57], [62, 63, 64, 65, 66, 67],
[72, 73, 74, 75, 76, 77]])
npt.assert_equal(consumer.distance_matrix, expected)
def test_streaming_calculate_columns_self_join(self):
dist_matrix = np.arange(10, 100).astype(dtype=np.float).reshape(
(9, 10))
calc = StreamingCalculator(
101, 106, trivial_match_buffer=-1
) # Distance Matrix in consumer has shape (6, 6)
calc.add_generator(0, MockGenerator(dist_matrix))
consumer = DistanceMatrix()
calc.add_consumer([0], consumer)
nn = np.nan
calc.calculate_columns()
expected = np.array([[nn, nn, nn, nn, nn,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn,
nn], [nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_series(np.zeros((1, 104)))
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns()
expected = np.array([[10, 11, 12, 13, nn,
nn], [20, 21, 22, 23, nn, nn],
[30, 31, 32, 33, nn,
nn], [40, 41, 42, 43, nn, nn],
[nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_series(np.zeros((1, 2)))
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns()
expected = np.array([[10, 11, 12, 13, 14,
15], [20, 21, 22, 23, 24, 25],
[30, 31, 32, 33, 34,
35], [40, 41, 42, 43, 44, 45],
[nn, nn, nn, nn, 54, 55],
[nn, nn, nn, nn, 64, 65]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.append_series(np.zeros((1, 2)))
expected = np.array([[32, 33, 34, 35, nn,
nn], [42, 43, 44, 45, nn, nn],
[nn, nn, 54, 55, nn,
nn], [nn, nn, 64, 65, nn, nn],
[nn, nn, nn, nn, nn, nn],
[nn, nn, nn, nn, nn, nn]])
npt.assert_equal(consumer.distance_matrix, expected)
calc.calculate_columns()
expected = np.array([[32, 33, 34, 35, 36,
37], [42, 43, 44, 45, 46, 47],
[nn, nn, 54, 55, 56,
57], [nn, nn, 64, 65, 66, 67],
[nn, nn, nn, nn, 76, 77],
[nn, nn, nn, nn, 86, 87]])
npt.assert_equal(consumer.distance_matrix, expected)
# Correct results when performing full recalculation
calc.calculate_columns(0, 1.)
expected = np.array([[32, 33, 34, 35, 36,
37], [42, 43, 44, 45, 46, 47],
[52, 53, 54, 55, 56,
57], [62, 63, 64, 65, 66, 67],
[72, 73, 74, 75, 76, 77],
[82, 83, 84, 85, 86, 87]])
npt.assert_equal(consumer.distance_matrix, expected)
def test_streaming_calc_diag_with_invalid_data(self):
mock_gen = MockGenerator(
np.arange(100, dtype=np.float).reshape((10, 10)))
filter_gen = FilterGenerator(
mock_gen,
invalid_data_function=is_not_finite).prepare_streaming(3, 6, 5)
filter_gen.append_series(np.array([0, 1, 2]))
filter_gen.append_query(np.array([np.Inf, 1, 2]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 1, 2])
npt.assert_equal(mock_gen.bound_gen.appended_query, [0, 1, 2])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.inf])
filter_gen.append_query(np.array([3, 4, 5]))
npt.assert_equal(mock_gen.bound_gen.appended_series, [0, 1, 2])
npt.assert_equal(mock_gen.bound_gen.appended_query, [0, 1, 2, 3, 4, 5])
npt.assert_equal(filter_gen.calc_diagonal(0), [10])
npt.assert_equal(filter_gen.calc_diagonal(-1), [20])
npt.assert_equal(filter_gen.calc_diagonal(-2), [30])
filter_gen.append_series(np.array([3, 4, np.nan]))
filter_gen.append_query(np.array([np.Inf, 6, 7]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 4, 0])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 1, 2, 3, 4, 5, 0, 6, 7])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(-1), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(-2), [np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(1), [np.Inf, np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(2), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(3), [np.Inf])
filter_gen.append_series(np.array([5, 6, 7, 8]))
filter_gen.append_query(np.array([8]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 4, 0, 5, 6, 7, 8])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 1, 2, 3, 4, 5, 0, 6, 7, 8])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.Inf, np.Inf, 76])
npt.assert_equal(filter_gen.calc_diagonal(-1), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(-2), [np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(1), [np.Inf, np.Inf, 77])
npt.assert_equal(filter_gen.calc_diagonal(2), [np.Inf, np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(3), [np.Inf])
# i i i i
# i i i i
# i i . .
filter_gen.append_series(np.array([9]))
filter_gen.append_query(np.array([9]))
npt.assert_equal(mock_gen.bound_gen.appended_series,
[0, 1, 2, 3, 4, 0, 5, 6, 7, 8, 9])
npt.assert_equal(mock_gen.bound_gen.appended_query,
[0, 1, 2, 3, 4, 5, 0, 6, 7, 8, 9])
npt.assert_equal(filter_gen.calc_diagonal(0), [np.Inf, 76, 87])
npt.assert_equal(filter_gen.calc_diagonal(-1), [np.Inf, 86])
npt.assert_equal(filter_gen.calc_diagonal(-2), [np.Inf])
npt.assert_equal(filter_gen.calc_diagonal(1), [np.Inf, 77, 88])
npt.assert_equal(filter_gen.calc_diagonal(2), [np.Inf, 78])
npt.assert_equal(filter_gen.calc_diagonal(3), [np.Inf])
