def test_streaming_process_diagonal(self):
dm = DistanceMatrix()
dm.initialise(1, 5, 5)
dm.process_diagonal(0, np.atleast_2d(self.dist_matrix[0, 0]))
diag_ind = diag_indices_of(self.dist_matrix[:3, :3], 1)
dm.process_diagonal(
1, np.atleast_2d(np.atleast_2d(self.dist_matrix[diag_ind])))
expected = np.full((5, 5), np.nan)
expected[0, 0] = self.dist_matrix[0, 0]
expected[0, 1] = self.dist_matrix[0, 1]
expected[1, 2] = self.dist_matrix[1, 2]
npt.assert_equal(dm.distance_matrix, expected)
for diag in range(-4, 5):
diag_ind = diag_indices_of(self.dist_matrix[:5, :5], diag)
dm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
npt.assert_equal(dm.distance_matrix, self.dist_matrix[:5, :5])
dm.shift_query(2)
dm.shift_series(1)
expected = self.dist_matrix[2:7, 1:6].copy()
expected[-2:, :] = np.nan
expected[:, -1:] = np.nan
npt.assert_equal(dm.distance_matrix, expected)
for diag in range(-4, 5):
diag_ind = diag_indices_of(self.dist_matrix[:5, :5], diag)
dm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
npt.assert_equal(dm.distance_matrix, self.dist_matrix[:5, :5])
def test_process_diagonal_partial_calculation(self):
# Ranges selected so that the diagonals have a mix of the number of numbers that are filled in
query_ranges = [
range(0, 2),
range(2, 5),
range(7, 8),
range(8, 9),
range(9, 10)
]
series_ranges = [range(0, 2), range(2, 4), range(4, 10), range(13, 14)]
part_dist_matrix = np.full_like(self.dist_matrix, np.inf)
cdm = ContextualMatrixProfile(
GeneralStaticManager(series_ranges, query_ranges))
self.mock_initialise(cdm)
for diag in range(-8, self.dist_matrix.shape[1], 4):
diag_ind = diag_indices_of(self.dist_matrix, diag)
cdm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
part_dist_matrix[diag_ind] = self.dist_matrix[diag_ind]
correct, correct_qi, correct_si = self.bruteforce_cdm(
part_dist_matrix, query_ranges, series_ranges)
npt.assert_allclose(cdm.distance_matrix, correct)
npt.assert_equal(cdm.match_index_query, correct_qi)
npt.assert_equal(cdm.match_index_series, correct_si)
def test_process_column_lr(self):
l_dist_matrix = self.dist_matrix.copy()
r_dist_matrix = self.dist_matrix.copy()
mmp = MultidimensionalMatrixProfileLR()
self.mock_initialise(mmp)
for diag in range(-self.dist_matrix.shape[1] + 1,
self.dist_matrix.shape[2]):
diag_ind = diag_indices_of(self.dist_matrix[0], diag)
if diag < 0:
l_dist_matrix[:, diag_ind[0], diag_ind[1]] = np.inf
else:
r_dist_matrix[:, diag_ind[0], diag_ind[1]] = np.inf
for column in range(self.dist_matrix.shape[2]):
mmp.process_column(column, self.dist_matrix[:, :, column])
correct_lmp, correct_lindex, correct_ldims = self.bruteforce_mmp(
l_dist_matrix)
correct_rmp, correct_rindex, correct_rdims = self.bruteforce_mmp(
r_dist_matrix)
npt.assert_allclose(mmp.md_matrix_profile_left, correct_lmp)
npt.assert_allclose(mmp.md_matrix_profile_right, correct_rmp)
npt.assert_equal(mmp.md_profile_index_left, correct_lindex)
npt.assert_equal(mmp.md_profile_index_right, correct_rindex)
for i, dims in enumerate(correct_ldims):
sorted_mmp_ldims = np.sort(mmp.md_profile_dimension_left[i],
axis=0)
sorted_mmp_rdims = np.sort(mmp.md_profile_dimension_right[i],
axis=0)
npt.assert_equal(sorted_mmp_ldims, correct_ldims[i])
npt.assert_equal(sorted_mmp_rdims, correct_rdims[i])
def test_simple_calculate_diagonals_partial(self):
query = np.arange(13)
series = np.arange(23)
m = 4
distance_matrix1 = np.arange(200.).reshape((10, 20))
distance_matrix2 = np.ones((10, 20), dtype=np.float)
distance_matrix3 = np.full((10, 20), np.nan, dtype=np.float)
distance_matrix4 = np.full((10, 20), 5., dtype=np.float)
summed_matrix = distance_matrix1 + distance_matrix2 + distance_matrix4
max_diag = min(len(query) - m + 1,
len(series) - m + 1) # Maximum length of a diagonal
calc = AnytimeCalculator(m, series, query)
calc.add_generator(0, MockGenerator(distance_matrix1))
calc.add_generator(0, MockGenerator(distance_matrix2))
calc.add_generator(0, MockGenerator(distance_matrix3))
calc.add_generator(0, MockGenerator(distance_matrix4))
consumer1 = DistanceMatrix()
consumer2 = DistanceMatrix()
consumer3 = SummingConsumer()
calc.add_consumer([0], consumer1)
calc.add_consumer([3], consumer2)
calc.add_consumer([0, 1, 3], consumer3)
calc.calculate_diagonals(partial=20)
npt.assert_(20 <= np.count_nonzero(
~np.isnan(consumer1.distance_matrix) < 20 + max_diag))
npt.assert_(20 <= np.count_nonzero(
~np.isnan(consumer2.distance_matrix) < 20 + max_diag))
npt.assert_(20 <= np.count_nonzero(
~np.isnan(consumer3.distance_matrix) < 20 + max_diag))
# For 20 items, at least 3 diagonals are calculated
for diagonal in calc._diagonal_calc_order[:3]:
diag_indices = diag_indices_of(distance_matrix1, diagonal)
npt.assert_equal(consumer1.distance_matrix[diag_indices],
distance_matrix1[diag_indices])
calc.calculate_diagonals(partial=.8)
npt.assert_(160 <= np.count_nonzero(
~np.isnan(consumer1.distance_matrix) < 160 + max_diag))
npt.assert_(160 <= np.count_nonzero(
~np.isnan(consumer2.distance_matrix) < 160 + max_diag))
npt.assert_(160 <= np.count_nonzero(
~np.isnan(consumer3.distance_matrix) < 160 + max_diag))
calc.calculate_diagonals()
npt.assert_(
200 == np.count_nonzero(~np.isnan(consumer1.distance_matrix)))
npt.assert_(
200 == np.count_nonzero(~np.isnan(consumer2.distance_matrix)))
npt.assert_(
200 == np.count_nonzero(~np.isnan(consumer3.distance_matrix)))
npt.assert_equal(consumer1.distance_matrix, distance_matrix1)
npt.assert_equal(consumer2.distance_matrix, distance_matrix4)
npt.assert_equal(consumer3.distance_matrix, summed_matrix)
def test_process_diagonal(self):
dm = DistanceMatrix()
self.mock_initialise(dm)
for diag in range(-self.dist_matrix.shape[0] + 1,
self.dist_matrix.shape[1]):
diag_ind = diag_indices_of(self.dist_matrix, diag)
dm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
npt.assert_equal(dm.distance_matrix, self.dist_matrix)
def test_process_diagonal_single_threshold(self):
threshold = 2.83
counter = ThresholdCounter(threshold)
counter.initialise(1, self.dm.shape[0], self.dm.shape[1])
for diag in range(-self.dm.shape[0] + 1, self.dm.shape[1]):
diag_ind = diag_indices_of(self.dm, diag)
counter.process_diagonal(diag, np.atleast_2d(self.dm[diag_ind]))
npt.assert_equal(counter.counts,
self.bruteforce_count(self.dm, [threshold]))
def test_process_diagonal_multiple_thresholds(self):
thresholds = [-1, 2.12, 2.83, 6]
counter = ThresholdCounter(thresholds)
counter.initialise(1, self.dm.shape[0], self.dm.shape[1])
for diag in range(-self.dm.shape[0] + 1, self.dm.shape[1]):
diag_ind = diag_indices_of(self.dm, diag)
counter.process_diagonal(diag, np.atleast_2d(self.dm[diag_ind]))
npt.assert_equal(counter.counts,
self.bruteforce_count(self.dm, thresholds))
def test_process_diagonal_partial_calculation(self):
dm = DistanceMatrix()
self.mock_initialise(dm)
correct = np.full_like(self.dist_matrix, np.nan, dtype=np.float)
for diag in range(-8, self.dist_matrix.shape[1], 3):
diag_ind = diag_indices_of(self.dist_matrix, diag)
dm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
correct[diag_ind] = self.dist_matrix[diag_ind]
npt.assert_equal(dm.distance_matrix, correct)
def test_diag_indices_of_rect(self):
data = np.array([[1, 2, 3], [4, 5, 6]])
npt.assert_equal(data[diag_indices_of(data, -2)], [])
npt.assert_equal(data[diag_indices_of(data, -1)], [4])
npt.assert_equal(data[diag_indices_of(data, 0)], [1, 5])
npt.assert_equal(data[diag_indices_of(data, 1)], [2, 6])
npt.assert_equal(data[diag_indices_of(data, 2)], [3])
npt.assert_equal(data[diag_indices_of(data, 3)], [])
def test_reservoid_sampling_diagonals_spreads_over_lower_upper_diag(self):
dm = np.zeros((500, 499))
self.mplr = MatrixProfileLRReservoir(random_seed=0)
self.mplr.initialise(1, dm.shape[0], dm.shape[1])
for i in range(-dm.shape[0] + 1, dm.shape[1]):
self.mplr.process_diagonal(
i, np.atleast_2d(dm[diag_indices_of(dm, i)]))
# Check correct value of matrix profile
npt.assert_equal(self.mplr.matrix_profile(), np.zeros(dm.shape[1]))
# Check uniform distribution of selected indices - spread between upper and lower diagonal
mp_index = self.mplr.profile_index()
w = dm.shape[1]
below_diag = mp_index[:w] > np.arange(w)
npt.assert_allclose(np.count_nonzero(below_diag), w / 2, rtol=0.05)
def test_process_diagonal(self):
mmp = MultidimensionalMatrixProfileLR()
self.mock_initialise(mmp)
for diag in range(-self.dist_matrix.shape[1] + 1,
self.dist_matrix.shape[2]):
diag_ind = diag_indices_of(self.dist_matrix[0], diag)
mmp.process_diagonal(diag, self.dist_matrix[:, diag_ind[0],
diag_ind[1]])
correct_mp, correct_index, correct_dims = self.bruteforce_mmp(
self.dist_matrix)
npt.assert_allclose(mmp.md_matrix_profile(), correct_mp)
npt.assert_equal(mmp.md_profile_index(), correct_index)
for i, dims in enumerate(correct_dims):
sorted_mmp_dims = np.sort(mmp.md_profile_dimensions()[i], axis=0)
npt.assert_equal(sorted_mmp_dims, dims)
def test_process_diagonal(self):
for diag in range(-self.dm.shape[0] + 1, self.dm.shape[1]):
diag_ind = diag_indices_of(self.dm, diag)
self.mplr.process_diagonal(diag, np.atleast_2d(self.dm[diag_ind]))
rmp, rmpi = brute_force_mp(
fill_diagonals(self.dm, range(0, self.dm.shape[1]), np.inf))
lmp, lmpi = brute_force_mp(
fill_diagonals(self.dm, range(-self.dm.shape[1] + 1, 0), np.inf))
mp, mpi = brute_force_mp(self.dm)
npt.assert_allclose(self.mplr.matrix_profile_right, rmp)
npt.assert_allclose(self.mplr.matrix_profile_left, lmp)
npt.assert_equal(self.mplr.profile_index_right, rmpi)
npt.assert_equal(self.mplr.profile_index_left, lmpi)
npt.assert_allclose(self.mplr.matrix_profile(), mp)
npt.assert_equal(self.mplr.profile_index(), mpi)
def test_reservoir_sampling_diagonals_spreads_over_rows(self):
dm = np.zeros((4, 1000))
self.mplr = MatrixProfileLRReservoir(random_seed=0)
self.mplr.initialise(1, dm.shape[0], dm.shape[1])
for i in range(-dm.shape[0] + 1, dm.shape[1]):
self.mplr.process_diagonal(
i, np.atleast_2d(dm[diag_indices_of(dm, i)]))
# Check correct value of matrix profile
npt.assert_equal(self.mplr.matrix_profile(), np.zeros(dm.shape[1]))
# Check uniform distribution of selected indices
mp_index = self.mplr.profile_index()
for i in range(dm.shape[0]):
npt.assert_allclose(np.count_nonzero(mp_index == i),
dm.shape[1] / dm.shape[0],
rtol=0.1)
def test_simple_calculate_self_join_columns(self):
series = np.arange(23)
m = 4
buffer = 2
distance_matrix = np.arange(1., 401.).reshape((20, 20))
calc = AnytimeCalculator(m, series, trivial_match_buffer=buffer)
calc.add_generator(0, MockGenerator(distance_matrix))
consumer = DistanceMatrix()
calc.add_consumer([0], consumer)
calc.calculate_columns()
expected = distance_matrix.copy()
for diag in range(-buffer, buffer + 1):
expected[diag_indices_of(expected, diag)] = np.inf
npt.assert_equal(consumer.distance_matrix, expected)
def test_process_diagonal_context_falls_outside_distancematrix(self):
query_ranges = [range(0, 8), range(8, 16), range(20, 30)]
series_ranges = [range(0, 10), range(10, 20), range(30, 40)]
correct, correct_qi, correct_si = self.bruteforce_cdm(
self.dist_matrix, query_ranges, series_ranges)
cdm = ContextualMatrixProfile(
GeneralStaticManager(series_ranges, query_ranges))
self.mock_initialise(cdm)
for diag in range(-self.dist_matrix.shape[0] + 1,
self.dist_matrix.shape[1]):
diag_ind = diag_indices_of(self.dist_matrix, diag)
cdm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
npt.assert_allclose(cdm.distance_matrix, correct)
npt.assert_equal(cdm.match_index_query, correct_qi)
npt.assert_equal(cdm.match_index_series, correct_si)
def test_process_diagonal_complex_ranges(self):
# Overlapping ranges and contexts consisting of multiple ranges
query_ranges = [
range(0, 10),
range(1, 5),
range(1, 2),
range(4, 5), [range(1, 2), range(3, 4),
range(7, 9)]
]
series_ranges = [
range(0, 2),
range(1, 3),
range(2, 4),
range(3, 6),
range(4, 8),
range(4, 10), [range(0, 3),
range(3, 5),
range(13, 15)]
]
correct, correct_qi, correct_si = self.bruteforce_cdm(
self.dist_matrix, query_ranges, series_ranges)
cdm = ContextualMatrixProfile(
GeneralStaticManager(series_ranges, query_ranges))
self.mock_initialise(cdm)
for diag in range(-self.dist_matrix.shape[0] + 1,
self.dist_matrix.shape[1]):
diag_ind = diag_indices_of(self.dist_matrix, diag)
cdm.process_diagonal(diag,
np.atleast_2d(self.dist_matrix[diag_ind]))
npt.assert_allclose(cdm.distance_matrix, correct)
npt.assert_equal(cdm.match_index_query, correct_qi)
npt.assert_equal(cdm.match_index_series, correct_si)
def calc_diagonal(self, diag):
view = self._dist_matrix[
max(self._q_index, 0):max(self._q_index + self._q_win, 0),
max(self._s_index, 0):max(self._s_index + self._s_win, 0)]
return view[diag_indices_of(view, diag)]
def fill_diagonals(matrix, diagonals, value):
result = matrix.copy()
for diag in diagonals:
result[diag_indices_of(result, diag)] = value
return result
def diag3(array, diag):
"""
Returns the diagonal indices for a 3D array, taking all elements from the first dimension.
"""
return (slice(None), *diag_indices_of(array[0], diag))
def process_diagonal(self, diagonal_index, values):
indices = diag_indices_of(self.distance_matrix, diagonal_index)
self.distance_matrix[indices] = np.sum(values, axis=0)
def test_streaming_process_diagonal(self):
cdm = ContextualMatrixProfile(MockContextManager())
cdm.initialise(1, 4, 6)
v = self.dist_matrix[0:4, 0:6]
cdm.process_diagonal(0, np.atleast_2d(v[diag_indices_of(v, 0)]))
cdm.process_diagonal(2, np.atleast_2d(v[diag_indices_of(v, 2)]))
cdm.process_diagonal(-1, np.atleast_2d(v[diag_indices_of(v, -1)]))
npt.assert_equal(cdm.distance_matrix,
[[3.1, 0.5, 8., np.Inf], [np.Inf, 5.8, 1., np.Inf]])
npt.assert_equal(cdm.match_index_query,
[[2, 0, 2, -1], [-1, 3, 3, -1]])
npt.assert_equal(cdm.match_index_series,
[[1, 2, 4, -1], [-1, 2, 5, -1]])
cdm.shift_series(1)
cdm.shift_query(2)
v = self.dist_matrix[2:6, 1:7]
npt.assert_equal(cdm.distance_matrix,
[[3.1, 0.5, 8., np.Inf], [np.Inf, 5.8, 1., np.Inf]])
npt.assert_equal(cdm.match_index_query,
[[2, 0, 2, -1], [-1, 3, 3, -1]])
npt.assert_equal(cdm.match_index_series,
[[1, 2, 4, -1], [-1, 2, 5, -1]])
cdm.process_diagonal(5, np.atleast_2d(v[diag_indices_of(v, 5)]))
cdm.process_diagonal(1, np.atleast_2d(v[diag_indices_of(v, 1)]))
cdm.process_diagonal(0, np.atleast_2d(v[diag_indices_of(v, 0)]))
cdm.process_diagonal(-3, np.atleast_2d(v[diag_indices_of(v, -3)]))
npt.assert_equal(cdm.distance_matrix,
[[3.1, 0.5, 8., 2.7], [8.1, 5.8, 1., np.Inf]])
npt.assert_equal(cdm.match_index_query, [[2, 0, 2, 2], [5, 3, 3, -1]])
npt.assert_equal(cdm.match_index_series, [[1, 2, 4, 6], [1, 2, 5, -1]])
cdm.shift_series(1)
cdm.shift_query(2)
v = self.dist_matrix[4:8, 2:8]
npt.assert_equal(
cdm.distance_matrix,
[[5.8, 1., np.Inf, np.Inf], [np.Inf, np.Inf, np.Inf, np.Inf]])
npt.assert_equal(cdm.match_index_query,
[[3, 3, -1, -1], [-1, -1, -1, -1]])
npt.assert_equal(cdm.match_index_series,
[[2, 5, -1, -1], [-1, -1, -1, -1]])
cdm.process_diagonal(-2, np.atleast_2d(v[diag_indices_of(v, -2)]))
cdm.process_diagonal(0, np.atleast_2d(v[diag_indices_of(v, 0)]))
cdm.process_diagonal(1, np.atleast_2d(v[diag_indices_of(v, 1)]))
cdm.process_diagonal(2, np.atleast_2d(v[diag_indices_of(v, 2)]))
cdm.process_diagonal(3, np.atleast_2d(v[diag_indices_of(v, 3)]))
cdm.process_diagonal(4, np.atleast_2d(v[diag_indices_of(v, 4)]))
npt.assert_equal(cdm.distance_matrix,
[[1.2, 1., 1.2, np.Inf], [1.2, 3.7, 2.5, np.Inf]])
npt.assert_equal(cdm.match_index_query, [[6, 3, 4, -1], [6, 6, 7, -1]])
npt.assert_equal(cdm.match_index_series,
[[2, 5, 6, -1], [2, 4, 6, -1]])
