def test_streaming_process_diagonal(self): dm = DistanceMatrix() dm.initialise(1, 5, 5) dm.process_diagonal(0, np.atleast_2d(self.dist_matrix[0, 0])) diag_ind = diag_indices_of(self.dist_matrix[:3, :3], 1) dm.process_diagonal( 1, np.atleast_2d(np.atleast_2d(self.dist_matrix[diag_ind]))) expected = np.full((5, 5), np.nan) expected[0, 0] = self.dist_matrix[0, 0] expected[0, 1] = self.dist_matrix[0, 1] expected[1, 2] = self.dist_matrix[1, 2] npt.assert_equal(dm.distance_matrix, expected) for diag in range(-4, 5): diag_ind = diag_indices_of(self.dist_matrix[:5, :5], diag) dm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) npt.assert_equal(dm.distance_matrix, self.dist_matrix[:5, :5]) dm.shift_query(2) dm.shift_series(1) expected = self.dist_matrix[2:7, 1:6].copy() expected[-2:, :] = np.nan expected[:, -1:] = np.nan npt.assert_equal(dm.distance_matrix, expected) for diag in range(-4, 5): diag_ind = diag_indices_of(self.dist_matrix[:5, :5], diag) dm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) npt.assert_equal(dm.distance_matrix, self.dist_matrix[:5, :5])
def test_process_diagonal_partial_calculation(self): # Ranges selected so that the diagonals have a mix of the number of numbers that are filled in query_ranges = [ range(0, 2), range(2, 5), range(7, 8), range(8, 9), range(9, 10) ] series_ranges = [range(0, 2), range(2, 4), range(4, 10), range(13, 14)] part_dist_matrix = np.full_like(self.dist_matrix, np.inf) cdm = ContextualMatrixProfile( GeneralStaticManager(series_ranges, query_ranges)) self.mock_initialise(cdm) for diag in range(-8, self.dist_matrix.shape[1], 4): diag_ind = diag_indices_of(self.dist_matrix, diag) cdm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) part_dist_matrix[diag_ind] = self.dist_matrix[diag_ind] correct, correct_qi, correct_si = self.bruteforce_cdm( part_dist_matrix, query_ranges, series_ranges) npt.assert_allclose(cdm.distance_matrix, correct) npt.assert_equal(cdm.match_index_query, correct_qi) npt.assert_equal(cdm.match_index_series, correct_si)
def test_process_column_lr(self): l_dist_matrix = self.dist_matrix.copy() r_dist_matrix = self.dist_matrix.copy() mmp = MultidimensionalMatrixProfileLR() self.mock_initialise(mmp) for diag in range(-self.dist_matrix.shape[1] + 1, self.dist_matrix.shape[2]): diag_ind = diag_indices_of(self.dist_matrix[0], diag) if diag < 0: l_dist_matrix[:, diag_ind[0], diag_ind[1]] = np.inf else: r_dist_matrix[:, diag_ind[0], diag_ind[1]] = np.inf for column in range(self.dist_matrix.shape[2]): mmp.process_column(column, self.dist_matrix[:, :, column]) correct_lmp, correct_lindex, correct_ldims = self.bruteforce_mmp( l_dist_matrix) correct_rmp, correct_rindex, correct_rdims = self.bruteforce_mmp( r_dist_matrix) npt.assert_allclose(mmp.md_matrix_profile_left, correct_lmp) npt.assert_allclose(mmp.md_matrix_profile_right, correct_rmp) npt.assert_equal(mmp.md_profile_index_left, correct_lindex) npt.assert_equal(mmp.md_profile_index_right, correct_rindex) for i, dims in enumerate(correct_ldims): sorted_mmp_ldims = np.sort(mmp.md_profile_dimension_left[i], axis=0) sorted_mmp_rdims = np.sort(mmp.md_profile_dimension_right[i], axis=0) npt.assert_equal(sorted_mmp_ldims, correct_ldims[i]) npt.assert_equal(sorted_mmp_rdims, correct_rdims[i])
def test_simple_calculate_diagonals_partial(self): query = np.arange(13) series = np.arange(23) m = 4 distance_matrix1 = np.arange(200.).reshape((10, 20)) distance_matrix2 = np.ones((10, 20), dtype=np.float) distance_matrix3 = np.full((10, 20), np.nan, dtype=np.float) distance_matrix4 = np.full((10, 20), 5., dtype=np.float) summed_matrix = distance_matrix1 + distance_matrix2 + distance_matrix4 max_diag = min(len(query) - m + 1, len(series) - m + 1) # Maximum length of a diagonal calc = AnytimeCalculator(m, series, query) calc.add_generator(0, MockGenerator(distance_matrix1)) calc.add_generator(0, MockGenerator(distance_matrix2)) calc.add_generator(0, MockGenerator(distance_matrix3)) calc.add_generator(0, MockGenerator(distance_matrix4)) consumer1 = DistanceMatrix() consumer2 = DistanceMatrix() consumer3 = SummingConsumer() calc.add_consumer([0], consumer1) calc.add_consumer([3], consumer2) calc.add_consumer([0, 1, 3], consumer3) calc.calculate_diagonals(partial=20) npt.assert_(20 <= np.count_nonzero( ~np.isnan(consumer1.distance_matrix) < 20 + max_diag)) npt.assert_(20 <= np.count_nonzero( ~np.isnan(consumer2.distance_matrix) < 20 + max_diag)) npt.assert_(20 <= np.count_nonzero( ~np.isnan(consumer3.distance_matrix) < 20 + max_diag)) # For 20 items, at least 3 diagonals are calculated for diagonal in calc._diagonal_calc_order[:3]: diag_indices = diag_indices_of(distance_matrix1, diagonal) npt.assert_equal(consumer1.distance_matrix[diag_indices], distance_matrix1[diag_indices]) calc.calculate_diagonals(partial=.8) npt.assert_(160 <= np.count_nonzero( ~np.isnan(consumer1.distance_matrix) < 160 + max_diag)) npt.assert_(160 <= np.count_nonzero( ~np.isnan(consumer2.distance_matrix) < 160 + max_diag)) npt.assert_(160 <= np.count_nonzero( ~np.isnan(consumer3.distance_matrix) < 160 + max_diag)) calc.calculate_diagonals() npt.assert_( 200 == np.count_nonzero(~np.isnan(consumer1.distance_matrix))) npt.assert_( 200 == np.count_nonzero(~np.isnan(consumer2.distance_matrix))) npt.assert_( 200 == np.count_nonzero(~np.isnan(consumer3.distance_matrix))) npt.assert_equal(consumer1.distance_matrix, distance_matrix1) npt.assert_equal(consumer2.distance_matrix, distance_matrix4) npt.assert_equal(consumer3.distance_matrix, summed_matrix)
def test_process_diagonal(self): dm = DistanceMatrix() self.mock_initialise(dm) for diag in range(-self.dist_matrix.shape[0] + 1, self.dist_matrix.shape[1]): diag_ind = diag_indices_of(self.dist_matrix, diag) dm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) npt.assert_equal(dm.distance_matrix, self.dist_matrix)
def test_process_diagonal_single_threshold(self): threshold = 2.83 counter = ThresholdCounter(threshold) counter.initialise(1, self.dm.shape[0], self.dm.shape[1]) for diag in range(-self.dm.shape[0] + 1, self.dm.shape[1]): diag_ind = diag_indices_of(self.dm, diag) counter.process_diagonal(diag, np.atleast_2d(self.dm[diag_ind])) npt.assert_equal(counter.counts, self.bruteforce_count(self.dm, [threshold]))
def test_process_diagonal_multiple_thresholds(self): thresholds = [-1, 2.12, 2.83, 6] counter = ThresholdCounter(thresholds) counter.initialise(1, self.dm.shape[0], self.dm.shape[1]) for diag in range(-self.dm.shape[0] + 1, self.dm.shape[1]): diag_ind = diag_indices_of(self.dm, diag) counter.process_diagonal(diag, np.atleast_2d(self.dm[diag_ind])) npt.assert_equal(counter.counts, self.bruteforce_count(self.dm, thresholds))
def test_process_diagonal_partial_calculation(self): dm = DistanceMatrix() self.mock_initialise(dm) correct = np.full_like(self.dist_matrix, np.nan, dtype=np.float) for diag in range(-8, self.dist_matrix.shape[1], 3): diag_ind = diag_indices_of(self.dist_matrix, diag) dm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) correct[diag_ind] = self.dist_matrix[diag_ind] npt.assert_equal(dm.distance_matrix, correct)
def test_diag_indices_of_rect(self): data = np.array([[1, 2, 3], [4, 5, 6]]) npt.assert_equal(data[diag_indices_of(data, -2)], []) npt.assert_equal(data[diag_indices_of(data, -1)], [4]) npt.assert_equal(data[diag_indices_of(data, 0)], [1, 5]) npt.assert_equal(data[diag_indices_of(data, 1)], [2, 6]) npt.assert_equal(data[diag_indices_of(data, 2)], [3]) npt.assert_equal(data[diag_indices_of(data, 3)], [])
def test_reservoid_sampling_diagonals_spreads_over_lower_upper_diag(self): dm = np.zeros((500, 499)) self.mplr = MatrixProfileLRReservoir(random_seed=0) self.mplr.initialise(1, dm.shape[0], dm.shape[1]) for i in range(-dm.shape[0] + 1, dm.shape[1]): self.mplr.process_diagonal( i, np.atleast_2d(dm[diag_indices_of(dm, i)])) # Check correct value of matrix profile npt.assert_equal(self.mplr.matrix_profile(), np.zeros(dm.shape[1])) # Check uniform distribution of selected indices - spread between upper and lower diagonal mp_index = self.mplr.profile_index() w = dm.shape[1] below_diag = mp_index[:w] > np.arange(w) npt.assert_allclose(np.count_nonzero(below_diag), w / 2, rtol=0.05)
def test_process_diagonal(self): mmp = MultidimensionalMatrixProfileLR() self.mock_initialise(mmp) for diag in range(-self.dist_matrix.shape[1] + 1, self.dist_matrix.shape[2]): diag_ind = diag_indices_of(self.dist_matrix[0], diag) mmp.process_diagonal(diag, self.dist_matrix[:, diag_ind[0], diag_ind[1]]) correct_mp, correct_index, correct_dims = self.bruteforce_mmp( self.dist_matrix) npt.assert_allclose(mmp.md_matrix_profile(), correct_mp) npt.assert_equal(mmp.md_profile_index(), correct_index) for i, dims in enumerate(correct_dims): sorted_mmp_dims = np.sort(mmp.md_profile_dimensions()[i], axis=0) npt.assert_equal(sorted_mmp_dims, dims)
def test_process_diagonal(self): for diag in range(-self.dm.shape[0] + 1, self.dm.shape[1]): diag_ind = diag_indices_of(self.dm, diag) self.mplr.process_diagonal(diag, np.atleast_2d(self.dm[diag_ind])) rmp, rmpi = brute_force_mp( fill_diagonals(self.dm, range(0, self.dm.shape[1]), np.inf)) lmp, lmpi = brute_force_mp( fill_diagonals(self.dm, range(-self.dm.shape[1] + 1, 0), np.inf)) mp, mpi = brute_force_mp(self.dm) npt.assert_allclose(self.mplr.matrix_profile_right, rmp) npt.assert_allclose(self.mplr.matrix_profile_left, lmp) npt.assert_equal(self.mplr.profile_index_right, rmpi) npt.assert_equal(self.mplr.profile_index_left, lmpi) npt.assert_allclose(self.mplr.matrix_profile(), mp) npt.assert_equal(self.mplr.profile_index(), mpi)
def test_reservoir_sampling_diagonals_spreads_over_rows(self): dm = np.zeros((4, 1000)) self.mplr = MatrixProfileLRReservoir(random_seed=0) self.mplr.initialise(1, dm.shape[0], dm.shape[1]) for i in range(-dm.shape[0] + 1, dm.shape[1]): self.mplr.process_diagonal( i, np.atleast_2d(dm[diag_indices_of(dm, i)])) # Check correct value of matrix profile npt.assert_equal(self.mplr.matrix_profile(), np.zeros(dm.shape[1])) # Check uniform distribution of selected indices mp_index = self.mplr.profile_index() for i in range(dm.shape[0]): npt.assert_allclose(np.count_nonzero(mp_index == i), dm.shape[1] / dm.shape[0], rtol=0.1)
def test_simple_calculate_self_join_columns(self): series = np.arange(23) m = 4 buffer = 2 distance_matrix = np.arange(1., 401.).reshape((20, 20)) calc = AnytimeCalculator(m, series, trivial_match_buffer=buffer) calc.add_generator(0, MockGenerator(distance_matrix)) consumer = DistanceMatrix() calc.add_consumer([0], consumer) calc.calculate_columns() expected = distance_matrix.copy() for diag in range(-buffer, buffer + 1): expected[diag_indices_of(expected, diag)] = np.inf npt.assert_equal(consumer.distance_matrix, expected)
def test_process_diagonal_context_falls_outside_distancematrix(self): query_ranges = [range(0, 8), range(8, 16), range(20, 30)] series_ranges = [range(0, 10), range(10, 20), range(30, 40)] correct, correct_qi, correct_si = self.bruteforce_cdm( self.dist_matrix, query_ranges, series_ranges) cdm = ContextualMatrixProfile( GeneralStaticManager(series_ranges, query_ranges)) self.mock_initialise(cdm) for diag in range(-self.dist_matrix.shape[0] + 1, self.dist_matrix.shape[1]): diag_ind = diag_indices_of(self.dist_matrix, diag) cdm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) npt.assert_allclose(cdm.distance_matrix, correct) npt.assert_equal(cdm.match_index_query, correct_qi) npt.assert_equal(cdm.match_index_series, correct_si)
def test_process_diagonal_complex_ranges(self): # Overlapping ranges and contexts consisting of multiple ranges query_ranges = [ range(0, 10), range(1, 5), range(1, 2), range(4, 5), [range(1, 2), range(3, 4), range(7, 9)] ] series_ranges = [ range(0, 2), range(1, 3), range(2, 4), range(3, 6), range(4, 8), range(4, 10), [range(0, 3), range(3, 5), range(13, 15)] ] correct, correct_qi, correct_si = self.bruteforce_cdm( self.dist_matrix, query_ranges, series_ranges) cdm = ContextualMatrixProfile( GeneralStaticManager(series_ranges, query_ranges)) self.mock_initialise(cdm) for diag in range(-self.dist_matrix.shape[0] + 1, self.dist_matrix.shape[1]): diag_ind = diag_indices_of(self.dist_matrix, diag) cdm.process_diagonal(diag, np.atleast_2d(self.dist_matrix[diag_ind])) npt.assert_allclose(cdm.distance_matrix, correct) npt.assert_equal(cdm.match_index_query, correct_qi) npt.assert_equal(cdm.match_index_series, correct_si)
def calc_diagonal(self, diag): view = self._dist_matrix[ max(self._q_index, 0):max(self._q_index + self._q_win, 0), max(self._s_index, 0):max(self._s_index + self._s_win, 0)] return view[diag_indices_of(view, diag)]
def fill_diagonals(matrix, diagonals, value): result = matrix.copy() for diag in diagonals: result[diag_indices_of(result, diag)] = value return result
def diag3(array, diag): """ Returns the diagonal indices for a 3D array, taking all elements from the first dimension. """ return (slice(None), *diag_indices_of(array[0], diag))
def process_diagonal(self, diagonal_index, values): indices = diag_indices_of(self.distance_matrix, diagonal_index) self.distance_matrix[indices] = np.sum(values, axis=0)
def test_streaming_process_diagonal(self): cdm = ContextualMatrixProfile(MockContextManager()) cdm.initialise(1, 4, 6) v = self.dist_matrix[0:4, 0:6] cdm.process_diagonal(0, np.atleast_2d(v[diag_indices_of(v, 0)])) cdm.process_diagonal(2, np.atleast_2d(v[diag_indices_of(v, 2)])) cdm.process_diagonal(-1, np.atleast_2d(v[diag_indices_of(v, -1)])) npt.assert_equal(cdm.distance_matrix, [[3.1, 0.5, 8., np.Inf], [np.Inf, 5.8, 1., np.Inf]]) npt.assert_equal(cdm.match_index_query, [[2, 0, 2, -1], [-1, 3, 3, -1]]) npt.assert_equal(cdm.match_index_series, [[1, 2, 4, -1], [-1, 2, 5, -1]]) cdm.shift_series(1) cdm.shift_query(2) v = self.dist_matrix[2:6, 1:7] npt.assert_equal(cdm.distance_matrix, [[3.1, 0.5, 8., np.Inf], [np.Inf, 5.8, 1., np.Inf]]) npt.assert_equal(cdm.match_index_query, [[2, 0, 2, -1], [-1, 3, 3, -1]]) npt.assert_equal(cdm.match_index_series, [[1, 2, 4, -1], [-1, 2, 5, -1]]) cdm.process_diagonal(5, np.atleast_2d(v[diag_indices_of(v, 5)])) cdm.process_diagonal(1, np.atleast_2d(v[diag_indices_of(v, 1)])) cdm.process_diagonal(0, np.atleast_2d(v[diag_indices_of(v, 0)])) cdm.process_diagonal(-3, np.atleast_2d(v[diag_indices_of(v, -3)])) npt.assert_equal(cdm.distance_matrix, [[3.1, 0.5, 8., 2.7], [8.1, 5.8, 1., np.Inf]]) npt.assert_equal(cdm.match_index_query, [[2, 0, 2, 2], [5, 3, 3, -1]]) npt.assert_equal(cdm.match_index_series, [[1, 2, 4, 6], [1, 2, 5, -1]]) cdm.shift_series(1) cdm.shift_query(2) v = self.dist_matrix[4:8, 2:8] npt.assert_equal( cdm.distance_matrix, [[5.8, 1., np.Inf, np.Inf], [np.Inf, np.Inf, np.Inf, np.Inf]]) npt.assert_equal(cdm.match_index_query, [[3, 3, -1, -1], [-1, -1, -1, -1]]) npt.assert_equal(cdm.match_index_series, [[2, 5, -1, -1], [-1, -1, -1, -1]]) cdm.process_diagonal(-2, np.atleast_2d(v[diag_indices_of(v, -2)])) cdm.process_diagonal(0, np.atleast_2d(v[diag_indices_of(v, 0)])) cdm.process_diagonal(1, np.atleast_2d(v[diag_indices_of(v, 1)])) cdm.process_diagonal(2, np.atleast_2d(v[diag_indices_of(v, 2)])) cdm.process_diagonal(3, np.atleast_2d(v[diag_indices_of(v, 3)])) cdm.process_diagonal(4, np.atleast_2d(v[diag_indices_of(v, 4)])) npt.assert_equal(cdm.distance_matrix, [[1.2, 1., 1.2, np.Inf], [1.2, 3.7, 2.5, np.Inf]]) npt.assert_equal(cdm.match_index_query, [[6, 3, 4, -1], [6, 6, 7, -1]]) npt.assert_equal(cdm.match_index_series, [[2, 5, 6, -1], [2, 4, 6, -1]])