def test_remove_stationary_noise(): """ The data has large amount of noise - switching between SD and LA every 10 seconds. It starts from SD, then noise, later it returns to SD. We test that LA is ignored """ data = remove_duplicate_points(_get_test_data())[561:576] fixed_data = apply_filter(data, VelocityOutliersFilter()) print(len(fixed_data)) assert len(fixed_data) == 11 stationary_point = [0, 33.004964, -117.060207] distances = np.array([distance(stationary_point, d) for d in fixed_data]) assert (distances < 246.6).all()
def merge_trajectories(raw_trajectories_folder, results_file): onlyfiles = [f for f in os.listdir(raw_trajectories_folder) if os.path.isfile(os.path.join(raw_trajectories_folder, f))] contents = [] for name in onlyfiles: full_name = os.path.join(raw_trajectories_folder, name) contents.append(read_compressed_trajectory(full_name)) contents.sort(key=lambda p: p[0, 0], reverse=False) merged_traj = np.vstack(contents) merged_traj = apply_filter(merged_traj, DuplicateTimeFilter()) dt = extract_delta_time(merged_traj) assert((dt > 0).all()) write_compressed_trajectory(merged_traj, results_file)
def test_remove_duplicate_readings(): data = _get_test_data() velocities = compute_velocities(data) number_of_duplicates = np.sum(np.isnan(velocities)) fixed_data = remove_duplicate_points(data) fixed_velocities = compute_velocities(fixed_data) assert np.sum(np.isnan(fixed_velocities)) == 0 # check that we deleted duplicates only print(data.shape[0]) print(fixed_data.shape[0]) print(number_of_duplicates) assert fixed_data.shape[0] == data.shape[0] - number_of_duplicates print(fixed_data.shape[0], data.shape[0]) fixed_data_processor = apply_filter(data, DuplicateTimeFilter()) assert (fixed_data == fixed_data_processor).all()
def test_remove_stationary_noise_return_to_stable(): """ The data has large amount of noise - switching between SD and LA every 10 seconds. It starts from the noisy point, later it returns to SD. Here we test that even if data starts with noisy value, we still converge to stable point """ data = remove_duplicate_points(_get_test_data())[563:576] fixed_data = apply_filter(data, VelocityOutliersFilter(85)) stationary_point = [0, 33.004964, -117.060207] distances = np.array([distance(stationary_point, d) for d in fixed_data]) print(fixed_data) assert len(fixed_data) == 7 # filter converged after few steps assert (distances[:4] > 157000).all() assert (distances[4:] < 246.6).all()
def test_remove_outliers(): np.set_printoptions(suppress=True) data = remove_stationary_points(remove_duplicate_points(_get_test_data())) velocity_threshold = 85.0 fixed_data = apply_filter(data, VelocityOutliersFilter(velocity_threshold)) # check that data has outliers in velocity and distance velocities = compute_velocities(data) outliers = np.where(velocities * ms_to_mph > 85)[0] assert len(outliers) > 0 assert np.amax(extract_delta_dist(data)) > 157900 # no large velocities left velocities = compute_velocities(fixed_data) assert np.amax(velocities) * ms_to_mph < velocity_threshold assert np.amax(extract_delta_dist(fixed_data)) < 330 # we expect this number of point to be removed print(data.shape[0] - fixed_data.shape[0]) assert data.shape[0] - fixed_data.shape[0] == 5
def test_finding_paths_with_state(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') filename = s3.download("_testing/testing_merged_1", folder) full_trajectory = read_compressed_trajectory(filename) pieces = [] pieces.append(full_trajectory[0:480]) pieces.append(full_trajectory[480:2000]) pieces.append(full_trajectory[2000:3000]) pieces.append(full_trajectory[3000:4000]) pieces.append(full_trajectory[4000:]) filtered_pieces = [] endpoints = [] findendpoints_state = None filter_state = None for piece in pieces: vel_filter = VelocityOutliersFilter() if filter_state is not None: vel_filter.set_state(filter_state) filter = FilterChain([DuplicateTimeFilter(), vel_filter]) filtered_piece = apply_filter(piece, filter) filter_state = vel_filter.get_state() filtered_pieces.append(filtered_piece) finder = FindEndpoints(endpoints=endpoints) if findendpoints_state is not None: finder.set_state(findendpoints_state) for d in filtered_piece: finder.process(d) endpoints = finder.get_endpoints() findendpoints_state = finder.get_state() data = np.vstack(filtered_pieces) assert(_get_point_index(data, endpoints[0]) == 479) assert(_get_point_index(data, endpoints[1]) == 670) AtoB_paths = [] BtoA_paths = [] finder_current_route = [] finder_endpoint_index = None for piece in filtered_pieces: finder = RoutesFinder(endpoints) finder.set_state(finder_current_route, finder_endpoint_index) for d in piece: finder.process(d) finder_current_route, finder_endpoint_index = finder.get_state() AtoB_paths_piece, BtoA_paths_piece = finder.get_routes() AtoB_paths += AtoB_paths_piece BtoA_paths += BtoA_paths_piece AtoB_paths_indices = _extract_indices(data, AtoB_paths) BtoA_paths_indices = _extract_indices(data, BtoA_paths) print(AtoB_paths_indices) print(BtoA_paths_indices) assert(AtoB_paths_indices == [[488, 656], [947, 1117], [1364, 1549], [2216, 2401], [2630, 2898], [4400, 4526]]) assert(BtoA_paths_indices == [[134, 455], [683, 887], [1141, 1316], [1582, 1783], [2429, 2597], [3975, 4170]])
def test_finding_paths_with_state_2(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') pieces = [] pieces_keys = [ "_testing/testing_sequence0/data/14-09-2015_09-15-01_PDT", "_testing/testing_sequence0/data/14-09-2015_11-03-24_PDT", "_testing/testing_sequence0/data/14-09-2015_13-49-55_PDT", "_testing/testing_sequence0/data/14-09-2015_18-20-13_PDT", "_testing/testing_sequence0/data/14-09-2015_19-59-23_PDT", "_testing/testing_sequence0/data/15-09-2015_09-32-15_PDT", "_testing/testing_sequence0/data/15-09-2015_22-31-21_PDT" ] for k in pieces_keys: filename = s3.download(k, folder) pieces.append(read_compressed_trajectory(filename)) filtered_pieces = [] endpoints = [] findendpoints_state = None filter_state = None for pi, piece in enumerate(pieces): vel_filter = VelocityOutliersFilter() if filter_state is not None: vel_filter.set_state(filter_state) filter = FilterChain([DuplicateTimeFilter(), vel_filter]) filtered_piece = apply_filter(piece, filter) filter_state = vel_filter.get_state() filtered_pieces.append(filtered_piece) finder = FindEndpoints(endpoints=endpoints) if findendpoints_state is not None: finder.set_state(findendpoints_state) for i, d in enumerate(filtered_piece): finder.process(d) endpoints = finder.get_endpoints() findendpoints_state = finder.get_state() data = np.vstack(filtered_pieces) print(trajectory_point_to_str(data, _get_point_index(data, endpoints[0]))) print(trajectory_point_to_str(data, _get_point_index(data, endpoints[1]))) assert(len(endpoints) == 2) assert(_get_point_index(data, endpoints[0]) == 5) assert(_get_point_index(data, endpoints[1]) == 122) AtoB_paths = [] BtoA_paths = [] finder_current_route = [] finder_endpoint_index = None for piece in filtered_pieces: finder = RoutesFinder(endpoints) finder.set_state(finder_current_route, finder_endpoint_index) for d in piece: finder.process(d) finder_current_route, finder_endpoint_index = finder.get_state() AtoB_paths_piece, BtoA_paths_piece = finder.get_routes() AtoB_paths += AtoB_paths_piece BtoA_paths += BtoA_paths_piece AtoB_paths_indices = _extract_indices(data, AtoB_paths) BtoA_paths_indices = _extract_indices(data, BtoA_paths) print(AtoB_paths_indices) print(BtoA_paths_indices) assert(AtoB_paths_indices == [[11, 111], [556, 730]]) assert(BtoA_paths_indices == [[288, 387]])