def test_trajectory_reader_2(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') filename = s3.download("_testing/testing_raw_0", folder) trajectory = read_compressed_trajectory(filename) write_compressed_trajectory(trajectory, os.path.join(folder, 'trajectory_copy')) trajectory_copy = read_compressed_trajectory(os.path.join(folder, 'trajectory_copy')) assert((trajectory == trajectory_copy).all())
def fix_single_trajectory(): artifacts = os.path.join(os.path.dirname(__file__), 'artifacts') to_fix_path = os.path.join(artifacts, 'raw', '03-08-2015_09-58-01') fixed_path = fix_trajectory_timezone(to_fix_path, os.path.join(artifacts, 'fixed')) original_trajectory = read_compressed_trajectory(to_fix_path, with_timezone=False) fixed_trajectory = read_compressed_trajectory(fixed_path, with_timezone=True) print(num_to_date_str_converter(original_trajectory[0, 0], with_timezone=False)) print(num_to_date_str_converter(fixed_trajectory[0, 0], with_timezone=True))
def fix_trajectory_timezone(filename, folder_to_put, change_filename=True): ''' Add timezone to the trajectory ''' # http://www.saltycrane.com/blog/2009/05/converting-time-zones-datetime-objects-python/ traj = read_compressed_trajectory(filename, with_timezone=False) tz = timezone('US/Pacific') for i in range(traj.shape[0]): d = num2date(traj[i, 0]) d = d.replace(tzinfo = None) d = tz.localize(d) traj[i, 0] = date2num(d) result_filename = os.path.split(filename)[-1] if change_filename: file_date = num2date(date_str_to_num_converter_no_timezone(result_filename)) file_date = file_date.replace(tzinfo = None) file_date = tz.localize(file_date) result_filename = num_to_date_str_converter(date2num(file_date), with_timezone=True) resulting_path = os.path.join(folder_to_put, result_filename) write_compressed_trajectory(traj, os.path.join(folder_to_put, result_filename), with_timezone=True) return resulting_path
def _get_test_data(): folder = tempfile.mkdtemp() s3 = S3("driverpete-storage") filename = s3.download("_testing/testing_raw_0", folder) data = read_compressed_trajectory(filename) # add few time duplicates data[54] = data[53] data[100] = data[99] # add few distance duplicates data[23][1:] = data[22][1:] data[40][1:] = data[39][1:] data[60][1:] = data[59][1:] return data
def merge_trajectories(raw_trajectories_folder, results_file): onlyfiles = [f for f in os.listdir(raw_trajectories_folder) if os.path.isfile(os.path.join(raw_trajectories_folder, f))] contents = [] for name in onlyfiles: full_name = os.path.join(raw_trajectories_folder, name) contents.append(read_compressed_trajectory(full_name)) contents.sort(key=lambda p: p[0, 0], reverse=False) merged_traj = np.vstack(contents) merged_traj = apply_filter(merged_traj, DuplicateTimeFilter()) dt = extract_delta_time(merged_traj) assert((dt > 0).all()) write_compressed_trajectory(merged_traj, results_file)
def test_find_endpoints_procedural(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') filename = s3.download("_testing/testing_merged_1", folder) data = filter_gps_data(read_compressed_trajectory(filename)) endpoints_indices_batch = find_endpoints_batch(data) assert(len(endpoints_indices_batch) == 2) print(endpoints_indices_batch) assert(endpoints_indices_batch == [479, 670]) endpoints = find_endpoints(data) assert(len(endpoints) == 2) assert((data[endpoints_indices_batch[0]] == endpoints[0]).all()) assert((data[endpoints_indices_batch[1]] == endpoints[1]).all())
def process_gps_data(filename): #stationary_distance_threshold=120 # clean up data data = read_compressed_trajectory(filename) print("Length of data: %d" % len(data)) data = filter_gps_data(data) print("Length of filtered data: %d" % len(data)) endpoints = find_endpoints(data) print("Found %d endpoints:" % len(endpoints)) for u in endpoints: print(trajectory_point_to_str([u], 0)) assert(len(endpoints) == 2) AtoB_paths_data, BtoA_paths_data = find_routes(data, endpoints, verbose=False) def _extract_indices(data, paths): result = [] for p in paths: indices = [(data[:, 0] == p[0, 0]).nonzero()[0][0], (data[:, 0] == p[p.shape[0]-1, 0]).nonzero()[0][0]] result.append(indices) return result AtoB_paths_indices = _extract_indices(data, AtoB_paths_data) BtoA_paths_indices = _extract_indices(data, BtoA_paths_data) print(AtoB_paths_indices) print(BtoA_paths_indices) AtoB_paths = [Path(d) for d in AtoB_paths_data] BtoA_paths = [Path(d) for d in BtoA_paths_data] AtoB_paths = sorted(AtoB_paths, key=lambda x: x.get_duration()) BtoA_paths = sorted(BtoA_paths, key=lambda x: x.get_duration()) print("ATOB") for p in AtoB_paths: print(p.get_duration()/60, str(p.start_time())) p.show(str(p.get_duration()/60)) print("BTOA") for p in BtoA_paths: print(p.get_duration()/60, str(p.start_time())) p.show(str(p.get_duration()/60))
def test_finding_paths(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') filename = s3.download("_testing/testing_merged_1", folder) data = filter_gps_data(read_compressed_trajectory(filename)) endpoints = find_endpoints(data) assert(len(endpoints) == 2) print(trajectory_point_to_str(data, _get_point_index(data, endpoints[0]))) print(trajectory_point_to_str(data, _get_point_index(data, endpoints[1]))) assert(_get_point_index(data, endpoints[0]) == 479) assert(_get_point_index(data, endpoints[1]) == 670) AtoB_paths, BtoA_paths = find_routes(data, endpoints, verbose=False) AtoB_paths_indices = _extract_indices(data, AtoB_paths) BtoA_paths_indices = _extract_indices(data, BtoA_paths) print(AtoB_paths_indices) print(BtoA_paths_indices) assert(AtoB_paths_indices == [[488, 656], [947, 1117], [1364, 1549], [2216, 2401], [2630, 2898], [4400, 4526]]) assert(BtoA_paths_indices == [[134, 455], [683, 887], [1141, 1316], [1582, 1783], [2429, 2597], [3975, 4170]])
def test_finding_paths_with_state(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') filename = s3.download("_testing/testing_merged_1", folder) full_trajectory = read_compressed_trajectory(filename) pieces = [] pieces.append(full_trajectory[0:480]) pieces.append(full_trajectory[480:2000]) pieces.append(full_trajectory[2000:3000]) pieces.append(full_trajectory[3000:4000]) pieces.append(full_trajectory[4000:]) filtered_pieces = [] endpoints = [] findendpoints_state = None filter_state = None for piece in pieces: vel_filter = VelocityOutliersFilter() if filter_state is not None: vel_filter.set_state(filter_state) filter = FilterChain([DuplicateTimeFilter(), vel_filter]) filtered_piece = apply_filter(piece, filter) filter_state = vel_filter.get_state() filtered_pieces.append(filtered_piece) finder = FindEndpoints(endpoints=endpoints) if findendpoints_state is not None: finder.set_state(findendpoints_state) for d in filtered_piece: finder.process(d) endpoints = finder.get_endpoints() findendpoints_state = finder.get_state() data = np.vstack(filtered_pieces) assert(_get_point_index(data, endpoints[0]) == 479) assert(_get_point_index(data, endpoints[1]) == 670) AtoB_paths = [] BtoA_paths = [] finder_current_route = [] finder_endpoint_index = None for piece in filtered_pieces: finder = RoutesFinder(endpoints) finder.set_state(finder_current_route, finder_endpoint_index) for d in piece: finder.process(d) finder_current_route, finder_endpoint_index = finder.get_state() AtoB_paths_piece, BtoA_paths_piece = finder.get_routes() AtoB_paths += AtoB_paths_piece BtoA_paths += BtoA_paths_piece AtoB_paths_indices = _extract_indices(data, AtoB_paths) BtoA_paths_indices = _extract_indices(data, BtoA_paths) print(AtoB_paths_indices) print(BtoA_paths_indices) assert(AtoB_paths_indices == [[488, 656], [947, 1117], [1364, 1549], [2216, 2401], [2630, 2898], [4400, 4526]]) assert(BtoA_paths_indices == [[134, 455], [683, 887], [1141, 1316], [1582, 1783], [2429, 2597], [3975, 4170]])
def test_finding_paths_with_state_2(): folder = tempfile.mkdtemp() s3 = S3('driverpete-storage') pieces = [] pieces_keys = [ "_testing/testing_sequence0/data/14-09-2015_09-15-01_PDT", "_testing/testing_sequence0/data/14-09-2015_11-03-24_PDT", "_testing/testing_sequence0/data/14-09-2015_13-49-55_PDT", "_testing/testing_sequence0/data/14-09-2015_18-20-13_PDT", "_testing/testing_sequence0/data/14-09-2015_19-59-23_PDT", "_testing/testing_sequence0/data/15-09-2015_09-32-15_PDT", "_testing/testing_sequence0/data/15-09-2015_22-31-21_PDT" ] for k in pieces_keys: filename = s3.download(k, folder) pieces.append(read_compressed_trajectory(filename)) filtered_pieces = [] endpoints = [] findendpoints_state = None filter_state = None for pi, piece in enumerate(pieces): vel_filter = VelocityOutliersFilter() if filter_state is not None: vel_filter.set_state(filter_state) filter = FilterChain([DuplicateTimeFilter(), vel_filter]) filtered_piece = apply_filter(piece, filter) filter_state = vel_filter.get_state() filtered_pieces.append(filtered_piece) finder = FindEndpoints(endpoints=endpoints) if findendpoints_state is not None: finder.set_state(findendpoints_state) for i, d in enumerate(filtered_piece): finder.process(d) endpoints = finder.get_endpoints() findendpoints_state = finder.get_state() data = np.vstack(filtered_pieces) print(trajectory_point_to_str(data, _get_point_index(data, endpoints[0]))) print(trajectory_point_to_str(data, _get_point_index(data, endpoints[1]))) assert(len(endpoints) == 2) assert(_get_point_index(data, endpoints[0]) == 5) assert(_get_point_index(data, endpoints[1]) == 122) AtoB_paths = [] BtoA_paths = [] finder_current_route = [] finder_endpoint_index = None for piece in filtered_pieces: finder = RoutesFinder(endpoints) finder.set_state(finder_current_route, finder_endpoint_index) for d in piece: finder.process(d) finder_current_route, finder_endpoint_index = finder.get_state() AtoB_paths_piece, BtoA_paths_piece = finder.get_routes() AtoB_paths += AtoB_paths_piece BtoA_paths += BtoA_paths_piece AtoB_paths_indices = _extract_indices(data, AtoB_paths) BtoA_paths_indices = _extract_indices(data, BtoA_paths) print(AtoB_paths_indices) print(BtoA_paths_indices) assert(AtoB_paths_indices == [[11, 111], [556, 730]]) assert(BtoA_paths_indices == [[288, 387]])
for t in trajectories_to_fix: fix_trajectory_timezone(os.path.join(artifacts, 'raw', t), os.path.join(artifacts, 'fixed')) def fix_single_trajectory(): artifacts = os.path.join(os.path.dirname(__file__), 'artifacts') to_fix_path = os.path.join(artifacts, 'raw', '03-08-2015_09-58-01') fixed_path = fix_trajectory_timezone(to_fix_path, os.path.join(artifacts, 'fixed')) original_trajectory = read_compressed_trajectory(to_fix_path, with_timezone=False) fixed_trajectory = read_compressed_trajectory(fixed_path, with_timezone=True) print(num_to_date_str_converter(original_trajectory[0, 0], with_timezone=False)) print(num_to_date_str_converter(fixed_trajectory[0, 0], with_timezone=True)) if __name__ == '__main__': artifacts = os.path.join(os.path.dirname(__file__), 'artifacts') to_fix_path = os.path.join(artifacts, '_testing', 'testing_raw_0') fixed_path = fix_trajectory_timezone(to_fix_path, os.path.join(artifacts, 'fixed'), change_filename=False) original_trajectory = read_compressed_trajectory(to_fix_path, with_timezone=False) fixed_trajectory = read_compressed_trajectory(fixed_path, with_timezone=True) print(num_to_date_str_converter(original_trajectory[0, 0], with_timezone=False)) print(num_to_date_str_converter(fixed_trajectory[0, 0], with_timezone=True))