def test_trajectory_reader_2():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
filename = s3.download("_testing/testing_raw_0", folder)
trajectory = read_compressed_trajectory(filename)
write_compressed_trajectory(trajectory, os.path.join(folder, 'trajectory_copy'))
trajectory_copy = read_compressed_trajectory(os.path.join(folder, 'trajectory_copy'))
assert((trajectory == trajectory_copy).all())
def fix_single_trajectory():
artifacts = os.path.join(os.path.dirname(__file__), 'artifacts')
to_fix_path = os.path.join(artifacts, 'raw', '03-08-2015_09-58-01')
fixed_path = fix_trajectory_timezone(to_fix_path,
os.path.join(artifacts, 'fixed'))
original_trajectory = read_compressed_trajectory(to_fix_path, with_timezone=False)
fixed_trajectory = read_compressed_trajectory(fixed_path, with_timezone=True)
print(num_to_date_str_converter(original_trajectory[0, 0], with_timezone=False))
print(num_to_date_str_converter(fixed_trajectory[0, 0], with_timezone=True))
def fix_trajectory_timezone(filename, folder_to_put, change_filename=True):
'''
Add timezone to the trajectory
'''
# http://www.saltycrane.com/blog/2009/05/converting-time-zones-datetime-objects-python/
traj = read_compressed_trajectory(filename, with_timezone=False)
tz = timezone('US/Pacific')
for i in range(traj.shape[0]):
d = num2date(traj[i, 0])
d = d.replace(tzinfo = None)
d = tz.localize(d)
traj[i, 0] = date2num(d)
result_filename = os.path.split(filename)[-1]
if change_filename:
file_date = num2date(date_str_to_num_converter_no_timezone(result_filename))
file_date = file_date.replace(tzinfo = None)
file_date = tz.localize(file_date)
result_filename = num_to_date_str_converter(date2num(file_date), with_timezone=True)
resulting_path = os.path.join(folder_to_put, result_filename)
write_compressed_trajectory(traj, os.path.join(folder_to_put, result_filename), with_timezone=True)
return resulting_path
def _get_test_data():
folder = tempfile.mkdtemp()
s3 = S3("driverpete-storage")
filename = s3.download("_testing/testing_raw_0", folder)
data = read_compressed_trajectory(filename)
# add few time duplicates
data[54] = data[53]
data[100] = data[99]
# add few distance duplicates
data[23][1:] = data[22][1:]
data[40][1:] = data[39][1:]
data[60][1:] = data[59][1:]
return data
def merge_trajectories(raw_trajectories_folder, results_file):
onlyfiles = [f for f in os.listdir(raw_trajectories_folder)
if os.path.isfile(os.path.join(raw_trajectories_folder, f))]
contents = []
for name in onlyfiles:
full_name = os.path.join(raw_trajectories_folder, name)
contents.append(read_compressed_trajectory(full_name))
contents.sort(key=lambda p: p[0, 0], reverse=False)
merged_traj = np.vstack(contents)
merged_traj = apply_filter(merged_traj, DuplicateTimeFilter())
dt = extract_delta_time(merged_traj)
assert((dt > 0).all())
write_compressed_trajectory(merged_traj, results_file)
def test_find_endpoints_procedural():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
filename = s3.download("_testing/testing_merged_1", folder)
data = filter_gps_data(read_compressed_trajectory(filename))
endpoints_indices_batch = find_endpoints_batch(data)
assert(len(endpoints_indices_batch) == 2)
print(endpoints_indices_batch)
assert(endpoints_indices_batch == [479, 670])
endpoints = find_endpoints(data)
assert(len(endpoints) == 2)
assert((data[endpoints_indices_batch[0]] == endpoints[0]).all())
assert((data[endpoints_indices_batch[1]] == endpoints[1]).all())
def process_gps_data(filename):
#stationary_distance_threshold=120
# clean up data
data = read_compressed_trajectory(filename)
print("Length of data: %d" % len(data))
data = filter_gps_data(data)
print("Length of filtered data: %d" % len(data))
endpoints = find_endpoints(data)
print("Found %d endpoints:" % len(endpoints))
for u in endpoints:
print(trajectory_point_to_str([u], 0))
assert(len(endpoints) == 2)
AtoB_paths_data, BtoA_paths_data = find_routes(data, endpoints, verbose=False)
def _extract_indices(data, paths):
result = []
for p in paths:
indices = [(data[:, 0] == p[0, 0]).nonzero()[0][0],
(data[:, 0] == p[p.shape[0]-1, 0]).nonzero()[0][0]]
result.append(indices)
return result
AtoB_paths_indices = _extract_indices(data, AtoB_paths_data)
BtoA_paths_indices = _extract_indices(data, BtoA_paths_data)
print(AtoB_paths_indices)
print(BtoA_paths_indices)
AtoB_paths = [Path(d) for d in AtoB_paths_data]
BtoA_paths = [Path(d) for d in BtoA_paths_data]
AtoB_paths = sorted(AtoB_paths, key=lambda x: x.get_duration())
BtoA_paths = sorted(BtoA_paths, key=lambda x: x.get_duration())
print("ATOB")
for p in AtoB_paths:
print(p.get_duration()/60, str(p.start_time()))
p.show(str(p.get_duration()/60))
print("BTOA")
for p in BtoA_paths:
print(p.get_duration()/60, str(p.start_time()))
p.show(str(p.get_duration()/60))
def test_finding_paths():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
filename = s3.download("_testing/testing_merged_1", folder)
data = filter_gps_data(read_compressed_trajectory(filename))
endpoints = find_endpoints(data)
assert(len(endpoints) == 2)
print(trajectory_point_to_str(data, _get_point_index(data, endpoints[0])))
print(trajectory_point_to_str(data, _get_point_index(data, endpoints[1])))
assert(_get_point_index(data, endpoints[0]) == 479)
assert(_get_point_index(data, endpoints[1]) == 670)
AtoB_paths, BtoA_paths = find_routes(data, endpoints, verbose=False)
AtoB_paths_indices = _extract_indices(data, AtoB_paths)
BtoA_paths_indices = _extract_indices(data, BtoA_paths)
print(AtoB_paths_indices)
print(BtoA_paths_indices)
assert(AtoB_paths_indices == [[488, 656], [947, 1117], [1364, 1549], [2216, 2401], [2630, 2898], [4400, 4526]])
assert(BtoA_paths_indices == [[134, 455], [683, 887], [1141, 1316], [1582, 1783], [2429, 2597], [3975, 4170]])
def test_finding_paths_with_state():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
filename = s3.download("_testing/testing_merged_1", folder)
full_trajectory = read_compressed_trajectory(filename)
pieces = []
pieces.append(full_trajectory[0:480])
pieces.append(full_trajectory[480:2000])
pieces.append(full_trajectory[2000:3000])
pieces.append(full_trajectory[3000:4000])
pieces.append(full_trajectory[4000:])
filtered_pieces = []
endpoints = []
findendpoints_state = None
filter_state = None
for piece in pieces:
vel_filter = VelocityOutliersFilter()
if filter_state is not None:
vel_filter.set_state(filter_state)
filter = FilterChain([DuplicateTimeFilter(),
vel_filter])
filtered_piece = apply_filter(piece, filter)
filter_state = vel_filter.get_state()
filtered_pieces.append(filtered_piece)
finder = FindEndpoints(endpoints=endpoints)
if findendpoints_state is not None:
finder.set_state(findendpoints_state)
for d in filtered_piece:
finder.process(d)
endpoints = finder.get_endpoints()
findendpoints_state = finder.get_state()
data = np.vstack(filtered_pieces)
assert(_get_point_index(data, endpoints[0]) == 479)
assert(_get_point_index(data, endpoints[1]) == 670)
AtoB_paths = []
BtoA_paths = []
finder_current_route = []
finder_endpoint_index = None
for piece in filtered_pieces:
finder = RoutesFinder(endpoints)
finder.set_state(finder_current_route, finder_endpoint_index)
for d in piece:
finder.process(d)
finder_current_route, finder_endpoint_index = finder.get_state()
AtoB_paths_piece, BtoA_paths_piece = finder.get_routes()
AtoB_paths += AtoB_paths_piece
BtoA_paths += BtoA_paths_piece
AtoB_paths_indices = _extract_indices(data, AtoB_paths)
BtoA_paths_indices = _extract_indices(data, BtoA_paths)
print(AtoB_paths_indices)
print(BtoA_paths_indices)
assert(AtoB_paths_indices == [[488, 656], [947, 1117], [1364, 1549], [2216, 2401], [2630, 2898], [4400, 4526]])
assert(BtoA_paths_indices == [[134, 455], [683, 887], [1141, 1316], [1582, 1783], [2429, 2597], [3975, 4170]])
def test_finding_paths_with_state_2():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
pieces = []
pieces_keys = [
"_testing/testing_sequence0/data/14-09-2015_09-15-01_PDT",
"_testing/testing_sequence0/data/14-09-2015_11-03-24_PDT",
"_testing/testing_sequence0/data/14-09-2015_13-49-55_PDT",
"_testing/testing_sequence0/data/14-09-2015_18-20-13_PDT",
"_testing/testing_sequence0/data/14-09-2015_19-59-23_PDT",
"_testing/testing_sequence0/data/15-09-2015_09-32-15_PDT",
"_testing/testing_sequence0/data/15-09-2015_22-31-21_PDT"
]
for k in pieces_keys:
filename = s3.download(k, folder)
pieces.append(read_compressed_trajectory(filename))
filtered_pieces = []
endpoints = []
findendpoints_state = None
filter_state = None
for pi, piece in enumerate(pieces):
vel_filter = VelocityOutliersFilter()
if filter_state is not None:
vel_filter.set_state(filter_state)
filter = FilterChain([DuplicateTimeFilter(),
vel_filter])
filtered_piece = apply_filter(piece, filter)
filter_state = vel_filter.get_state()
filtered_pieces.append(filtered_piece)
finder = FindEndpoints(endpoints=endpoints)
if findendpoints_state is not None:
finder.set_state(findendpoints_state)
for i, d in enumerate(filtered_piece):
finder.process(d)
endpoints = finder.get_endpoints()
findendpoints_state = finder.get_state()
data = np.vstack(filtered_pieces)
print(trajectory_point_to_str(data, _get_point_index(data, endpoints[0])))
print(trajectory_point_to_str(data, _get_point_index(data, endpoints[1])))
assert(len(endpoints) == 2)
assert(_get_point_index(data, endpoints[0]) == 5)
assert(_get_point_index(data, endpoints[1]) == 122)
AtoB_paths = []
BtoA_paths = []
finder_current_route = []
finder_endpoint_index = None
for piece in filtered_pieces:
finder = RoutesFinder(endpoints)
finder.set_state(finder_current_route, finder_endpoint_index)
for d in piece:
finder.process(d)
finder_current_route, finder_endpoint_index = finder.get_state()
AtoB_paths_piece, BtoA_paths_piece = finder.get_routes()
AtoB_paths += AtoB_paths_piece
BtoA_paths += BtoA_paths_piece
AtoB_paths_indices = _extract_indices(data, AtoB_paths)
BtoA_paths_indices = _extract_indices(data, BtoA_paths)
print(AtoB_paths_indices)
print(BtoA_paths_indices)
assert(AtoB_paths_indices == [[11, 111], [556, 730]])
assert(BtoA_paths_indices == [[288, 387]])
for t in trajectories_to_fix:
fix_trajectory_timezone(os.path.join(artifacts, 'raw', t),
os.path.join(artifacts, 'fixed'))
def fix_single_trajectory():
artifacts = os.path.join(os.path.dirname(__file__), 'artifacts')
to_fix_path = os.path.join(artifacts, 'raw', '03-08-2015_09-58-01')
fixed_path = fix_trajectory_timezone(to_fix_path,
os.path.join(artifacts, 'fixed'))
original_trajectory = read_compressed_trajectory(to_fix_path, with_timezone=False)
fixed_trajectory = read_compressed_trajectory(fixed_path, with_timezone=True)
print(num_to_date_str_converter(original_trajectory[0, 0], with_timezone=False))
print(num_to_date_str_converter(fixed_trajectory[0, 0], with_timezone=True))
if __name__ == '__main__':
artifacts = os.path.join(os.path.dirname(__file__), 'artifacts')
to_fix_path = os.path.join(artifacts, '_testing', 'testing_raw_0')
fixed_path = fix_trajectory_timezone(to_fix_path,
os.path.join(artifacts, 'fixed'),
change_filename=False)
original_trajectory = read_compressed_trajectory(to_fix_path, with_timezone=False)
fixed_trajectory = read_compressed_trajectory(fixed_path, with_timezone=True)
print(num_to_date_str_converter(original_trajectory[0, 0], with_timezone=False))
print(num_to_date_str_converter(fixed_trajectory[0, 0], with_timezone=True))
