def test_remove_stationary_noise():
"""
The data has large amount of noise - switching between SD and LA every 10 seconds.
It starts from SD, then noise, later it returns to SD. We test that LA is ignored
"""
data = remove_duplicate_points(_get_test_data())[561:576]
fixed_data = apply_filter(data, VelocityOutliersFilter())
print(len(fixed_data))
assert len(fixed_data) == 11
stationary_point = [0, 33.004964, -117.060207]
distances = np.array([distance(stationary_point, d) for d in fixed_data])
assert (distances < 246.6).all()
def merge_trajectories(raw_trajectories_folder, results_file):
onlyfiles = [f for f in os.listdir(raw_trajectories_folder)
if os.path.isfile(os.path.join(raw_trajectories_folder, f))]
contents = []
for name in onlyfiles:
full_name = os.path.join(raw_trajectories_folder, name)
contents.append(read_compressed_trajectory(full_name))
contents.sort(key=lambda p: p[0, 0], reverse=False)
merged_traj = np.vstack(contents)
merged_traj = apply_filter(merged_traj, DuplicateTimeFilter())
dt = extract_delta_time(merged_traj)
assert((dt > 0).all())
write_compressed_trajectory(merged_traj, results_file)
def test_remove_duplicate_readings():
data = _get_test_data()
velocities = compute_velocities(data)
number_of_duplicates = np.sum(np.isnan(velocities))
fixed_data = remove_duplicate_points(data)
fixed_velocities = compute_velocities(fixed_data)
assert np.sum(np.isnan(fixed_velocities)) == 0
# check that we deleted duplicates only
print(data.shape[0])
print(fixed_data.shape[0])
print(number_of_duplicates)
assert fixed_data.shape[0] == data.shape[0] - number_of_duplicates
print(fixed_data.shape[0], data.shape[0])
fixed_data_processor = apply_filter(data, DuplicateTimeFilter())
assert (fixed_data == fixed_data_processor).all()
def test_remove_stationary_noise_return_to_stable():
"""
The data has large amount of noise - switching between SD and LA every 10 seconds.
It starts from the noisy point, later it returns to SD.
Here we test that even if data starts with noisy value, we still converge
to stable point
"""
data = remove_duplicate_points(_get_test_data())[563:576]
fixed_data = apply_filter(data, VelocityOutliersFilter(85))
stationary_point = [0, 33.004964, -117.060207]
distances = np.array([distance(stationary_point, d) for d in fixed_data])
print(fixed_data)
assert len(fixed_data) == 7
# filter converged after few steps
assert (distances[:4] > 157000).all()
assert (distances[4:] < 246.6).all()
def test_remove_outliers():
np.set_printoptions(suppress=True)
data = remove_stationary_points(remove_duplicate_points(_get_test_data()))
velocity_threshold = 85.0
fixed_data = apply_filter(data, VelocityOutliersFilter(velocity_threshold))
# check that data has outliers in velocity and distance
velocities = compute_velocities(data)
outliers = np.where(velocities * ms_to_mph > 85)[0]
assert len(outliers) > 0
assert np.amax(extract_delta_dist(data)) > 157900
# no large velocities left
velocities = compute_velocities(fixed_data)
assert np.amax(velocities) * ms_to_mph < velocity_threshold
assert np.amax(extract_delta_dist(fixed_data)) < 330
# we expect this number of point to be removed
print(data.shape[0] - fixed_data.shape[0])
assert data.shape[0] - fixed_data.shape[0] == 5
def test_finding_paths_with_state():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
filename = s3.download("_testing/testing_merged_1", folder)
full_trajectory = read_compressed_trajectory(filename)
pieces = []
pieces.append(full_trajectory[0:480])
pieces.append(full_trajectory[480:2000])
pieces.append(full_trajectory[2000:3000])
pieces.append(full_trajectory[3000:4000])
pieces.append(full_trajectory[4000:])
filtered_pieces = []
endpoints = []
findendpoints_state = None
filter_state = None
for piece in pieces:
vel_filter = VelocityOutliersFilter()
if filter_state is not None:
vel_filter.set_state(filter_state)
filter = FilterChain([DuplicateTimeFilter(),
vel_filter])
filtered_piece = apply_filter(piece, filter)
filter_state = vel_filter.get_state()
filtered_pieces.append(filtered_piece)
finder = FindEndpoints(endpoints=endpoints)
if findendpoints_state is not None:
finder.set_state(findendpoints_state)
for d in filtered_piece:
finder.process(d)
endpoints = finder.get_endpoints()
findendpoints_state = finder.get_state()
data = np.vstack(filtered_pieces)
assert(_get_point_index(data, endpoints[0]) == 479)
assert(_get_point_index(data, endpoints[1]) == 670)
AtoB_paths = []
BtoA_paths = []
finder_current_route = []
finder_endpoint_index = None
for piece in filtered_pieces:
finder = RoutesFinder(endpoints)
finder.set_state(finder_current_route, finder_endpoint_index)
for d in piece:
finder.process(d)
finder_current_route, finder_endpoint_index = finder.get_state()
AtoB_paths_piece, BtoA_paths_piece = finder.get_routes()
AtoB_paths += AtoB_paths_piece
BtoA_paths += BtoA_paths_piece
AtoB_paths_indices = _extract_indices(data, AtoB_paths)
BtoA_paths_indices = _extract_indices(data, BtoA_paths)
print(AtoB_paths_indices)
print(BtoA_paths_indices)
assert(AtoB_paths_indices == [[488, 656], [947, 1117], [1364, 1549], [2216, 2401], [2630, 2898], [4400, 4526]])
assert(BtoA_paths_indices == [[134, 455], [683, 887], [1141, 1316], [1582, 1783], [2429, 2597], [3975, 4170]])
def test_finding_paths_with_state_2():
folder = tempfile.mkdtemp()
s3 = S3('driverpete-storage')
pieces = []
pieces_keys = [
"_testing/testing_sequence0/data/14-09-2015_09-15-01_PDT",
"_testing/testing_sequence0/data/14-09-2015_11-03-24_PDT",
"_testing/testing_sequence0/data/14-09-2015_13-49-55_PDT",
"_testing/testing_sequence0/data/14-09-2015_18-20-13_PDT",
"_testing/testing_sequence0/data/14-09-2015_19-59-23_PDT",
"_testing/testing_sequence0/data/15-09-2015_09-32-15_PDT",
"_testing/testing_sequence0/data/15-09-2015_22-31-21_PDT"
]
for k in pieces_keys:
filename = s3.download(k, folder)
pieces.append(read_compressed_trajectory(filename))
filtered_pieces = []
endpoints = []
findendpoints_state = None
filter_state = None
for pi, piece in enumerate(pieces):
vel_filter = VelocityOutliersFilter()
if filter_state is not None:
vel_filter.set_state(filter_state)
filter = FilterChain([DuplicateTimeFilter(),
vel_filter])
filtered_piece = apply_filter(piece, filter)
filter_state = vel_filter.get_state()
filtered_pieces.append(filtered_piece)
finder = FindEndpoints(endpoints=endpoints)
if findendpoints_state is not None:
finder.set_state(findendpoints_state)
for i, d in enumerate(filtered_piece):
finder.process(d)
endpoints = finder.get_endpoints()
findendpoints_state = finder.get_state()
data = np.vstack(filtered_pieces)
print(trajectory_point_to_str(data, _get_point_index(data, endpoints[0])))
print(trajectory_point_to_str(data, _get_point_index(data, endpoints[1])))
assert(len(endpoints) == 2)
assert(_get_point_index(data, endpoints[0]) == 5)
assert(_get_point_index(data, endpoints[1]) == 122)
AtoB_paths = []
BtoA_paths = []
finder_current_route = []
finder_endpoint_index = None
for piece in filtered_pieces:
finder = RoutesFinder(endpoints)
finder.set_state(finder_current_route, finder_endpoint_index)
for d in piece:
finder.process(d)
finder_current_route, finder_endpoint_index = finder.get_state()
AtoB_paths_piece, BtoA_paths_piece = finder.get_routes()
AtoB_paths += AtoB_paths_piece
BtoA_paths += BtoA_paths_piece
AtoB_paths_indices = _extract_indices(data, AtoB_paths)
BtoA_paths_indices = _extract_indices(data, BtoA_paths)
print(AtoB_paths_indices)
print(BtoA_paths_indices)
assert(AtoB_paths_indices == [[11, 111], [556, 730]])
assert(BtoA_paths_indices == [[288, 387]])
