def get_dynamic_simulation_data(self, simulation):
"""
Reads simulation dictionary and pulls out info for Static Simulation data:
Args:
simulation: a dictionary {"metadata":{....}, "data": [list of data_dicts]}
Returns:
A list containing the broadcasted GpsData points in order
"""
gps_data_list = []
start_time = self.parse_time(simulation["metadata"]["start_time"])
for data_point in simulation["data"]:
x, y, z = float(data_point["x"]), float(data_point["y"]), float(
data_point["z"])
time_from_zero = float(data_point["time_from_zero"])
lat, lon, alt = utils.cartesian_to_geodetic(x, y, z)
time_stamp = start_time + timedelta(seconds=time_from_zero)
# calculate speed for the point
if gps_data_list:
previous_point = gps_data_list[-1]
time_elapsed = (time_stamp -
previous_point.time).total_seconds()
distance = utils.calculate_distance(
(lat, lon),
(previous_point.latitude, previous_point.longitude))
speed = distance / time_elapsed
else:
speed = 0.0 # first point has speed of zero
gps_data_list.append(GpsData(lat, lon, alt, speed, time_stamp))
return gps_data_list
def test_calculate_distance_returns_correct_value(self):
coordinate_1 = (37.1111, -122.1111)
coordinate_2 = (40.7777, -125.7777)
result = utils.calculate_distance(coordinate_1, coordinate_2)
self.assertAlmostEqual(result, 516346.3, places=1)
def find_lineup_naive(set1, set2):
"""
Initial naive implementation to find lineup between two data sets.
Finds the point in the secondary data set closest to the start of the
primary data set, and then uses those starting indices to find the offset
with the lowest mean distance in the range of points around it index-wise
in the data set
Args:
set1: GpsDataSet object
set2: GpsDataSet object
Returns:
The indexes of the starting points in each data set
"""
sets = [set1, set2]
starting_indexes = [0, 0]
set1_start_time = set1.gps_data_list[0].time
set2_start_time = set2.gps_data_list[0].time
if set1_start_time > set2_start_time:
later_start_time = set1_start_time
primary_set_index = 0
secondary_set_index = 1
else:
later_start_time = set2_start_time
primary_set_index = 1
secondary_set_index = 0
smallest_time_difference = None
for i, data_point in enumerate(sets[secondary_set_index].gps_data_list):
time_difference = abs(
(data_point.time - later_start_time).total_seconds())
if smallest_time_difference is None or time_difference < smallest_time_difference:
smallest_time_difference = time_difference
starting_indexes[secondary_set_index] = i
best_index = 0
best_mean_distance = None
for offset in range(-100, 100):
distances = []
moving_index = starting_indexes[secondary_set_index] + offset
if moving_index >= 0:
primary_points = sets[primary_set_index].gps_data_list
secondary_points = sets[secondary_set_index].gps_data_list[
moving_index:]
for primary_point, secondary_point in zip(primary_points,
secondary_points):
location1 = (primary_point.latitude, primary_point.longitude)
location2 = (secondary_point.latitude,
secondary_point.longitude)
distances.append(utils.calculate_distance(
location1, location2))
if distances and (best_mean_distance == None
or np.mean(distances) < best_mean_distance):
best_mean_distance = np.mean(distances)
best_index = moving_index
print("optimal mean distance: " + str(best_mean_distance))
starting_indexes[secondary_set_index] = best_index
return starting_indexes
def get_deviation_dataframe(self):
"""
Extracts and returns deviation for each valid timestamp & other information.
Returns:
A pandas dataframe including the shared timestamp with the offset included,
the deviations of lat/lon, the difference in speed, the difference in
altitude, and the original timestamps for each set
"""
if self.deviations_dataframe is not None:
return self.deviations_dataframe
time_list, distance_deviation_list, speed_deviation_list, altitude_deviation_list= [], [], [], []
set1_time_list, set2_time_list = [], []
set1_average_signal_list, set2_average_signal_list, signal_deviation_list = [], [], []
for timestamp in self.offset_mapping_1:
if timestamp in self.offset_mapping_2:
time_list.append(timestamp)
# Get the mapping pair of data points in each dataset
point1 = self.offset_mapping_1[timestamp][0]
point2 = self.offset_mapping_2[timestamp][0]
# Calculate the distance deviation
location1 = (point1.latitude, point1.longitude)
location2 = (point2.latitude, point2.longitude)
distance_deviation_list.append(utils.calculate_distance(location1, location2))
# Calculate the speed differentials
speed_deviation_list.append(point2.speed - point1.speed)
# Calculate the altitude differentials
if point1.altitude is None or point2.altitude is None:
altitude_deviation_list.append(None)
else:
altitude_deviation_list.append(point2.altitude - point1.altitude)
# Append the original timestamp in each dataset
set1_time_list.append(point1.time)
set2_time_list.append(point2.time)
# Append the average signal if have
set1_average_signal_list.append(point1.average_signal)
set2_average_signal_list.append(point2.average_signal)
signal_deviation_list.append(point2.average_signal - point1.average_signal)
self.deviations_dataframe = pd.DataFrame({"Common Timestamp": time_list,
"Distance Deviations": distance_deviation_list,
"Speed Deviations": speed_deviation_list,
"Altitude Deviations": altitude_deviation_list,
"Set 1 Timestamp": set1_time_list,
"Set 2 Timestamp": set2_time_list,
"Set 1 Average Signal": set1_average_signal_list,
"Set 2 Average Signal": set2_average_signal_list,
"Signal Deviations": signal_deviation_list})
return self.deviations_dataframe
def find_optimal_offset(primary_set, secondary_set, start_time,
offset_range_length, point_checking_range_length):
"""
Find optimal offset from offset range to shift the secondary set
Finds the optimal offset meaning, an offset that is in the given offset
range to check and that also has overlap between the two sets. If the
mean distance is over 100m, it returns None, since this indicates sets
that do not align even if their timestamps can be shifted to align.
Args:
primary_set: dictionary in format {DateTime: [GpsData,], ...}
secondary_set: dictionary in format {DateTime: [GpsData,], ...}
start_time: Datetime to start calculation at
offset_range: int, how many offset values to check i.e. the range of
(-offset_range//2, offset_range//2)
point_checking_range: int, how many points around the start to use in the deviation
mean calculations
primary_set_index: which set (1 or 2) is the primary (i.e. not being shifted)
Returns:
int, the offset in seconds that should be applied to the secondary
data set if an optimal offset exists,
None otherwise
"""
optimal_offset = 0
optimal_mean_distance = MEAN_DISTANCE_THRESHOLD + 1
for offset in range(-offset_range_length // 2, offset_range_length // 2):
distances = []
skipped_point_count = 0
for i in range(-point_checking_range_length // 2,
point_checking_range_length // 2):
# apply offset to secondary set
primary_time = start_time + timedelta(seconds=i)
secondary_time = start_time + timedelta(seconds=i + offset)
if primary_time in primary_set and secondary_time in secondary_set:
# if more than one point at that time, choose first one
# TODO(ameles) consider if more complex downsampling needed
primary_point = primary_set[primary_time][0]
secondary_point = secondary_set[secondary_time][0]
location1 = (primary_point.latitude, primary_point.longitude)
location2 = (secondary_point.latitude,
secondary_point.longitude)
distances.append(utils.calculate_distance(
location1, location2))
else:
skipped_point_count += 1
if distances and np.mean(distances) < optimal_mean_distance:
optimal_mean_distance = np.mean(distances)
optimal_offset = offset
print("optimal mean distance: " + str(optimal_mean_distance))
if optimal_mean_distance is None or optimal_mean_distance > MEAN_DISTANCE_THRESHOLD:
return None
return optimal_offset
def test_calculate_distance_returns_zero_for_same_coordinate(self):
coordinate_1 = (37.1111, -122.1124)
result = utils.calculate_distance(coordinate_1, coordinate_1)
self.assertAlmostEqual(result, 0.0, places=8)
def parse_xml_trkpts(self, root) -> []:
"""
Helper function to parse trkpts in xml file
"""
xml_gpsdatalist = []
trk = root.find(PREFIX_URL + 'trk')
if trk is None:
LOGGER.debug('trk is None, could not parse trkpts.')
return None
trkseg = trk.find(PREFIX_URL + 'trkseg')
if trkseg is None:
LOGGER.debug('trkseg is None, could not parse trkpts.')
return None
if len(trkseg) == 0:
LOGGER.debug('trkseg is empty, could not parse trkpts.')
return None
# Get the start location
first_trkpt = trkseg[0]
prev_location = (first_trkpt.get('lat'), first_trkpt.get('lon'))
# Get every track point information
for trkpt in trkseg:
# Get the latitude and longitude
lat = float(trkpt.get('lat'))
lon = float(trkpt.get('lon'))
# Calculate the distance from previous location, not considering the altitude for now
cur_location = (lat, lon)
distance = utils.calculate_distance(cur_location, prev_location)
prev_location = cur_location
# Get the altitude, speed and time
alt, speed, tz_time = None, None, None
sat = 0
first_signal, second_signal, third_signal, forth_signal, average_signal = 0.0, 0.0, 0.0, 0.0, 0.0
for data in trkpt.iter():
if data.tag == PREFIX_URL + 'ele':
alt = float(data.text)
elif data.tag == PREFIX_URL + 'speed':
speed = float(data.text)
elif data.tag == PREFIX_URL + 'time':
tz_time = self.parse_time(data.text)
elif data.tag == PREFIX_URL + 'sat':
sat = int(data.text)
elif data.tag == PREFIX_URL + 'signal01':
first_signal = float(data.text)
elif data.tag == PREFIX_URL + 'signal02':
second_signal = float(data.text)
elif data.tag == PREFIX_URL + 'signal03':
third_signal = float(data.text)
elif data.tag == PREFIX_URL + 'signal04':
forth_signal = float(data.text)
elif data.tag == PREFIX_URL + 'average':
average_signal = float(data.text)
data_point = GpsData(latitude=lat,
longitude=lon,
altitude=alt,
speed=speed,
time=tz_time,
distance=distance,
satellites=sat,
first_signal=first_signal,
second_signal=second_signal,
third_signal=third_signal,
forth_signal=forth_signal,
average_signal=average_signal)
xml_gpsdatalist.append(data_point)
return xml_gpsdatalist