def testPointFilteringShanghaiJump(self):
classicJumpTrip1 = self.trips[0]
self.loadPointsForTrip(classicJumpTrip1.get_id())
classicJumpSections1 = [s for s in self.sections if s.trip_id == classicJumpTrip1.get_id()]
outlier_algo = eaics.BoxplotOutlier()
jump_algo = eaicj.SmoothZigzag()
for i, section in enumerate(classicJumpSections1):
logging.debug("-" * 20 + "Considering section %s" % i + "-" * 20)
section_df = self.ts.get_data_df("background/filtered_location", esds.get_time_query_for_section(section.get_id()))
with_speeds_df = eaicl.add_dist_heading_speed(section_df)
maxSpeed = outlier_algo.get_threshold(with_speeds_df)
logging.debug("Max speed for section %s = %s" % (i, maxSpeed))
jump_algo.filter(with_speeds_df)
logging.debug("Retaining points %s" % np.nonzero(jump_algo.inlier_mask_))
to_delete_mask = np.logical_not(jump_algo.inlier_mask_)
logging.debug("Deleting points %s" % np.nonzero(to_delete_mask))
delete_ids = list(with_speeds_df[to_delete_mask]._id)
logging.debug("Deleting ids %s" % delete_ids)
# Automated checks. Might be able to remove logging statements later
if i != 2:
# Not the bad section. Should not be filtered
self.assertEqual(np.count_nonzero(to_delete_mask), 0)
self.assertEqual(len(delete_ids), 0)
else:
# The bad section, should have the third point filtered
self.assertEqual(np.count_nonzero(to_delete_mask), 1)
self.assertEqual([str(id) for id in delete_ids], ["55d8c4837d65cb39ee983cb4"])
def filter_jumps(user_id, section_id):
"""
filters out any jumps in the points related to this section and stores a entry that lists the deleted points for
this trip and this section.
:param user_id: the user id to filter the trips for
:param section_id: the section_id to filter the trips for
:return: none. saves an entry with the filtered points into the database.
"""
logging.debug("filter_jumps(%s, %s) called" % (user_id, section_id))
outlier_algo = eaico.BoxplotOutlier()
filtering_algo = eaicj.SmoothZigzag()
tq = esds.get_time_query_for_section(section_id)
ts = esta.TimeSeries.get_time_series(user_id)
section_points_df = ts.get_data_df("background/filtered_location", tq)
logging.debug("len(section_points_df) = %s" % len(section_points_df))
points_to_ignore_df = get_points_to_filter(section_points_df, outlier_algo,
filtering_algo)
if points_to_ignore_df is None:
# There were no points to delete
return
deleted_point_id_list = list(points_to_ignore_df._id)
logging.debug("deleted %s points" % len(deleted_point_id_list))
filter_result = ecws.Smoothresults()
filter_result.section = section_id
filter_result.deleted_points = deleted_point_id_list
filter_result.outlier_algo = "BoxplotOutlier"
filter_result.filtering_algo = "SmoothZigzag"
result_entry = ecwe.Entry.create_entry(user_id, "analysis/smoothing",
filter_result)
ts.insert(result_entry)
def testPointFilteringRichmondJump(self):
classicJumpTrip1 = self.trips[6]
self.loadPointsForTrip(classicJumpTrip1.get_id())
classicJumpSections1 = [s for s in self.sections if s.trip_id == classicJumpTrip1.get_id()]
outlier_algo = eaics.BoxplotOutlier()
jump_algo = eaicj.SmoothZigzag()
for i, section in enumerate(classicJumpSections1):
logging.debug("-" * 20 + "Considering section %s" % i + "-" * 20)
section_df = self.ts.get_data_df("background/filtered_location", esds.get_time_query_for_section(section.get_id()))
with_speeds_df = eaicl.add_dist_heading_speed(section_df)
maxSpeed = outlier_algo.get_threshold(with_speeds_df)
logging.debug("Max speed for section %s = %s" % (i, maxSpeed))
jump_algo.filter(with_speeds_df)
logging.debug("Retaining points %s" % np.nonzero(jump_algo.inlier_mask_))
to_delete_mask = np.logical_not(jump_algo.inlier_mask_)
logging.debug("Deleting points %s" % np.nonzero(to_delete_mask))
delete_ids = list(with_speeds_df[to_delete_mask]._id)
logging.debug("Deleting ids %s" % delete_ids)
# There is only one section
self.assertEqual(i, 0)
# The bad section, should have the third point filtered
self.assertEqual(np.count_nonzero(to_delete_mask), 1)
self.assertEqual([str(id) for id in delete_ids], ["55e86dbb7d65cb39ee987e09"])
def filter_jumps(user_id, section_id):
"""
filters out any jumps in the points related to this section and stores a entry that lists the deleted points for
this trip and this section.
:param user_id: the user id to filter the trips for
:param section_id: the section_id to filter the trips for
:return: none. saves an entry with the filtered points into the database.
"""
logging.debug("filter_jumps(%s, %s) called" % (user_id, section_id))
outlier_algo = eaico.BoxplotOutlier()
filtering_algo = eaicj.SmoothZigzag()
tq = esds.get_time_query_for_section(section_id)
ts = esta.TimeSeries.get_time_series(user_id)
section_points_df = ts.get_data_df("background/filtered_location", tq)
logging.debug("len(section_points_df) = %s" % len(section_points_df))
points_to_ignore_df = get_points_to_filter(section_points_df, outlier_algo, filtering_algo)
if points_to_ignore_df is None:
# There were no points to delete
return
deleted_point_id_list = list(points_to_ignore_df._id)
logging.debug("deleted %s points" % len(deleted_point_id_list))
filter_result = ecws.Smoothresults()
filter_result.section = section_id
filter_result.deleted_points = deleted_point_id_list
filter_result.outlier_algo = "BoxplotOutlier"
filter_result.filtering_algo = "SmoothZigzag"
result_entry = ecwe.Entry.create_entry(user_id, "analysis/smoothing", filter_result)
ts.insert(result_entry)
def get_maps_for_range_old(user_id, start_ts, end_ts):
# First, get the timeline for that range.
ts = esta.TimeSeries.get_time_series(user_id)
trip_list = esdt.get_trips(user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts))
# TODO: Should the timeline support random access as well?
# If it did, we wouldn't need this additional map
# I think that it would be good to support a doubly linked list, i.e. prev and next in addition
# to the iteration interface
place_list = esdp.get_places(user_id, enua.UserCache.TimeQuery("exit_ts", start_ts, end_ts))
place_list = place_list + (esdp.get_places(user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts)))
place_map = dict([(p.get_id(), p) for p in place_list])
map_list = []
flipped_midpoint = lambda(p1, p2): [(p1.coordinates[1] + p2.coordinates[1])/2,
(p1.coordinates[0] + p2.coordinates[0])/2]
for i, trip in enumerate(trip_list):
logging.debug("-" * 20 + trip.start_fmt_time + "=>" + trip.end_fmt_time
+ "(" + str(trip.end_ts - trip.start_ts) + ")")
if (len(esdt.get_sections_for_trip(user_id, trip.get_id())) == 0 and
len(esdt.get_stops_for_trip(user_id, trip.get_id())) == 0):
logging.debug("Skipping trip because it has no stops and no sections")
continue
start_point = gj.GeoJSON.to_instance(trip.start_loc)
end_point = gj.GeoJSON.to_instance(trip.end_loc)
curr_map = folium.Map(flipped_midpoint((start_point, end_point)))
map_list.append(curr_map)
logging.debug("About to display places %s and %s" % (trip.start_place, trip.end_place))
update_place(curr_map, trip.start_place, place_map, marker_color='green')
update_place(curr_map, trip.end_place, place_map, marker_color='red')
# TODO: Should get_timeline_for_trip work on a trip_id or on a trip object
# it seems stupid to convert trip object -> id -> trip object
curr_trip_timeline = esdt.get_timeline_for_trip(user_id, trip.get_id())
for i, trip_element in enumerate(curr_trip_timeline):
# logging.debug("Examining element %s of type %s" % (trip_element, type(trip_element)))
if type(trip_element) == ecws.Stop:
time_query = esds.get_time_query_for_stop(trip_element.get_id())
logging.debug("time_query for stop %s = %s" % (trip_element, time_query))
stop_points_df = ts.get_data_df("background/filtered_location", time_query)
# logging.debug("stop_points_df.head() = %s" % stop_points_df.head())
if len(stop_points_df) > 0:
update_line(curr_map, stop_points_df, line_color = sel_color_list[-1],
popup="%s -> %s" % (trip_element.enter_fmt_time, trip_element.exit_fmt_time))
else:
assert(type(trip_element) == ecwsc.Section)
time_query = esdsc.get_time_query_for_section(trip_element.get_id())
logging.debug("time_query for section %s = %s" %
(trip_element, "[%s,%s,%s]" % (time_query.timeType, time_query.startTs, time_query.endTs)))
section_points_df = ts.get_data_df("background/filtered_location", time_query)
logging.debug("section_points_df.tail() = %s" % section_points_df.tail())
if len(section_points_df) > 0:
update_line(curr_map, section_points_df, line_color = sel_color_list[trip_element.sensed_mode.value],
popup="%s (%s -> %s)" % (trip_element.sensed_mode, trip_element.start_fmt_time,
trip_element.end_fmt_time))
else:
logging.warn("found no points for section %s" % trip_element)
return map_list
def get_maps_for_range_old(user_id, start_ts, end_ts):
# First, get the timeline for that range.
ts = esta.TimeSeries.get_time_series(user_id)
trip_list = esdt.get_trips(user_id, estt.TimeQuery("data.start_ts", start_ts, end_ts))
# TODO: Should the timeline support random access as well?
# If it did, we wouldn't need this additional map
# I think that it would be good to support a doubly linked list, i.e. prev and next in addition
# to the iteration interface
place_list = esdp.get_places(user_id, estt.TimeQuery("data.exit_ts", start_ts, end_ts))
place_list = place_list + (esdp.get_places(user_id, estt.TimeQuery("data.enter_ts", start_ts, end_ts)))
place_map = dict([(p.get_id(), p) for p in place_list])
map_list = []
flipped_midpoint = lambda p1_p22: [old_div((p1_p22[0].coordinates[1] + p1_p22[1].coordinates[1]),2),
old_div((p1_p22[0].coordinates[0] + p1_p22[1].coordinates[0]),2)]
for i, trip in enumerate(trip_list):
logging.debug("-" * 20 + trip.start_fmt_time + "=>" + trip.end_fmt_time
+ "(" + str(trip.end_ts - trip.start_ts) + ")")
if (len(esdt.get_raw_sections_for_trip(user_id, trip.get_id())) == 0 and
len(esdt.get_raw_stops_for_trip(user_id, trip.get_id())) == 0):
logging.debug("Skipping trip because it has no stops and no sections")
continue
start_point = gj.GeoJSON.to_instance(trip.start_loc)
end_point = gj.GeoJSON.to_instance(trip.end_loc)
curr_map = folium.Map(flipped_midpoint((start_point, end_point)))
map_list.append(curr_map)
logging.debug("About to display places %s and %s" % (trip.start_place, trip.end_place))
update_place(curr_map, trip.start_place, place_map, marker_color='green')
update_place(curr_map, trip.end_place, place_map, marker_color='red')
# TODO: Should get_timeline_for_trip work on a trip_id or on a trip object
# it seems stupid to convert trip object -> id -> trip object
curr_trip_timeline = esdt.get_raw_timeline_for_trip(user_id, trip.get_id())
for i, trip_element in enumerate(curr_trip_timeline):
# logging.debug("Examining element %s of type %s" % (trip_element, type(trip_element)))
if type(trip_element) == ecws.Stop:
time_query = esds.get_time_query_for_stop(trip_element.get_id())
logging.debug("time_query for stop %s = %s" % (trip_element, time_query))
stop_points_df = ts.get_data_df("background/filtered_location", time_query)
# logging.debug("stop_points_df.head() = %s" % stop_points_df.head())
if len(stop_points_df) > 0:
update_line(curr_map, stop_points_df, line_color = sel_color_list[-1],
popup="%s -> %s" % (trip_element.enter_fmt_time, trip_element.exit_fmt_time))
else:
assert(type(trip_element) == ecwsc.Section)
time_query = esdsc.get_time_query_for_section(trip_element.get_id())
logging.debug("time_query for section %s = %s" %
(trip_element, "[%s,%s,%s]" % (time_query.timeType, time_query.startTs, time_query.endTs)))
section_points_df = ts.get_data_df("background/filtered_location", time_query)
logging.debug("section_points_df.tail() = %s" % section_points_df.tail())
if len(section_points_df) > 0:
update_line(curr_map, section_points_df, line_color = sel_color_list[trip_element.sensed_mode.value],
popup="%s (%s -> %s)" % (trip_element.sensed_mode, trip_element.start_fmt_time,
trip_element.end_fmt_time))
else:
logging.warning("found no points for section %s" % trip_element)
return map_list
def testPointFilteringZigzag(self):
classicJumpTrip1 = self.trips[8]
self.loadPointsForTrip(classicJumpTrip1.get_id())
classicJumpSections1 = [
s for s in self.sections if s.trip_id == classicJumpTrip1.get_id()
]
outlier_algo = eaics.BoxplotOutlier()
jump_algo = eaicj.SmoothZigzag()
for i, section in enumerate(classicJumpSections1):
logging.debug("-" * 20 + "Considering section %s" % i + "-" * 20)
section_df = self.ts.get_data_df(
"background/filtered_location",
esds.get_time_query_for_section(section.get_id()))
with_speeds_df = eaicl.add_dist_heading_speed(section_df)
maxSpeed = outlier_algo.get_threshold(with_speeds_df)
logging.debug("Max speed for section %s = %s" % (i, maxSpeed))
jump_algo.filter(with_speeds_df)
logging.debug("Retaining points %s" %
np.nonzero(jump_algo.inlier_mask_))
to_delete_mask = np.logical_not(jump_algo.inlier_mask_)
logging.debug("Deleting points %s" % np.nonzero(to_delete_mask))
delete_ids = list(with_speeds_df[to_delete_mask]._id)
logging.debug("Deleting ids %s" % delete_ids)
if i == 0:
# this is the zigzag section
self.assertEqual(
np.nonzero(to_delete_mask)[0].tolist(),
[25, 64, 114, 115, 116, 117, 118, 119, 120, 123, 126])
self.assertEqual(delete_ids, [
boi.ObjectId('55edafe77d65cb39ee9882ff'),
boi.ObjectId('55edcc157d65cb39ee98836e'),
boi.ObjectId('55edcc1f7d65cb39ee988400'),
boi.ObjectId('55edcc1f7d65cb39ee988403'),
boi.ObjectId('55edcc1f7d65cb39ee988406'),
boi.ObjectId('55edcc1f7d65cb39ee988409'),
boi.ObjectId('55edcc1f7d65cb39ee98840c'),
boi.ObjectId('55edcc207d65cb39ee988410'),
boi.ObjectId('55edcc207d65cb39ee988412'),
boi.ObjectId('55edcc217d65cb39ee98841f'),
boi.ObjectId('55edcc217d65cb39ee988429')
])
else:
self.assertEqual(len(np.nonzero(to_delete_mask)[0]), 0)
self.assertEqual(len(delete_ids), 0)
def testPointFilteringZigzag(self):
classicJumpTrip1 = self.trips[8]
self.loadPointsForTrip(classicJumpTrip1.get_id())
classicJumpSections1 = [s for s in self.sections if s.trip_id == classicJumpTrip1.get_id()]
outlier_algo = eaics.BoxplotOutlier()
jump_algo = eaicj.SmoothZigzag()
for i, section in enumerate(classicJumpSections1):
logging.debug("-" * 20 + "Considering section %s" % i + "-" * 20)
section_df = self.ts.get_data_df("background/filtered_location", esds.get_time_query_for_section(section.get_id()))
with_speeds_df = eaicl.add_dist_heading_speed(section_df)
maxSpeed = outlier_algo.get_threshold(with_speeds_df)
logging.debug("Max speed for section %s = %s" % (i, maxSpeed))
jump_algo.filter(with_speeds_df)
logging.debug("Retaining points %s" % np.nonzero(jump_algo.inlier_mask_))
to_delete_mask = np.logical_not(jump_algo.inlier_mask_)
logging.debug("Deleting points %s" % np.nonzero(to_delete_mask))
delete_ids = list(with_speeds_df[to_delete_mask]._id)
logging.debug("Deleting ids %s" % delete_ids)
if i == 0:
# this is the zigzag section
self.assertEqual(np.nonzero(to_delete_mask)[0].tolist(),
[25, 64, 114, 115, 116, 117, 118, 119, 120, 123, 126])
self.assertEqual(delete_ids,
[boi.ObjectId('55edafe77d65cb39ee9882ff'),
boi.ObjectId('55edcc157d65cb39ee98836e'),
boi.ObjectId('55edcc1f7d65cb39ee988400'),
boi.ObjectId('55edcc1f7d65cb39ee988403'),
boi.ObjectId('55edcc1f7d65cb39ee988406'),
boi.ObjectId('55edcc1f7d65cb39ee988409'),
boi.ObjectId('55edcc1f7d65cb39ee98840c'),
boi.ObjectId('55edcc207d65cb39ee988410'),
boi.ObjectId('55edcc207d65cb39ee988412'),
boi.ObjectId('55edcc217d65cb39ee98841f'),
boi.ObjectId('55edcc217d65cb39ee988429')])
else:
self.assertEqual(len(np.nonzero(to_delete_mask)[0]), 0)
self.assertEqual(len(delete_ids), 0)
def testPointFilteringShanghaiJump(self):
classicJumpTrip1 = self.trips[0]
self.loadPointsForTrip(classicJumpTrip1.get_id())
classicJumpSections1 = [
s for s in self.sections if s.trip_id == classicJumpTrip1.get_id()
]
outlier_algo = eaics.BoxplotOutlier()
jump_algo = eaicj.SmoothZigzag()
for i, section in enumerate(classicJumpSections1):
logging.debug("-" * 20 + "Considering section %s" % i + "-" * 20)
section_df = self.ts.get_data_df(
"background/filtered_location",
esds.get_time_query_for_section(section.get_id()))
with_speeds_df = eaicl.add_dist_heading_speed(section_df)
maxSpeed = outlier_algo.get_threshold(with_speeds_df)
logging.debug("Max speed for section %s = %s" % (i, maxSpeed))
jump_algo.filter(with_speeds_df)
logging.debug("Retaining points %s" %
np.nonzero(jump_algo.inlier_mask_))
to_delete_mask = np.logical_not(jump_algo.inlier_mask_)
logging.debug("Deleting points %s" % np.nonzero(to_delete_mask))
delete_ids = list(with_speeds_df[to_delete_mask]._id)
logging.debug("Deleting ids %s" % delete_ids)
# Automated checks. Might be able to remove logging statements later
if i != 2:
# Not the bad section. Should not be filtered
self.assertEqual(np.count_nonzero(to_delete_mask), 0)
self.assertEqual(len(delete_ids), 0)
else:
# The bad section, should have the third point filtered
self.assertEqual(np.count_nonzero(to_delete_mask), 1)
self.assertEqual([str(id) for id in delete_ids],
["55d8c4837d65cb39ee983cb4"])
def testPointFilteringRichmondJump(self):
classicJumpTrip1 = self.trips[6]
self.loadPointsForTrip(classicJumpTrip1.get_id())
classicJumpSections1 = [
s for s in self.sections if s.trip_id == classicJumpTrip1.get_id()
]
outlier_algo = eaics.BoxplotOutlier()
jump_algo = eaicj.SmoothZigzag()
for i, section in enumerate(classicJumpSections1):
logging.debug("-" * 20 + "Considering section %s" % i + "-" * 20)
section_df = self.ts.get_data_df(
"background/filtered_location",
esds.get_time_query_for_section(section.get_id()))
with_speeds_df = eaicl.add_dist_heading_speed(section_df)
maxSpeed = outlier_algo.get_threshold(with_speeds_df)
logging.debug("Max speed for section %s = %s" % (i, maxSpeed))
jump_algo.filter(with_speeds_df)
logging.debug("Retaining points %s" %
np.nonzero(jump_algo.inlier_mask_))
to_delete_mask = np.logical_not(jump_algo.inlier_mask_)
logging.debug("Deleting points %s" % np.nonzero(to_delete_mask))
delete_ids = list(with_speeds_df[to_delete_mask]._id)
logging.debug("Deleting ids %s" % delete_ids)
# There is only one section
self.assertEqual(i, 0)
# The bad section, should have the third point filtered
self.assertEqual(np.count_nonzero(to_delete_mask), 1)
self.assertEqual([str(id) for id in delete_ids],
["55e86dbb7d65cb39ee987e09"])
def section_to_geojson(section, tl):
"""
This is the trickiest part of the visualization.
The section is basically a collection of points with a line through them.
So the representation is a feature in which one feature which is the line, and one feature collection which is the set of point features.
:param section: the section to be converted
:return: a feature collection which is the geojson version of the section
"""
ts = esta.TimeSeries.get_time_series(section.user_id)
entry_it = ts.find_entries(["background/filtered_location"], esds.get_time_query_for_section(section.get_id()))
# points_df = ts.get_data_df("background/filtered_location", esds.get_time_query_for_section(section.get_id()))
# points_df = points_df.drop("elapsedRealTimeNanos", axis=1)
# logging.debug("points_df.columns = %s" % points_df.columns)
# TODO: Decide whether we want to use Rewrite to use dataframes throughout instead of python arrays.
# dataframes insert nans. We could use fillna to fill with default values, but if we are not actually
# using dataframe features here, it is unclear how much that would help.
feature_array = []
section_location_array = [ecwl.Location(ts._to_df_entry(entry)) for entry in entry_it]
logging.debug("first element in section_location_array = %s" % section_location_array[0])
# Fudge the end point so that we don't have a gap because of the ts != write_ts mismatch
# TODO: Fix this once we are able to query by the data timestamp instead of the metadata ts
if section_location_array[-1].loc != section.end_loc:
last_loc_doc = ts.get_entry_at_ts("background/filtered_location", "data.ts", section.end_ts)
last_loc_data = ecwe.Entry(last_loc_doc).data
last_loc_data["_id"] = last_loc_doc["_id"]
section_location_array.append(last_loc_data)
logging.debug("Adding new entry %s to fill the end point gap between %s and %s"
% (last_loc_data.loc, section_location_array[-2].loc, section.end_loc))
# Find the list of points to filter
filtered_points_entry_doc = ts.get_entry_at_ts("analysis/smoothing", "data.section",
section.get_id())
if filtered_points_entry_doc is None:
logging.debug("No filtered_points_entry, returning unchanged array")
filtered_section_location_array = section_location_array
else:
# TODO: Figure out how to make collections work for the wrappers and then change this to an Entry
filtered_points_entry = ad.AttrDict(filtered_points_entry_doc)
filtered_point_list = list(filtered_points_entry.data.deleted_points)
logging.debug("deleting %s points from section points" % len(filtered_point_list))
filtered_section_location_array = [l for l in section_location_array if l.get_id() not in filtered_point_list]
with_speeds = eaicl.add_dist_heading_speed(pd.DataFrame(filtered_section_location_array))
speeds = list(with_speeds.speed)
distances = list(with_speeds.distance)
for idx, row in with_speeds.iterrows():
# TODO: Remove instance of setting value without going through wrapper class
filtered_section_location_array[idx]["speed"] = row["speed"]
filtered_section_location_array[idx]["distance"] = row["distance"]
points_feature_array = [location_to_geojson(l) for l in filtered_section_location_array]
points_line_feature = point_array_to_line(filtered_section_location_array)
# If this is the first section, we already start from the trip start. But we actually need to start from the
# prior place. Fudge this too. Note also that we may want to figure out how to handle this properly in the model
# without needing fudging. TODO: Unclear how exactly to do this
if section.start_stop is None:
# This is the first section. So we need to find the start place of the parent trip
parent_trip = tl.get_object(section.trip_id)
start_place_of_parent_trip = tl.get_object(parent_trip.start_place)
points_line_feature.geometry.coordinates.insert(0, start_place_of_parent_trip.location.coordinates)
for i, point_feature in enumerate(points_feature_array):
point_feature.properties["idx"] = i
points_line_feature.id = str(section.get_id())
points_line_feature.properties = copy.copy(section)
points_line_feature.properties["feature_type"] = "section"
points_line_feature.properties["sensed_mode"] = str(points_line_feature.properties.sensed_mode)
points_line_feature.properties["distance"] = sum(distances)
points_line_feature.properties["speeds"] = speeds
points_line_feature.properties["distances"] = distances
_del_non_derializable(points_line_feature.properties, ["start_loc", "end_loc"])
feature_array.append(gj.FeatureCollection(points_feature_array))
feature_array.append(points_line_feature)
return gj.FeatureCollection(feature_array)
def section_to_geojson(section, tl):
"""
This is the trickiest part of the visualization.
The section is basically a collection of points with a line through them.
So the representation is a feature in which one feature which is the line, and one feature collection which is the set of point features.
:param section: the section to be converted
:return: a feature collection which is the geojson version of the section
"""
ts = esta.TimeSeries.get_time_series(section.user_id)
entry_it = ts.find_entries(["background/filtered_location"],
esds.get_time_query_for_section(
section.get_id()))
# points_df = ts.get_data_df("background/filtered_location", esds.get_time_query_for_section(section.get_id()))
# points_df = points_df.drop("elapsedRealTimeNanos", axis=1)
# logging.debug("points_df.columns = %s" % points_df.columns)
# TODO: Decide whether we want to use Rewrite to use dataframes throughout instead of python arrays.
# dataframes insert nans. We could use fillna to fill with default values, but if we are not actually
# using dataframe features here, it is unclear how much that would help.
feature_array = []
section_location_array = [
ecwl.Location(ts._to_df_entry(entry)) for entry in entry_it
]
if len(section_location_array) != 0:
logging.debug("first element in section_location_array = %s" %
section_location_array[0])
# Fudge the end point so that we don't have a gap because of the ts != write_ts mismatch
# TODO: Fix this once we are able to query by the data timestamp instead of the metadata ts
if section_location_array[-1].loc != section.end_loc:
last_loc_doc = ts.get_entry_at_ts("background/filtered_location",
"data.ts", section.end_ts)
last_loc_data = ecwe.Entry(last_loc_doc).data
last_loc_data["_id"] = last_loc_doc["_id"]
section_location_array.append(last_loc_data)
logging.debug(
"Adding new entry %s to fill the end point gap between %s and %s"
% (last_loc_data.loc, section_location_array[-2].loc,
section.end_loc))
# Find the list of points to filter
filtered_points_entry_doc = ts.get_entry_at_ts("analysis/smoothing",
"data.section",
section.get_id())
if filtered_points_entry_doc is None:
logging.debug("No filtered_points_entry, returning unchanged array")
filtered_section_location_array = section_location_array
else:
# TODO: Figure out how to make collections work for the wrappers and then change this to an Entry
filtered_points_entry = ad.AttrDict(filtered_points_entry_doc)
filtered_point_list = list(filtered_points_entry.data.deleted_points)
logging.debug("deleting %s points from section points" %
len(filtered_point_list))
filtered_section_location_array = [
l for l in section_location_array
if l.get_id() not in filtered_point_list
]
with_speeds = eaicl.add_dist_heading_speed(
pd.DataFrame(filtered_section_location_array))
speeds = list(with_speeds.speed)
distances = list(with_speeds.distance)
if len(filtered_section_location_array) != 0:
for idx, row in with_speeds.iterrows():
# TODO: Remove instance of setting value without going through wrapper class
filtered_section_location_array[idx]["speed"] = row["speed"]
filtered_section_location_array[idx]["distance"] = row["distance"]
points_feature_array = [
location_to_geojson(l) for l in filtered_section_location_array
]
points_line_feature = point_array_to_line(filtered_section_location_array)
# If this is the first section, we already start from the trip start. But we actually need to start from the
# prior place. Fudge this too. Note also that we may want to figure out how to handle this properly in the model
# without needing fudging. TODO: Unclear how exactly to do this
if section.start_stop is None:
# This is the first section. So we need to find the start place of the parent trip
parent_trip = tl.get_object(section.trip_id)
start_place_of_parent_trip = tl.get_object(parent_trip.start_place)
points_line_feature.geometry.coordinates.insert(
0, start_place_of_parent_trip.location.coordinates)
for i, point_feature in enumerate(points_feature_array):
point_feature.properties["idx"] = i
points_line_feature.id = str(section.get_id())
points_line_feature.properties = copy.copy(section)
points_line_feature.properties["feature_type"] = "section"
points_line_feature.properties["sensed_mode"] = str(
points_line_feature.properties.sensed_mode)
points_line_feature.properties["distance"] = sum(distances)
points_line_feature.properties["speeds"] = speeds
points_line_feature.properties["distances"] = distances
_del_non_derializable(points_line_feature.properties,
["start_loc", "end_loc"])
feature_array.append(gj.FeatureCollection(points_feature_array))
feature_array.append(points_line_feature)
return gj.FeatureCollection(feature_array)
