def get_timeline(user_id, start_ts, end_ts):
"""
Return a timeline of the trips and places from this start timestamp to this end timestamp.
Note that each place and each trip has *two* associated timestamps, so we need to define which trips need to be
returned. Right now, we define this as all places that are entered and all trips that are started within the
specified time frame. Note that this means that, by definition, this may not include the starting and ending places
for all trips, which is something that we need for our visualization. But we don't want the timeline to be
visualization specific.
Let's compromise by adding method to fill in start and end places which we will call if the timeline is used
for visualization and not if not. This also means that we can use the id map to avoid duplicates in case the
place does exist.
:param user_id: the user whose timeline we are considering
:param start_ts: the starting timestamp. we will include all places and trips that start after this.
:param end_ts: the ending timestamp. we will include all places and trips that end after this.
:return: a timeline object
"""
# These imports are in here to avoid circular import dependencies between
# trip_queries and this file (timeline)
import emission.storage.decorations.place_queries as esdp
import emission.storage.decorations.trip_queries as esdt
places = esdp.get_places(
user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts))
trips = esdt.get_trips(
user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts))
for place in places:
logging.debug(
"Considering place %s: %s -> %s " %
(place.get_id(), place.enter_fmt_time, place.exit_fmt_time))
for trip in trips:
logging.debug("Considering trip %s: %s -> %s " %
(trip.get_id(), trip.start_fmt_time, trip.end_fmt_time))
return Timeline(places, trips)
def get_timeline(user_id, start_ts, end_ts):
"""
Return a timeline of the trips and places from this start timestamp to this end timestamp.
Note that each place and each trip has *two* associated timestamps, so we need to define which trips need to be
returned. Right now, we define this as all places that are entered and all trips that are started within the
specified time frame. Note that this means that, by definition, this may not include the starting and ending places
for all trips, which is something that we need for our visualization. But we don't want the timeline to be
visualization specific.
Let's compromise by adding method to fill in start and end places which we will call if the timeline is used
for visualization and not if not. This also means that we can use the id map to avoid duplicates in case the
place does exist.
:param user_id: the user whose timeline we are considering
:param start_ts: the starting timestamp. we will include all places and trips that start after this.
:param end_ts: the ending timestamp. we will include all places and trips that end after this.
:return: a timeline object
"""
# These imports are in here to avoid circular import dependencies between
# trip_queries and this file (timeline)
import emission.storage.decorations.place_queries as esdp
import emission.storage.decorations.trip_queries as esdt
places = esdp.get_places(user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts))
trips = esdt.get_trips(user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts))
for place in places:
logging.debug("Considering place %s: %s -> %s " % (place.get_id(), place.enter_fmt_time, place.exit_fmt_time))
for trip in trips:
logging.debug("Considering trip %s: %s -> %s " % (trip.get_id(), trip.start_fmt_time, trip.end_fmt_time))
return Timeline(places, trips)
def get_maps_for_range_old(user_id, start_ts, end_ts):
# First, get the timeline for that range.
ts = esta.TimeSeries.get_time_series(user_id)
trip_list = esdt.get_trips(user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts))
# TODO: Should the timeline support random access as well?
# If it did, we wouldn't need this additional map
# I think that it would be good to support a doubly linked list, i.e. prev and next in addition
# to the iteration interface
place_list = esdp.get_places(user_id, enua.UserCache.TimeQuery("exit_ts", start_ts, end_ts))
place_list = place_list + (esdp.get_places(user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts)))
place_map = dict([(p.get_id(), p) for p in place_list])
map_list = []
flipped_midpoint = lambda(p1, p2): [(p1.coordinates[1] + p2.coordinates[1])/2,
(p1.coordinates[0] + p2.coordinates[0])/2]
for i, trip in enumerate(trip_list):
logging.debug("-" * 20 + trip.start_fmt_time + "=>" + trip.end_fmt_time
+ "(" + str(trip.end_ts - trip.start_ts) + ")")
if (len(esdt.get_sections_for_trip(user_id, trip.get_id())) == 0 and
len(esdt.get_stops_for_trip(user_id, trip.get_id())) == 0):
logging.debug("Skipping trip because it has no stops and no sections")
continue
start_point = gj.GeoJSON.to_instance(trip.start_loc)
end_point = gj.GeoJSON.to_instance(trip.end_loc)
curr_map = folium.Map(flipped_midpoint((start_point, end_point)))
map_list.append(curr_map)
logging.debug("About to display places %s and %s" % (trip.start_place, trip.end_place))
update_place(curr_map, trip.start_place, place_map, marker_color='green')
update_place(curr_map, trip.end_place, place_map, marker_color='red')
# TODO: Should get_timeline_for_trip work on a trip_id or on a trip object
# it seems stupid to convert trip object -> id -> trip object
curr_trip_timeline = esdt.get_timeline_for_trip(user_id, trip.get_id())
for i, trip_element in enumerate(curr_trip_timeline):
# logging.debug("Examining element %s of type %s" % (trip_element, type(trip_element)))
if type(trip_element) == ecws.Stop:
time_query = esds.get_time_query_for_stop(trip_element.get_id())
logging.debug("time_query for stop %s = %s" % (trip_element, time_query))
stop_points_df = ts.get_data_df("background/filtered_location", time_query)
# logging.debug("stop_points_df.head() = %s" % stop_points_df.head())
if len(stop_points_df) > 0:
update_line(curr_map, stop_points_df, line_color = sel_color_list[-1],
popup="%s -> %s" % (trip_element.enter_fmt_time, trip_element.exit_fmt_time))
else:
assert(type(trip_element) == ecwsc.Section)
time_query = esdsc.get_time_query_for_section(trip_element.get_id())
logging.debug("time_query for section %s = %s" %
(trip_element, "[%s,%s,%s]" % (time_query.timeType, time_query.startTs, time_query.endTs)))
section_points_df = ts.get_data_df("background/filtered_location", time_query)
logging.debug("section_points_df.tail() = %s" % section_points_df.tail())
if len(section_points_df) > 0:
update_line(curr_map, section_points_df, line_color = sel_color_list[trip_element.sensed_mode.value],
popup="%s (%s -> %s)" % (trip_element.sensed_mode, trip_element.start_fmt_time,
trip_element.end_fmt_time))
else:
logging.warn("found no points for section %s" % trip_element)
return map_list
def get_maps_for_range_old(user_id, start_ts, end_ts):
# First, get the timeline for that range.
ts = esta.TimeSeries.get_time_series(user_id)
trip_list = esdt.get_trips(user_id, estt.TimeQuery("data.start_ts", start_ts, end_ts))
# TODO: Should the timeline support random access as well?
# If it did, we wouldn't need this additional map
# I think that it would be good to support a doubly linked list, i.e. prev and next in addition
# to the iteration interface
place_list = esdp.get_places(user_id, estt.TimeQuery("data.exit_ts", start_ts, end_ts))
place_list = place_list + (esdp.get_places(user_id, estt.TimeQuery("data.enter_ts", start_ts, end_ts)))
place_map = dict([(p.get_id(), p) for p in place_list])
map_list = []
flipped_midpoint = lambda p1_p22: [old_div((p1_p22[0].coordinates[1] + p1_p22[1].coordinates[1]),2),
old_div((p1_p22[0].coordinates[0] + p1_p22[1].coordinates[0]),2)]
for i, trip in enumerate(trip_list):
logging.debug("-" * 20 + trip.start_fmt_time + "=>" + trip.end_fmt_time
+ "(" + str(trip.end_ts - trip.start_ts) + ")")
if (len(esdt.get_raw_sections_for_trip(user_id, trip.get_id())) == 0 and
len(esdt.get_raw_stops_for_trip(user_id, trip.get_id())) == 0):
logging.debug("Skipping trip because it has no stops and no sections")
continue
start_point = gj.GeoJSON.to_instance(trip.start_loc)
end_point = gj.GeoJSON.to_instance(trip.end_loc)
curr_map = folium.Map(flipped_midpoint((start_point, end_point)))
map_list.append(curr_map)
logging.debug("About to display places %s and %s" % (trip.start_place, trip.end_place))
update_place(curr_map, trip.start_place, place_map, marker_color='green')
update_place(curr_map, trip.end_place, place_map, marker_color='red')
# TODO: Should get_timeline_for_trip work on a trip_id or on a trip object
# it seems stupid to convert trip object -> id -> trip object
curr_trip_timeline = esdt.get_raw_timeline_for_trip(user_id, trip.get_id())
for i, trip_element in enumerate(curr_trip_timeline):
# logging.debug("Examining element %s of type %s" % (trip_element, type(trip_element)))
if type(trip_element) == ecws.Stop:
time_query = esds.get_time_query_for_stop(trip_element.get_id())
logging.debug("time_query for stop %s = %s" % (trip_element, time_query))
stop_points_df = ts.get_data_df("background/filtered_location", time_query)
# logging.debug("stop_points_df.head() = %s" % stop_points_df.head())
if len(stop_points_df) > 0:
update_line(curr_map, stop_points_df, line_color = sel_color_list[-1],
popup="%s -> %s" % (trip_element.enter_fmt_time, trip_element.exit_fmt_time))
else:
assert(type(trip_element) == ecwsc.Section)
time_query = esdsc.get_time_query_for_section(trip_element.get_id())
logging.debug("time_query for section %s = %s" %
(trip_element, "[%s,%s,%s]" % (time_query.timeType, time_query.startTs, time_query.endTs)))
section_points_df = ts.get_data_df("background/filtered_location", time_query)
logging.debug("section_points_df.tail() = %s" % section_points_df.tail())
if len(section_points_df) > 0:
update_line(curr_map, section_points_df, line_color = sel_color_list[trip_element.sensed_mode.value],
popup="%s (%s -> %s)" % (trip_element.sensed_mode, trip_element.start_fmt_time,
trip_element.end_fmt_time))
else:
logging.warning("found no points for section %s" % trip_element)
return map_list
def testSegmentationWrapperWithAutoTrip(self):
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1440745200)
created_trips = esdt.get_trips(self.testUUID, tq_trip)
for i, trip in enumerate(created_trips):
created_stops = esdt.get_stops_for_trip(self.testUUID, trip.get_id())
created_sections = esdt.get_sections_for_trip(self.testUUID, trip.get_id())
for j, stop in enumerate(created_stops):
logging.info("Retrieved stops %s: %s -> %s" % (j, stop.enter_fmt_time, stop.exit_fmt_time))
for j, section in enumerate(created_sections):
logging.info("Retrieved sections %s: %s -> %s" % (j, section.start_fmt_time, section.end_fmt_time))
def segment_current_sections(user_id):
time_query = epq.get_time_range_for_sectioning(user_id)
try:
trips_to_process = esdt.get_trips(user_id, time_query)
for trip in trips_to_process:
logging.info("+" * 20 + ("Processing trip %s for user %s" % (trip.get_id(), user_id)) + "+" * 20)
segment_trip_into_sections(user_id, trip.get_id())
if len(trips_to_process) == 0:
# Didn't process anything new so start at the same point next time
last_trip_processed = None
else:
last_trip_processed = trips_to_process[-1]
epq.mark_sectioning_done(user_id, last_trip_processed)
except:
logging.exception("Sectioning failed for user %s" % user_id)
epq.mark_sectioning_failed(user_id)
def segment_current_sections(user_id):
time_query = epq.get_time_range_for_sectioning(user_id)
try:
trips_to_process = esdt.get_trips(user_id, time_query)
for trip in trips_to_process:
logging.info("+" * 20 + ("Processing trip %s for user %s" %
(trip.get_id(), user_id)) + "+" * 20)
segment_trip_into_sections(user_id, trip.get_id(), trip.source)
if len(trips_to_process) == 0:
# Didn't process anything new so start at the same point next time
last_trip_processed = None
else:
last_trip_processed = trips_to_process[-1]
epq.mark_sectioning_done(user_id, last_trip_processed)
except:
logging.exception("Sectioning failed for user %s" % user_id)
epq.mark_sectioning_failed(user_id)
def testSegmentationWrapperWithAutoTrip(self):
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1440745200)
created_trips = esdt.get_trips(self.testUUID, tq_trip)
for i, trip in enumerate(created_trips):
created_stops = esdt.get_stops_for_trip(self.testUUID,
trip.get_id())
created_sections = esdt.get_sections_for_trip(
self.testUUID, trip.get_id())
for j, stop in enumerate(created_stops):
logging.info("Retrieved stops %s: %s -> %s" %
(j, stop.enter_fmt_time, stop.exit_fmt_time))
for j, section in enumerate(created_sections):
logging.info("Retrieved sections %s: %s -> %s" %
(j, section.start_fmt_time, section.end_fmt_time))
def testSegmentationWrapperIOS(self):
eaist.segment_current_trips(self.iosUUID)
# The previous line should have created places and trips and stored
# them into the database. Now, we want to query to ensure that they
# were created correctly.
tq_place = enua.UserCache.TimeQuery("enter_ts", 1446796800, 1446847600)
created_places = esdp.get_places(self.iosUUID, tq_place)
tq_trip = enua.UserCache.TimeQuery("start_ts", 1446796800, 1446847600)
created_trips = esdt.get_trips(self.iosUUID, tq_trip)
for i, place in enumerate(created_places):
logging.debug("Retrieved places %s: %s -> %s" %
(i, place.enter_fmt_time, place.exit_fmt_time))
for i, trip in enumerate(created_trips):
logging.debug("Retrieved trips %s: %s -> %s" %
(i, trip.start_fmt_time, trip.end_fmt_time))
# We expect there to be 4 places, but the first one is that start of
# the chain, so it has a start_time of None and it won't be retrieved
# by the query on the start_time that we show here.
self.assertEqual(len(created_places), 3)
self.assertEqual(len(created_trips), 3)
# Pick the first two trips and the first place and ensure that they are all linked correctly
# Note that this is the first place, not the second place because the true first place will not
# be retrieved by the query, as shown above
# The first trip here is a dummy trip, so let's check the second and third trip instead
trip0 = created_trips[1]
trip1 = created_trips[2]
place0 = created_places[1]
self.assertEqual(trip0.end_place, place0.get_id())
self.assertEqual(trip1.start_place, place0.get_id())
self.assertEqual(place0.ending_trip, trip0.get_id())
self.assertEqual(place0.starting_trip, trip1.get_id())
self.assertEqual(round(trip0.duration), 58 * 60 + 51)
self.assertEqual(round(trip1.duration), 38 * 60 + 57)
self.assertIsNotNone(place0.location)
def testSegmentationWrapperIOS(self):
eaist.segment_current_trips(self.iosUUID)
# The previous line should have created places and trips and stored
# them into the database. Now, we want to query to ensure that they
# were created correctly.
tq_place = enua.UserCache.TimeQuery("enter_ts", 1446796800, 1446847600)
created_places = esdp.get_places(self.iosUUID, tq_place)
tq_trip = enua.UserCache.TimeQuery("start_ts", 1446796800, 1446847600)
created_trips = esdt.get_trips(self.iosUUID, tq_trip)
for i, place in enumerate(created_places):
logging.debug("Retrieved places %s: %s -> %s" % (i, place.enter_fmt_time, place.exit_fmt_time))
for i, trip in enumerate(created_trips):
logging.debug("Retrieved trips %s: %s -> %s" % (i, trip.start_fmt_time, trip.end_fmt_time))
# We expect there to be 4 places, but the first one is that start of
# the chain, so it has a start_time of None and it won't be retrieved
# by the query on the start_time that we show here.
self.assertEqual(len(created_places), 3)
self.assertEqual(len(created_trips), 3)
# Pick the first two trips and the first place and ensure that they are all linked correctly
# Note that this is the first place, not the second place because the true first place will not
# be retrieved by the query, as shown above
# The first trip here is a dummy trip, so let's check the second and third trip instead
trip0 = created_trips[1]
trip1 = created_trips[2]
place0 = created_places[1]
self.assertEqual(trip0.end_place, place0.get_id())
self.assertEqual(trip1.start_place, place0.get_id())
self.assertEqual(place0.ending_trip, trip0.get_id())
self.assertEqual(place0.starting_trip, trip1.get_id())
self.assertEqual(round(trip0.duration), 58 * 60 + 51)
self.assertEqual(round(trip1.duration), 38 * 60 + 57)
self.assertIsNotNone(place0.location)
def testSegmentationWrapperCombined(self):
# Change iOS entries to have the android UUID
for entry in esta.TimeSeries.get_time_series(self.iosUUID).find_entries():
entry["user_id"] = self.androidUUID
edb.get_timeseries_db().save(entry)
# Now, segment the data for the combined UUID, which will include both
# android and ios
eaist.segment_current_trips(self.androidUUID)
tq_place = enua.UserCache.TimeQuery("enter_ts", 1440658800, 1446847600)
created_places = esdp.get_places(self.androidUUID, tq_place)
tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1446847600)
created_trips = esdt.get_trips(self.androidUUID, tq_trip)
for i, place in enumerate(created_places):
logging.debug("Retrieved places %s: %s -> %s" % (i, place.enter_fmt_time, place.exit_fmt_time))
for i, trip in enumerate(created_trips):
logging.debug("Retrieved trips %s: %s -> %s" % (i, trip.start_fmt_time, trip.end_fmt_time))
# We expect there to be 12 places, but the first one is that start of
# the chain, so it has a start_time of None and it won't be retrieved
# by the query on the start_time that we show here.
self.assertEqual(len(created_places), 11)
self.assertEqual(len(created_trips), 11)
# Pick the first two trips and the first place and ensure that they are all linked correctly
# Note that this is the first place, not the second place because the true first place will not
# be retrieved by the query, as shown above
# The first trip here is a dummy trip, so let's check the second and third trip instead
trip0time = created_trips[0]
trip1time = created_trips[1]
place0time = created_places[0]
self.assertEqual(trip0time.end_place, place0time.get_id())
self.assertEqual(trip1time.start_place, place0time.get_id())
self.assertEqual(place0time.ending_trip, trip0time.get_id())
self.assertEqual(place0time.starting_trip, trip1time.get_id())
self.assertEqual(round(trip0time.duration), 11 * 60 + 9)
self.assertEqual(round(trip1time.duration), 6 * 60 + 54)
self.assertIsNotNone(place0time.location)
# There are 8 android trips first (index: 0-7).
# index 8 is the short, bogus trip
# So we want to check trips 9 and 10
trip0dist = created_trips[9]
trip1dist = created_trips[10]
place0dist = created_places[9]
self.assertEqual(trip0dist.end_place, place0dist.get_id())
self.assertEqual(trip1dist.start_place, place0dist.get_id())
self.assertEqual(place0dist.ending_trip, trip0dist.get_id())
self.assertEqual(place0dist.starting_trip, trip1dist.get_id())
self.assertEqual(round(trip0dist.duration), 58 * 60 + 51)
self.assertEqual(round(trip1dist.duration), 38 * 60 + 57)
self.assertIsNotNone(place0dist.location)
def testQueryTrips(self):
new_trip = self.create_fake_trip()
ret_arr_time = esdt.get_trips(self.testUserId, enua.UserCache.TimeQuery("start_ts", 4, 6))
self.assertEqual(ret_arr_time, [new_trip])
def testSegmentationWrapperCombined(self):
# Change iOS entries to have the android UUID
for entry in esta.TimeSeries.get_time_series(
self.iosUUID).find_entries():
entry["user_id"] = self.androidUUID
edb.get_timeseries_db().save(entry)
# Now, segment the data for the combined UUID, which will include both
# android and ios
eaist.segment_current_trips(self.androidUUID)
tq_place = enua.UserCache.TimeQuery("enter_ts", 1440658800, 1446847600)
created_places = esdp.get_places(self.androidUUID, tq_place)
tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1446847600)
created_trips = esdt.get_trips(self.androidUUID, tq_trip)
for i, place in enumerate(created_places):
logging.debug("Retrieved places %s: %s -> %s" %
(i, place.enter_fmt_time, place.exit_fmt_time))
for i, trip in enumerate(created_trips):
logging.debug("Retrieved trips %s: %s -> %s" %
(i, trip.start_fmt_time, trip.end_fmt_time))
# We expect there to be 12 places, but the first one is that start of
# the chain, so it has a start_time of None and it won't be retrieved
# by the query on the start_time that we show here.
self.assertEqual(len(created_places), 11)
self.assertEqual(len(created_trips), 11)
# Pick the first two trips and the first place and ensure that they are all linked correctly
# Note that this is the first place, not the second place because the true first place will not
# be retrieved by the query, as shown above
# The first trip here is a dummy trip, so let's check the second and third trip instead
trip0time = created_trips[0]
trip1time = created_trips[1]
place0time = created_places[0]
self.assertEqual(trip0time.end_place, place0time.get_id())
self.assertEqual(trip1time.start_place, place0time.get_id())
self.assertEqual(place0time.ending_trip, trip0time.get_id())
self.assertEqual(place0time.starting_trip, trip1time.get_id())
self.assertEqual(round(trip0time.duration), 11 * 60 + 9)
self.assertEqual(round(trip1time.duration), 6 * 60 + 54)
self.assertIsNotNone(place0time.location)
# There are 8 android trips first (index: 0-7).
# index 8 is the short, bogus trip
# So we want to check trips 9 and 10
trip0dist = created_trips[9]
trip1dist = created_trips[10]
place0dist = created_places[9]
self.assertEqual(trip0dist.end_place, place0dist.get_id())
self.assertEqual(trip1dist.start_place, place0dist.get_id())
self.assertEqual(place0dist.ending_trip, trip0dist.get_id())
self.assertEqual(place0dist.starting_trip, trip1dist.get_id())
self.assertEqual(round(trip0dist.duration), 58 * 60 + 51)
self.assertEqual(round(trip1dist.duration), 38 * 60 + 57)
self.assertIsNotNone(place0dist.location)
