def get_timeline(user_id, start_ts, end_ts): """ Return a timeline of the trips and places from this start timestamp to this end timestamp. Note that each place and each trip has *two* associated timestamps, so we need to define which trips need to be returned. Right now, we define this as all places that are entered and all trips that are started within the specified time frame. Note that this means that, by definition, this may not include the starting and ending places for all trips, which is something that we need for our visualization. But we don't want the timeline to be visualization specific. Let's compromise by adding method to fill in start and end places which we will call if the timeline is used for visualization and not if not. This also means that we can use the id map to avoid duplicates in case the place does exist. :param user_id: the user whose timeline we are considering :param start_ts: the starting timestamp. we will include all places and trips that start after this. :param end_ts: the ending timestamp. we will include all places and trips that end after this. :return: a timeline object """ # These imports are in here to avoid circular import dependencies between # trip_queries and this file (timeline) import emission.storage.decorations.place_queries as esdp import emission.storage.decorations.trip_queries as esdt places = esdp.get_places( user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts)) trips = esdt.get_trips( user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts)) for place in places: logging.debug( "Considering place %s: %s -> %s " % (place.get_id(), place.enter_fmt_time, place.exit_fmt_time)) for trip in trips: logging.debug("Considering trip %s: %s -> %s " % (trip.get_id(), trip.start_fmt_time, trip.end_fmt_time)) return Timeline(places, trips)
def get_timeline(user_id, start_ts, end_ts): """ Return a timeline of the trips and places from this start timestamp to this end timestamp. Note that each place and each trip has *two* associated timestamps, so we need to define which trips need to be returned. Right now, we define this as all places that are entered and all trips that are started within the specified time frame. Note that this means that, by definition, this may not include the starting and ending places for all trips, which is something that we need for our visualization. But we don't want the timeline to be visualization specific. Let's compromise by adding method to fill in start and end places which we will call if the timeline is used for visualization and not if not. This also means that we can use the id map to avoid duplicates in case the place does exist. :param user_id: the user whose timeline we are considering :param start_ts: the starting timestamp. we will include all places and trips that start after this. :param end_ts: the ending timestamp. we will include all places and trips that end after this. :return: a timeline object """ # These imports are in here to avoid circular import dependencies between # trip_queries and this file (timeline) import emission.storage.decorations.place_queries as esdp import emission.storage.decorations.trip_queries as esdt places = esdp.get_places(user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts)) trips = esdt.get_trips(user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts)) for place in places: logging.debug("Considering place %s: %s -> %s " % (place.get_id(), place.enter_fmt_time, place.exit_fmt_time)) for trip in trips: logging.debug("Considering trip %s: %s -> %s " % (trip.get_id(), trip.start_fmt_time, trip.end_fmt_time)) return Timeline(places, trips)
def get_maps_for_range_old(user_id, start_ts, end_ts): # First, get the timeline for that range. ts = esta.TimeSeries.get_time_series(user_id) trip_list = esdt.get_trips(user_id, enua.UserCache.TimeQuery("start_ts", start_ts, end_ts)) # TODO: Should the timeline support random access as well? # If it did, we wouldn't need this additional map # I think that it would be good to support a doubly linked list, i.e. prev and next in addition # to the iteration interface place_list = esdp.get_places(user_id, enua.UserCache.TimeQuery("exit_ts", start_ts, end_ts)) place_list = place_list + (esdp.get_places(user_id, enua.UserCache.TimeQuery("enter_ts", start_ts, end_ts))) place_map = dict([(p.get_id(), p) for p in place_list]) map_list = [] flipped_midpoint = lambda(p1, p2): [(p1.coordinates[1] + p2.coordinates[1])/2, (p1.coordinates[0] + p2.coordinates[0])/2] for i, trip in enumerate(trip_list): logging.debug("-" * 20 + trip.start_fmt_time + "=>" + trip.end_fmt_time + "(" + str(trip.end_ts - trip.start_ts) + ")") if (len(esdt.get_sections_for_trip(user_id, trip.get_id())) == 0 and len(esdt.get_stops_for_trip(user_id, trip.get_id())) == 0): logging.debug("Skipping trip because it has no stops and no sections") continue start_point = gj.GeoJSON.to_instance(trip.start_loc) end_point = gj.GeoJSON.to_instance(trip.end_loc) curr_map = folium.Map(flipped_midpoint((start_point, end_point))) map_list.append(curr_map) logging.debug("About to display places %s and %s" % (trip.start_place, trip.end_place)) update_place(curr_map, trip.start_place, place_map, marker_color='green') update_place(curr_map, trip.end_place, place_map, marker_color='red') # TODO: Should get_timeline_for_trip work on a trip_id or on a trip object # it seems stupid to convert trip object -> id -> trip object curr_trip_timeline = esdt.get_timeline_for_trip(user_id, trip.get_id()) for i, trip_element in enumerate(curr_trip_timeline): # logging.debug("Examining element %s of type %s" % (trip_element, type(trip_element))) if type(trip_element) == ecws.Stop: time_query = esds.get_time_query_for_stop(trip_element.get_id()) logging.debug("time_query for stop %s = %s" % (trip_element, time_query)) stop_points_df = ts.get_data_df("background/filtered_location", time_query) # logging.debug("stop_points_df.head() = %s" % stop_points_df.head()) if len(stop_points_df) > 0: update_line(curr_map, stop_points_df, line_color = sel_color_list[-1], popup="%s -> %s" % (trip_element.enter_fmt_time, trip_element.exit_fmt_time)) else: assert(type(trip_element) == ecwsc.Section) time_query = esdsc.get_time_query_for_section(trip_element.get_id()) logging.debug("time_query for section %s = %s" % (trip_element, "[%s,%s,%s]" % (time_query.timeType, time_query.startTs, time_query.endTs))) section_points_df = ts.get_data_df("background/filtered_location", time_query) logging.debug("section_points_df.tail() = %s" % section_points_df.tail()) if len(section_points_df) > 0: update_line(curr_map, section_points_df, line_color = sel_color_list[trip_element.sensed_mode.value], popup="%s (%s -> %s)" % (trip_element.sensed_mode, trip_element.start_fmt_time, trip_element.end_fmt_time)) else: logging.warn("found no points for section %s" % trip_element) return map_list
def get_maps_for_range_old(user_id, start_ts, end_ts): # First, get the timeline for that range. ts = esta.TimeSeries.get_time_series(user_id) trip_list = esdt.get_trips(user_id, estt.TimeQuery("data.start_ts", start_ts, end_ts)) # TODO: Should the timeline support random access as well? # If it did, we wouldn't need this additional map # I think that it would be good to support a doubly linked list, i.e. prev and next in addition # to the iteration interface place_list = esdp.get_places(user_id, estt.TimeQuery("data.exit_ts", start_ts, end_ts)) place_list = place_list + (esdp.get_places(user_id, estt.TimeQuery("data.enter_ts", start_ts, end_ts))) place_map = dict([(p.get_id(), p) for p in place_list]) map_list = [] flipped_midpoint = lambda p1_p22: [old_div((p1_p22[0].coordinates[1] + p1_p22[1].coordinates[1]),2), old_div((p1_p22[0].coordinates[0] + p1_p22[1].coordinates[0]),2)] for i, trip in enumerate(trip_list): logging.debug("-" * 20 + trip.start_fmt_time + "=>" + trip.end_fmt_time + "(" + str(trip.end_ts - trip.start_ts) + ")") if (len(esdt.get_raw_sections_for_trip(user_id, trip.get_id())) == 0 and len(esdt.get_raw_stops_for_trip(user_id, trip.get_id())) == 0): logging.debug("Skipping trip because it has no stops and no sections") continue start_point = gj.GeoJSON.to_instance(trip.start_loc) end_point = gj.GeoJSON.to_instance(trip.end_loc) curr_map = folium.Map(flipped_midpoint((start_point, end_point))) map_list.append(curr_map) logging.debug("About to display places %s and %s" % (trip.start_place, trip.end_place)) update_place(curr_map, trip.start_place, place_map, marker_color='green') update_place(curr_map, trip.end_place, place_map, marker_color='red') # TODO: Should get_timeline_for_trip work on a trip_id or on a trip object # it seems stupid to convert trip object -> id -> trip object curr_trip_timeline = esdt.get_raw_timeline_for_trip(user_id, trip.get_id()) for i, trip_element in enumerate(curr_trip_timeline): # logging.debug("Examining element %s of type %s" % (trip_element, type(trip_element))) if type(trip_element) == ecws.Stop: time_query = esds.get_time_query_for_stop(trip_element.get_id()) logging.debug("time_query for stop %s = %s" % (trip_element, time_query)) stop_points_df = ts.get_data_df("background/filtered_location", time_query) # logging.debug("stop_points_df.head() = %s" % stop_points_df.head()) if len(stop_points_df) > 0: update_line(curr_map, stop_points_df, line_color = sel_color_list[-1], popup="%s -> %s" % (trip_element.enter_fmt_time, trip_element.exit_fmt_time)) else: assert(type(trip_element) == ecwsc.Section) time_query = esdsc.get_time_query_for_section(trip_element.get_id()) logging.debug("time_query for section %s = %s" % (trip_element, "[%s,%s,%s]" % (time_query.timeType, time_query.startTs, time_query.endTs))) section_points_df = ts.get_data_df("background/filtered_location", time_query) logging.debug("section_points_df.tail() = %s" % section_points_df.tail()) if len(section_points_df) > 0: update_line(curr_map, section_points_df, line_color = sel_color_list[trip_element.sensed_mode.value], popup="%s (%s -> %s)" % (trip_element.sensed_mode, trip_element.start_fmt_time, trip_element.end_fmt_time)) else: logging.warning("found no points for section %s" % trip_element) return map_list
def testSegmentationWrapperWithAutoTrip(self): eaist.segment_current_trips(self.testUUID) eaiss.segment_current_sections(self.testUUID) tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1440745200) created_trips = esdt.get_trips(self.testUUID, tq_trip) for i, trip in enumerate(created_trips): created_stops = esdt.get_stops_for_trip(self.testUUID, trip.get_id()) created_sections = esdt.get_sections_for_trip(self.testUUID, trip.get_id()) for j, stop in enumerate(created_stops): logging.info("Retrieved stops %s: %s -> %s" % (j, stop.enter_fmt_time, stop.exit_fmt_time)) for j, section in enumerate(created_sections): logging.info("Retrieved sections %s: %s -> %s" % (j, section.start_fmt_time, section.end_fmt_time))
def segment_current_sections(user_id): time_query = epq.get_time_range_for_sectioning(user_id) try: trips_to_process = esdt.get_trips(user_id, time_query) for trip in trips_to_process: logging.info("+" * 20 + ("Processing trip %s for user %s" % (trip.get_id(), user_id)) + "+" * 20) segment_trip_into_sections(user_id, trip.get_id()) if len(trips_to_process) == 0: # Didn't process anything new so start at the same point next time last_trip_processed = None else: last_trip_processed = trips_to_process[-1] epq.mark_sectioning_done(user_id, last_trip_processed) except: logging.exception("Sectioning failed for user %s" % user_id) epq.mark_sectioning_failed(user_id)
def segment_current_sections(user_id): time_query = epq.get_time_range_for_sectioning(user_id) try: trips_to_process = esdt.get_trips(user_id, time_query) for trip in trips_to_process: logging.info("+" * 20 + ("Processing trip %s for user %s" % (trip.get_id(), user_id)) + "+" * 20) segment_trip_into_sections(user_id, trip.get_id(), trip.source) if len(trips_to_process) == 0: # Didn't process anything new so start at the same point next time last_trip_processed = None else: last_trip_processed = trips_to_process[-1] epq.mark_sectioning_done(user_id, last_trip_processed) except: logging.exception("Sectioning failed for user %s" % user_id) epq.mark_sectioning_failed(user_id)
def testSegmentationWrapperWithAutoTrip(self): eaist.segment_current_trips(self.testUUID) eaiss.segment_current_sections(self.testUUID) tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1440745200) created_trips = esdt.get_trips(self.testUUID, tq_trip) for i, trip in enumerate(created_trips): created_stops = esdt.get_stops_for_trip(self.testUUID, trip.get_id()) created_sections = esdt.get_sections_for_trip( self.testUUID, trip.get_id()) for j, stop in enumerate(created_stops): logging.info("Retrieved stops %s: %s -> %s" % (j, stop.enter_fmt_time, stop.exit_fmt_time)) for j, section in enumerate(created_sections): logging.info("Retrieved sections %s: %s -> %s" % (j, section.start_fmt_time, section.end_fmt_time))
def testSegmentationWrapperIOS(self): eaist.segment_current_trips(self.iosUUID) # The previous line should have created places and trips and stored # them into the database. Now, we want to query to ensure that they # were created correctly. tq_place = enua.UserCache.TimeQuery("enter_ts", 1446796800, 1446847600) created_places = esdp.get_places(self.iosUUID, tq_place) tq_trip = enua.UserCache.TimeQuery("start_ts", 1446796800, 1446847600) created_trips = esdt.get_trips(self.iosUUID, tq_trip) for i, place in enumerate(created_places): logging.debug("Retrieved places %s: %s -> %s" % (i, place.enter_fmt_time, place.exit_fmt_time)) for i, trip in enumerate(created_trips): logging.debug("Retrieved trips %s: %s -> %s" % (i, trip.start_fmt_time, trip.end_fmt_time)) # We expect there to be 4 places, but the first one is that start of # the chain, so it has a start_time of None and it won't be retrieved # by the query on the start_time that we show here. self.assertEqual(len(created_places), 3) self.assertEqual(len(created_trips), 3) # Pick the first two trips and the first place and ensure that they are all linked correctly # Note that this is the first place, not the second place because the true first place will not # be retrieved by the query, as shown above # The first trip here is a dummy trip, so let's check the second and third trip instead trip0 = created_trips[1] trip1 = created_trips[2] place0 = created_places[1] self.assertEqual(trip0.end_place, place0.get_id()) self.assertEqual(trip1.start_place, place0.get_id()) self.assertEqual(place0.ending_trip, trip0.get_id()) self.assertEqual(place0.starting_trip, trip1.get_id()) self.assertEqual(round(trip0.duration), 58 * 60 + 51) self.assertEqual(round(trip1.duration), 38 * 60 + 57) self.assertIsNotNone(place0.location)
def testSegmentationWrapperCombined(self): # Change iOS entries to have the android UUID for entry in esta.TimeSeries.get_time_series(self.iosUUID).find_entries(): entry["user_id"] = self.androidUUID edb.get_timeseries_db().save(entry) # Now, segment the data for the combined UUID, which will include both # android and ios eaist.segment_current_trips(self.androidUUID) tq_place = enua.UserCache.TimeQuery("enter_ts", 1440658800, 1446847600) created_places = esdp.get_places(self.androidUUID, tq_place) tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1446847600) created_trips = esdt.get_trips(self.androidUUID, tq_trip) for i, place in enumerate(created_places): logging.debug("Retrieved places %s: %s -> %s" % (i, place.enter_fmt_time, place.exit_fmt_time)) for i, trip in enumerate(created_trips): logging.debug("Retrieved trips %s: %s -> %s" % (i, trip.start_fmt_time, trip.end_fmt_time)) # We expect there to be 12 places, but the first one is that start of # the chain, so it has a start_time of None and it won't be retrieved # by the query on the start_time that we show here. self.assertEqual(len(created_places), 11) self.assertEqual(len(created_trips), 11) # Pick the first two trips and the first place and ensure that they are all linked correctly # Note that this is the first place, not the second place because the true first place will not # be retrieved by the query, as shown above # The first trip here is a dummy trip, so let's check the second and third trip instead trip0time = created_trips[0] trip1time = created_trips[1] place0time = created_places[0] self.assertEqual(trip0time.end_place, place0time.get_id()) self.assertEqual(trip1time.start_place, place0time.get_id()) self.assertEqual(place0time.ending_trip, trip0time.get_id()) self.assertEqual(place0time.starting_trip, trip1time.get_id()) self.assertEqual(round(trip0time.duration), 11 * 60 + 9) self.assertEqual(round(trip1time.duration), 6 * 60 + 54) self.assertIsNotNone(place0time.location) # There are 8 android trips first (index: 0-7). # index 8 is the short, bogus trip # So we want to check trips 9 and 10 trip0dist = created_trips[9] trip1dist = created_trips[10] place0dist = created_places[9] self.assertEqual(trip0dist.end_place, place0dist.get_id()) self.assertEqual(trip1dist.start_place, place0dist.get_id()) self.assertEqual(place0dist.ending_trip, trip0dist.get_id()) self.assertEqual(place0dist.starting_trip, trip1dist.get_id()) self.assertEqual(round(trip0dist.duration), 58 * 60 + 51) self.assertEqual(round(trip1dist.duration), 38 * 60 + 57) self.assertIsNotNone(place0dist.location)
def testQueryTrips(self): new_trip = self.create_fake_trip() ret_arr_time = esdt.get_trips(self.testUserId, enua.UserCache.TimeQuery("start_ts", 4, 6)) self.assertEqual(ret_arr_time, [new_trip])
def testSegmentationWrapperCombined(self): # Change iOS entries to have the android UUID for entry in esta.TimeSeries.get_time_series( self.iosUUID).find_entries(): entry["user_id"] = self.androidUUID edb.get_timeseries_db().save(entry) # Now, segment the data for the combined UUID, which will include both # android and ios eaist.segment_current_trips(self.androidUUID) tq_place = enua.UserCache.TimeQuery("enter_ts", 1440658800, 1446847600) created_places = esdp.get_places(self.androidUUID, tq_place) tq_trip = enua.UserCache.TimeQuery("start_ts", 1440658800, 1446847600) created_trips = esdt.get_trips(self.androidUUID, tq_trip) for i, place in enumerate(created_places): logging.debug("Retrieved places %s: %s -> %s" % (i, place.enter_fmt_time, place.exit_fmt_time)) for i, trip in enumerate(created_trips): logging.debug("Retrieved trips %s: %s -> %s" % (i, trip.start_fmt_time, trip.end_fmt_time)) # We expect there to be 12 places, but the first one is that start of # the chain, so it has a start_time of None and it won't be retrieved # by the query on the start_time that we show here. self.assertEqual(len(created_places), 11) self.assertEqual(len(created_trips), 11) # Pick the first two trips and the first place and ensure that they are all linked correctly # Note that this is the first place, not the second place because the true first place will not # be retrieved by the query, as shown above # The first trip here is a dummy trip, so let's check the second and third trip instead trip0time = created_trips[0] trip1time = created_trips[1] place0time = created_places[0] self.assertEqual(trip0time.end_place, place0time.get_id()) self.assertEqual(trip1time.start_place, place0time.get_id()) self.assertEqual(place0time.ending_trip, trip0time.get_id()) self.assertEqual(place0time.starting_trip, trip1time.get_id()) self.assertEqual(round(trip0time.duration), 11 * 60 + 9) self.assertEqual(round(trip1time.duration), 6 * 60 + 54) self.assertIsNotNone(place0time.location) # There are 8 android trips first (index: 0-7). # index 8 is the short, bogus trip # So we want to check trips 9 and 10 trip0dist = created_trips[9] trip1dist = created_trips[10] place0dist = created_places[9] self.assertEqual(trip0dist.end_place, place0dist.get_id()) self.assertEqual(trip1dist.start_place, place0dist.get_id()) self.assertEqual(place0dist.ending_trip, trip0dist.get_id()) self.assertEqual(place0dist.starting_trip, trip1dist.get_id()) self.assertEqual(round(trip0dist.duration), 58 * 60 + 51) self.assertEqual(round(trip1dist.duration), 38 * 60 + 57) self.assertIsNotNone(place0dist.location)