def testFeatureGenWithOnePoint(self): # ensure that the start and end datetimes are the same, since the average calculation uses # the total distance and the total duration ts = esta.TimeSeries.get_time_series(self.testUUID) trackpoint1 = ecwlo.Location({u'coordinates': [0, 0], 'type': 'Point'}) ts.insert_data(self.testUUID, "analysis/recreated_location", trackpoint1) testSeg = ecws.Section({ "start_loc": trackpoint1, "end_loc": trackpoint1, "distance": 500, "sensed_mode": 1, "duration": 150, "start_ts": arrow.now().timestamp, "end_ts": arrow.now().timestamp, "_id": 2, "speeds": [], "distances": [], }) testSegEntry = ecwe.Entry.create_entry(self.testUUID, "analysis/cleaned_section", testSeg) d = testSegEntry.data m = testSegEntry.metadata enufc.expand_start_end_data_times(d, m) testSegEntry["data"] = d testSegEntry["metadata"] = m inserted_id = ts.insert(testSegEntry) featureMatrix = np.zeros([1, len(self.pipeline.featureLabels)]) resultVector = np.zeros(1) self.pipeline.updateFeatureMatrixRowWithSection( featureMatrix, 0, testSegEntry) logging.debug("featureMatrix = %s" % featureMatrix) self.assertEqual(np.count_nonzero(featureMatrix[0][5:16]), 0) self.assertEqual(np.count_nonzero(featureMatrix[0][19:21]), 0)
def turn_into_new_trip(self, user_id): print("new trip") ts = esta.TimeSeries.get_time_series(user_id) trip = ecwrt.Rawtrip() sections = [] our_json = self.get_json() mode_list = set ( ) if "plan" not in our_json: print("While querying alternatives from %s to %s" % (self.start_point, self.end_point)) print("query URL is %s" % self.make_url()) print("Response %s does not have a plan " % our_json) raise PathNotFoundException(our_json['debugOutput']) trip.start_loc = gj.Point( (float(our_json["plan"]["from"]["lat"]), float(our_json["plan"]["from"]["lon"])) ) trip.end_loc = gj.Point( (float(our_json["plan"]["to"]["lat"]), float(our_json["plan"]["to"]["lon"])) ) trip.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours( our_json['plan']['itineraries'][0]["startTime"]).timestamp, "UTC") trip.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours( our_json['plan']['itineraries'][0]["endTime"]).timestamp, "UTC") trip_id = ts.insert(ecwe.Entry.create_entry(user_id, "segmentation/raw_trip", trip)) for leg in our_json["plan"]["itineraries"][0]['legs']: section = ecws.Section() section.trip_id = trip_id section.start_local_dt = ecsdlq.get_local_date(otp_time_to_ours( leg["startTime"]).timestamp, "UTC") section.end_local_dt = ecsdlq.get_local_date(otp_time_to_ours( leg["endTime"]).timestamp, "UTC") section.distance = float(leg["distance"]) section.start_loc = gj.Point( (float(leg["from"]["lat"]), float(leg["from"]["lon"])) ) section.end_loc = gj.Point( (float(leg["to"]["lat"]), float(leg["to"]["lon"])) ) ts.insert_data(user_id, "segmentation/raw_section", section)
def setUp(self): # We need to access the database directly sometimes in order to # forcibly insert entries for the tests to pass. But we put the import # in here to reduce the temptation to use the database directly elsewhere. import emission.core.get_database as edb import uuid self.testUUID = uuid.uuid4() self.trips = json.load( open("emission/tests/data/smoothing_data/trip_list.txt"), object_hook=bju.object_hook) for trip in self.trips: trip["user_id"] = self.testUUID edb.get_trip_new_db().save(trip) self.trips = [ecwt.Trip(t) for t in self.trips] self.sections = json.load( open("emission/tests/data/smoothing_data/section_list.txt"), object_hook=bju.object_hook) for section in self.sections: section["user_id"] = self.testUUID edb.get_section_new_db().save(section) self.sections = [ecws.Section(s) for s in self.sections] self.ts = esta.TimeSeries.get_time_series(self.testUUID)
def get_sections_for_trip(user_id, trip_id): """ Get the set of sections that are children of this trip. """ section_doc_cursor = edb.get_section_new_db().find({ "user_id": user_id, "trip_id": trip_id }).sort("start_ts", pymongo.ASCENDING) return [ecws.Section(doc) for doc in section_doc_cursor]
def testQuerySectionsForTrip(self): new_trip = self.create_fake_trip() new_section = ecwc.Section() new_section.trip_id = new_trip.get_id() new_section.start_ts = 5 new_section.end_ts = 6 ts = esta.TimeSeries.get_time_series(self.testUserId) ts.insert_data(self.testUserId, esda.RAW_SECTION_KEY, new_section) ret_entries = esdt.get_raw_sections_for_trip(self.testUserId, new_trip.get_id()) self.assertEqual([entry.data for entry in ret_entries], [new_section])
def _createTestSection(self, start_ardt, start_timezone): section = ecws.Section() self._fillDates(section, "start_", start_ardt, start_timezone) # Hackily fill in the end with the same values as the start # so that the field exists # in cases where the end is important (mainly for range timezone # calculation with local times), it can be overridden using _fillDates # from the test case self._fillDates(section, "end_", start_ardt, start_timezone) #logging.debug("created section %s" % (section.start_fmt_time)) entry = ecwe.Entry.create_entry(self.testUUID, esda.CLEANED_SECTION_KEY, section, create_id=True) self.ts.insert(entry) return entry
def testQuerySections(self): new_section = ecws.Section() new_section.start_ts = 5 new_section.end_ts = 6 new_section.trip_id = self.test_trip_id esta.TimeSeries.get_time_series(self.testUserId).insert_data(self.testUserId, esda.RAW_SECTION_KEY, new_section) ret_arr_one = esds.get_sections_for_trip(self.testUserId, self.test_trip_id) self.assertEqual(len(ret_arr_one), 1) self.assertEqual([entry.data for entry in ret_arr_one], [new_section]) ret_arr_list = esds.get_sections_for_trip_list(self.testUserId, [self.test_trip_id]) self.assertEqual(ret_arr_one, ret_arr_list) ret_arr_time = esda.get_objects(esda.RAW_SECTION_KEY, self.testUserId, estt.TimeQuery("data.start_ts", 4, 6)) self.assertEqual([entry.data for entry in ret_arr_list], ret_arr_time)
def segment_trip_into_sections(user_id, trip_entry, trip_source): ts = esta.TimeSeries.get_time_series(user_id) time_query = esda.get_time_query_for_trip_like(esda.RAW_TRIP_KEY, trip_entry.get_id()) distance_from_place = _get_distance_from_start_place_to_end(trip_entry) if (trip_source == "DwellSegmentationTimeFilter"): import emission.analysis.intake.segmentation.section_segmentation_methods.smoothed_high_confidence_motion as shcm shcmsm = shcm.SmoothedHighConfidenceMotion(60, 100, [ ecwm.MotionTypes.TILTING, ecwm.MotionTypes.UNKNOWN, ecwm.MotionTypes.STILL ]) else: assert (trip_source == "DwellSegmentationDistFilter") import emission.analysis.intake.segmentation.section_segmentation_methods.smoothed_high_confidence_with_visit_transitions as shcmvt shcmsm = shcmvt.SmoothedHighConfidenceMotionWithVisitTransitions( 49, 50, [ ecwm.MotionTypes.TILTING, ecwm.MotionTypes.UNKNOWN, ecwm.MotionTypes.STILL, ecwm.MotionTypes.NONE, # iOS only ecwm.MotionTypes.STOPPED_WHILE_IN_VEHICLE ]) # iOS only segmentation_points = shcmsm.segment_into_sections(ts, distance_from_place, time_query) # Since we are segmenting an existing trip into sections, we do not need to worry about linking with # a prior place, since it will be linked through the trip object. # So this is much simpler than the trip case. # Again, since this is segmenting a trip, we can just start with a section prev_section_entry = None # TODO: Should we link the locations to the trips this way, or by using a foreign key? # If we want to use a foreign key, then we need to include the object id in the data df as well so that we can # set it properly. ts = esta.TimeSeries.get_time_series(user_id) get_loc_for_ts = lambda time: ecwl.Location( ts.get_entry_at_ts("background/filtered_location", "data.ts", time)[ "data"]) trip_start_loc = get_loc_for_ts(trip_entry.data.start_ts) trip_end_loc = get_loc_for_ts(trip_entry.data.end_ts) logging.debug("trip_start_loc = %s, trip_end_loc = %s" % (trip_start_loc, trip_end_loc)) for (i, (start_loc_doc, end_loc_doc, sensed_mode)) in enumerate(segmentation_points): logging.debug("start_loc_doc = %s, end_loc_doc = %s" % (start_loc_doc, end_loc_doc)) get_loc_for_row = lambda row: ts.df_row_to_entry( "background/filtered_location", row).data start_loc = get_loc_for_row(start_loc_doc) end_loc = get_loc_for_row(end_loc_doc) logging.debug("start_loc = %s, end_loc = %s" % (start_loc, end_loc)) section = ecwc.Section() section.trip_id = trip_entry.get_id() if prev_section_entry is None: # This is the first point, so we want to start from the start of the trip, not the start of this segment start_loc = trip_start_loc if i == len(segmentation_points) - 1: # This is the last point, so we want to end at the end of the trip, not at the end of this segment # Particularly in this case, if we don't do this, then the trip end may overshoot the section end end_loc = trip_end_loc fill_section(section, start_loc, end_loc, sensed_mode) # We create the entry after filling in the section so that we know # that the data is included properly section_entry = ecwe.Entry.create_entry(user_id, esda.RAW_SECTION_KEY, section, create_id=True) if prev_section_entry is not None: # If this is not the first section, create a stop to link the two sections together # The expectation is prev_section -> stop -> curr_section stop = ecws.Stop() stop.trip_id = trip_entry.get_id() stop_entry = ecwe.Entry.create_entry(user_id, esda.RAW_STOP_KEY, stop, create_id=True) logging.debug("stop = %s, stop_entry = %s" % (stop, stop_entry)) stitch_together(prev_section_entry, stop_entry, section_entry) ts.insert(stop_entry) ts.update(prev_section_entry) # After we go through the loop, we will be left with the last section, # which does not have an ending stop. We insert that too. ts.insert(section_entry) prev_section_entry = section_entry
def create_new_section(user_id, trip_id): _id = edb.get_section_new_db().save({ "user_id": user_id, "trip_id": trip_id }) return ecws.Section({"_id": _id, "user_id": user_id, "trip_id": trip_id})
def _get_sections_for_query(section_query, sort_field): logging.debug("Returning sections for query %s" % section_query) section_doc_cursor = edb.get_section_new_db().find(section_query).sort( sort_field, pymongo.ASCENDING) # TODO: Fix "TripIterator" and return it instead of this list return [ecws.Section(doc) for doc in section_doc_cursor]
def get_section(section_id): return ecws.Section(edb.get_section_new_db().find_one({"_id": section_id}))