def setUp(self): self.clearRelatedDb() edb.get_trip_db().remove() edb.get_section_db().remove() edb.get_trip_new_db().remove() edb.get_section_new_db().remove() etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-21") eaicf.filter_accuracy(self.testUUID) estfm.move_all_filters_to_data() eaist.segment_current_trips(self.testUUID) eaiss.segment_current_sections(self.testUUID)
def read_data(uuid=None, size=None, old=True): db = edb.get_trip_db() if not old: logging.debug("not old") trips = esda.get_entries(esda.RAW_TRIP_KEY, uuid, time_query=None, geo_query=None) return trips if old: data = [] trip_db = db if uuid: trips = trip_db.find({'user_id' : uuid, 'type' : 'move'}) else: trips = trip_db.find({'type' : 'move'}) for t in trips: try: trip = Trip.trip_from_json(t) except: continue if not (trip.trip_start_location and trip.trip_end_location and trip.start_time): continue data.append(trip) if size: if len(data) == size: break return data
def getResult(user_uuid): # This is in here, as opposed to the top level as recommended by the PEP # because then we don't have to worry about loading bottle in the unit tests from bottle import template original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}}) if original_trip is None: return template("clients/recommendation/no_recommendation.html") del original_trip['trip_start_datetime'] del original_trip['trip_end_datetime'] del original_trip['user_id'] del original_trip['pipelineFlags'] del original_trip['recommended_alternative']['user_id'] recommended_trip = original_trip['recommended_alternative'] original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']})) for section in original_sections: del section['user_id'] del section['section_start_datetime'] del section['section_end_datetime'] if 'retained' in section: del section['retained'] del section['manual'] del section['commute'] logging.debug("original sections = %s" % original_sections) logging.debug("recommended trip = %s" % recommended_trip) renderedTemplate = template("clients/recommendation/result_template.html", originalSections = json.dumps(original_sections), recommendedTrip = json.dumps(recommended_trip)) return renderedTemplate
def getNoAlternatives(uid): # If pipelineFlags exists then we have started alternatives, and so have # already scheduled the query. No need to reschedule unless the query fails. # TODO: If the query fails, then remove the pipelineFlags so that we will # reschedule. query = {'user_id':uid, 'type':'move', 'pipelineFlags': {'$exists': False}} return get_trip_db().find(query)
def testPlaceLoad(self): result = self.loadTestJSON("emission/tests/data/test20140410") collect.processResult(self.testUUID, result) # Check that the trips are loaded correctly TripColl = get_trip_db() firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140409T191531-0700'}]}) logging.debug("selected trip = %s" % firstStoredTrip) # For some reason, the place is always "unknown", at least for this set of test trips. # Maybe it is related to the fact that they haven't been tagged in FourSquare self.assertEqual(firstStoredTrip['type'], 'place') self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700') self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700") self.assertIn('place_location', firstStoredTrip['place']) self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point', 'coordinates': [-122.08632, 37.391]}) # Now, check that we have the sections as well. The previous trip did not # have any sections. This one does tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140410T071320-0700'}]}) self.assertNotEqual(tripWithSections, None) self.assertEqual(tripWithSections['sections'], [0]) SectionColl = get_section_db() sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140410T071320-0700', 'section_id': 0}]}) self.assertNotEqual(sectionForTrip, None)
def obtain_alternatives(trip_id, user_id): db = edb.get_trip_db() trip = ecwt.E_Mission_Trip.trip_from_json(db.find_one({"trip_id": trip_id, "user_id": user_id})) logging.debug(trip.sections) start_coord = trip.trip_start_location.maps_coordinate() end_coord = trip.trip_end_location.maps_coordinate() logging.debug("Start: %s " % start_coord) logging.debug("End: %s " % end_coord) curr_time = datetime.datetime.now() curr_year = curr_time.year curr_month = curr_time.month curr_day = curr_time.day curr_hour = curr_time.hour curr_minute = curr_time.minute otp_modes = ['CAR', 'WALK', 'BICYCLE', 'TRANSIT'] for mode in otp_modes: try: otp_trip = otp.OTP(start_coord, end_coord, mode, write_day(curr_month, curr_day, curr_year), write_time(curr_hour, curr_minute), False) otp_trip = otp_trip.turn_into_trip(None, user_id, trip_id) otp_trip.save_to_db() except otp.PathNotFoundException as e: #modes = ['driving', 'walking', 'bicycling', 'transit'] logging.debug("Got error %s from OTP, defaulting to Google Maps" % e) otp_to_google_mode = {"CAR":"driving", "WALK":"walking", "BICYCLE":"bicycling", "TRANSIT":"transit"} mode = otp_to_google_mode[mode] gmaps = gmaps_lib.googlemaps.GoogleMaps('AIzaSyBEkw4PXVv_bsAdUmrFwatEyS6xLw3Bd9c') result = gmaps.directions(origin=start_coord, destination=end_coord, mode=mode) gmaps_trip = gmaps_lib.common.google_maps_to_our_trip(result, None, user_id, trip_id, mode, curr_time) gmaps_trip.save_to_db() '''
def save_trip_to_db(trip): db = edb.get_trip_db() print "start loc = %s" % trip.trip_start_location.coordinate_list() print "end loc = %s" % trip.trip_end_location.coordinate_list() db.insert({"_id": trip._id, "user_id": trip.user_id, "trip_id": trip.trip_id, "type" : "move", "sections": range(len(trip.sections)), "trip_start_datetime": trip.start_time, "trip_end_datetime": trip.end_time, "trip_start_location": trip.trip_start_location.coordinate_list(), "trip_end_location": trip.trip_end_location.coordinate_list(), "mode_list": trip.mode_list}) for section in trip.sections: save_section_to_db(section)
def testCreation(self): # This is mostly just a sanity check db = get_trip_db() db.remove() create_fake_trips() list_of_cluster_data = eamtcp.main() tm = create_tour_model('test_user', list_of_cluster_data) self.assertEquals(len(tm.get_top_trips(1)), 1) tour = tm.build_tour_model() self.assertEquals(len(tour), 7)
def save_to_db(self): db = edb.get_trip_db() result = db.update({"_id": self._id}, {"$set": {"mode" : self.mode_list, "confirmed_mode" : self.confirmed_mode_list}}, upsert=False,multi=False) print result if not result["updatedExisting"]: self._create_new(db) self._save_alternatives(self.alternatives) self._save_perturbed(self.perturbed_trips)
def loadPipelineFlags(self, _id): db = edb.get_trip_db() json_object = db.find_one({'_id': _id}) if json_object: tf = json_object.get('pipelineFlags') if tf: if tf['alternativesStarted'] == 'True': self.alternativesStarted = True if tf['alternativesFinished'] == 'True': self.alternativesFinished = True
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" #self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**", # "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' self.testUsers = [] #for userEmail in self.testUserEmails: # User.register(userEmail) # self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = edb.get_mode_db() self.ModesColl.remove() edb.get_trip_db().remove() edb.get_section_db().remove() edb.get_alternatives_db().remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) # register each of the users and add sample trips to each user result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) for trip in edb.get_trip_db().find(): trip['trip_start_datetime'] = pydt.datetime.now() + pydt.timedelta(hours=-5) trip['trip_end_datetime'] = pydt.datetime.now() edb.get_trip_db().update({"_id": trip["_id"]}, trip) for section in edb.get_section_db().find(): section['section_start_datetime'] = pydt.datetime.now() + pydt.timedelta(hours=-5) section['section_end_datetime'] = pydt.datetime.now() edb.get_section_db().update({"_id": section["_id"]}, section) self.pipeline = AlternativeTripsPipeline()
def save_trip_to_db(trip): print("saving trip to db") print(trip.user_id) db = edb.get_trip_db() print("RHRHRH") print("start loc = %s" % trip.trip_start_location.coordinate_list()) print("end loc = %s" % trip.trip_end_location.coordinate_list()) db.insert({"_id": trip._id, "user_id": trip.user_id, "trip_id": trip.trip_id, "type" : "move", "sections": list(range(len(trip.sections))), "trip_start_datetime": trip.start_time.datetime, "trip_end_datetime": trip.end_time.datetime, "trip_start_location": trip.trip_start_location.coordinate_list(), "trip_end_location": trip.trip_end_location.coordinate_list(), "mode_list": trip.mode_list}) print("len(trip.sections) in trip gen is %s" % len(trip.sections)) for section in trip.sections: save_section_to_db(section)
def testUpdateSectionForExistingTrip(self): result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) SectionColl = get_section_db() storedSections = SectionColl.find({'user_id': self.testUUID}) self.assertEquals(storedSections.count(), 21) # Trip 20140407T175709-0700 has two sections storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTripSections.count(), 2) TripColl = get_trip_db() storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTrips.count(), 1) for trip in storedTrips: self.assertEquals(len(trip['sections']), 2) selTripFromMoves = None for i, seg in enumerate(result[0]['segments']): if seg['startTime'] == '20140407T175709-0700': selTripFromMoves = seg copiedTripSections = [] for i, act in enumerate(selTripFromMoves['activities']): act['startTime'] = '20140407T18%s039-0700' % (i + 2) copiedTripSections.append(act) self.assertEquals(len(copiedTripSections), 2) [selTripFromMoves['activities'].append(act) for act in copiedTripSections] self.assertEquals(len(selTripFromMoves['activities']), 4) collect.processResult(self.testUUID, result) storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTripSections.count(), 4) storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTrips.count(), 1) # This is actually a bug in the existing code. Need to fix it. for trip in storedTrips: self.assertEquals(len(trip['sections']), 2)
def mark_recommended(self, alternative): db = edb.get_trip_db() ''' point_list = [] for section in self.sections: point_list.append([{'coordinates':[point.lon, point.lat]} for point in section.points]) ''' alternative_json = {"user_id": alternative.user_id, "trip_id": alternative.trip_id, "trip_start_time": alternative.start_time.strftime(DATE_FORMAT), "trip_end_time": alternative.end_time.strftime(DATE_FORMAT), "trip_start_location": alternative.trip_start_location.coordinate_list(), "trip_end_location": alternative.trip_end_location.coordinate_list(), "mode_list": alternative.mode_list, "track_points": alternative.track_points} print "recommending" result = db.update({"trip_id": self.trip_id, "user_id": self.user_id}, {"$set": {"recommended_alternative" : alternative_json}}, upsert=False,multi=False)
def set_up(_id): #fake_alternative = open("testTrip.txt", "r") g = sdk.GoogleMaps('AIzaSyBEkw4PXVv_bsAdUmrFwatEyS6xLw3Bd9c') #fake_alternative = json.loads(fake_alternative.read()) #t1 = google_maps_to_our_trip(g.directions('2703 Hallmark Dr Belmont CA', '2341 Ellsworth Berkeley CA', 0, 0, 0, 0) #t2 = google_maps_to_our_trip(g.directions('1114 Madera way Belmont CA', '2510 Bancroft st Berkeley CA', 0, 0, 0, 0) db = edb.get_trip_db() our_trip = et.E_Mission_Trip.trip_from_json(db.find_one({'trip_id':_id})) print "our_trip duration: " , our_trip.get_duration() pt = ep.find_perturbed_trips(our_trip) new_trip = pt[0] #print our_trip.trip_start_location #print our_trip.trip_end_location modes = ['driving', 'walking', 'bicycling', 'transit'] for mode in modes: thing = g.directions(our_trip.trip_start_location.maps_coordinate(), our_trip.trip_end_location.maps_coordinate(), mode=mode) alt = ec.google_maps_to_our_trip(thing, random.randint(1,10), 0, _id, 0, our_trip.start_time) print type(alt) alt.save_to_db() print "alternative duration: " , alt.get_duration()
def getResult(user_uuid): # This is in here, as opposed to the top level as recommended by the PEP # because then we don't have to worry about loading bottle in the unit tests from bottle import template original_trip = get_trip_db().find_one({ 'user_id': user_uuid, 'recommended_alternative': { '$exists': True } }) if original_trip is None: return template("clients/recommendation/no_recommendation.html") del original_trip['trip_start_datetime'] del original_trip['trip_end_datetime'] del original_trip['user_id'] del original_trip['pipelineFlags'] del original_trip['recommended_alternative']['user_id'] recommended_trip = original_trip['recommended_alternative'] original_sections = list(get_section_db().find( {'trip_id': original_trip['trip_id']})) for section in original_sections: del section['user_id'] del section['section_start_datetime'] del section['section_end_datetime'] if 'retained' in section: del section['retained'] del section['manual'] del section['commute'] logging.debug("original sections = %s" % original_sections) logging.debug("recommended trip = %s" % recommended_trip) renderedTemplate = template("clients/recommendation/result_template.html", originalSections=json.dumps(original_sections), recommendedTrip=json.dumps(recommended_trip)) return renderedTemplate
def testStoreSensedTrips(self): fakeEmail = "*****@*****.**" fakeUUID = User.fromEmail(fakeEmail).uuid trip_array = json.load(open("emission/tests/data/sensed_trips.json")) self.assertEqual(len(trip_array), 2) tripManager.storeSensedTrips(fakeUUID, trip_array) insertedTrips = [trip for trip in get_trip_db().find({"user_id": fakeUUID})] # We load two sections for each user in the setup. Here we only want to # look at sections that we added here. We distinguish between the two by looking # to see whether the predicted mode exists insertedSections = [section for section in get_section_db().find({"$and": [{"user_id": fakeUUID}, {"predicted_mode": {"$exists": False}}]})] # insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})] self.assertEqual(len(insertedTrips), 2) self.assertEqual(insertedTrips[0]["type"], "place") self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500") # self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52)) self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500") # self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51)) startPlaceLocation = insertedTrips[0]["place"]["place_location"] self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866]) self.assertEqual(insertedTrips[1]["type"], "move") self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500") self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500") self.assertEqual(len(insertedSections), 2) walkingSection = insertedSections[0] walkingTrackPointArray = insertedSections[0]["track_points"] self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500") self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500") self.assertEqual(walkingSection["duration"], 180631) self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2) self.assertEqual(len(walkingTrackPointArray), 7) self.assertEqual(walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866])
def save_trip_to_db(trip): print "saving trip to db" print trip.user_id db = edb.get_trip_db() print "RHRHRH" print "start loc = %s" % trip.trip_start_location.coordinate_list() print "end loc = %s" % trip.trip_end_location.coordinate_list() db.insert({ "_id": trip._id, "user_id": trip.user_id, "trip_id": trip.trip_id, "type": "move", "sections": range(len(trip.sections)), "trip_start_datetime": trip.start_time, "trip_end_datetime": trip.end_time, "trip_start_location": trip.trip_start_location.coordinate_list(), "trip_end_location": trip.trip_end_location.coordinate_list(), "mode_list": trip.mode_list }) print "len(trip.sections) in trip gen is %s" % len(trip.sections) for section in trip.sections: save_section_to_db(section)
def read_data(uuid=None,size=None): data = [] db = edb.get_trip_db() if uuid: trips = db.find({'user_id' : uuid, 'type' : 'move'}) else: trips = db.find({'type' : 'move'}) if trips.count() == 0: return [] for t in trips: try: trip = Trip.trip_from_json(t) except: continue if not (trip.trip_start_location and trip.trip_end_location and trip.start_time): continue data.append(trip) if size: if len(data) == size: break if len(data) == 0: return [] return data
def read_data(uuid=None, size=None, old=True): data = [] trip_db = edb.get_trip_db() if not old: trip_db = edb.get_trip_new_db() trips = trip_db.find({"user_id" : uuid}) else: if uuid: trips = trip_db.find({'user_id' : uuid, 'type' : 'move'}) else: trips = trip_db.find({'type' : 'move'}) for t in trips: try: trip = Trip.trip_from_json(t) except: continue if not (trip.trip_start_location and trip.trip_end_location and trip.start_time): continue data.append(trip) if size: if len(data) == size: break return data return [ecwt.Trip(trip) for trip in trips]
def get_training_uuid_list(): return edb.get_trip_db().find({}).distinct("user_id")
def get_recommender_uuid_list(): return edb.get_trip_db().find({}).distinct("user_id")
def processTripArray(user_uuid, trip_array): # First, we open a connection to the database Stage_Trips = get_trip_db() Stage_Sections = get_section_db() Modes = get_mode_db() number_of_trips = len(trip_array) logging.info("number of trips = %s" % number_of_trips) for trip in range(number_of_trips): seg_note = trip_array[trip] trip_id = seg_note["startTime"] _id_trip = str(user_uuid) + '_' + seg_note["startTime"] #logging.debug(json.dumps(seg_note)) if "activities" in seg_note: number_of_sections = len(seg_note["activities"]) logging.debug("number of sections = %s" % number_of_sections) for sectionindex in range(number_of_sections): seg_act_note = seg_note["activities"][sectionindex] # if the section is missing some data that we access later, then we skip it if Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }, { "section_id": sectionindex }] }).count() == 0: try: _id_section = str(user_uuid) + '_' + seg_act_note[ "startTime"] + '_' + str(sectionindex) _mode = convertModeNameToIndex( Modes, seg_act_note["activity"]) isManual = seg_act_note[ "manual"] if "manual" in seg_act_note else False sections_todo = { 'source': 'Shankari', '_id': _id_section, 'user_id': user_uuid, 'trip_id': trip_id, 'type': seg_note["type"], 'section_id': sectionindex, 'mode': _mode, # SHANKARI: what does seg_act_note["manual"] mean? 'confirmed_mode': _mode if isManual else '', # 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})]))) # if "group" in seg_act_note else '', } fillSectionWithMovesData(seg_act_note, sections_todo) label_filtered_section(sections_todo) # Now that we have created this section, let's insert it into the database try: logging.info( "About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " % (trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"])) Stage_Sections.insert(sections_todo) except DuplicateKeyError: logging.warning( "DuplicateKeyError, skipping insert %s" % sections_todo) logging.warning( "Existing section is %s" % Stage_Sections.find_one({"_id": _id_section})) except KeyError, e: logging.warning( "Missing key %s, skipping section insert %s" % (e, seg_act_note)) insertedSectionCount = Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }, { "section_id": sectionindex }] }).count() if insertedSectionCount == 0: logging.error( "Insert appears to have FAILED. No entry for %s, %s, %s found" % (user_uuid, trip_id, sectionindex)) else: logging.debug( "Found existing matching entry for %s, %s, %s, skipping entry" % (user_uuid, trip_id, sectionindex)) # Insert a trip if it doesn't already exist # SHANKARI: What if we get other sections for a trip later? When do we update the trip? # Do we even need to keep this linkage, with the concomittant # management cost if we can just find all sections by trip_id # instead? How expensive is the query? if Stage_Trips.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }] }).count() == 0: trips_todo = { 'source': 'Shankari', '_id': _id_trip, 'user_id': user_uuid, 'trip_id': trip_id, 'sections': [ sections['section_id'] for sections in Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }] }) ] } fillTripWithMovesData(seg_note, trips_todo) logging.info("About to insert trip with trip_id = %s " % (trip_id)) Stage_Trips.insert(trips_todo) else: logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def getAllTrips_Date(uid, dys): #trips = list(get_trip_db().find({"user_id":uid, "type":"move"})) d = datetime.datetime.now() - datetime.timedelta(days=dys) query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}} return get_trip_db().find(query)
def getAllTrips(uid): #trips = list(get_trip_db().find({"user_id":uid, "type":"move"})) query = {'user_id':uid, 'type':'move'} return get_trip_db().find(query)
def savePipelineFlags(self): db = edb.get_trip_db() db.update({"_id": self._id}, {"$set": {"pipelineFlags" : {'alternativesStarted': self.alternativesStarted, 'alternativesFinished': self.alternativesFinished}}}, multi=False, upsert=False)
def getNoAlternativesPastMonth(uid): d = datetime.datetime.now() - datetime.timedelta(days=30) query = {'user_id':uid, 'type':'move', 'trip_start_datetime':{"$gt":d}, 'pipelineFlags': {'$exists': False}} return get_trip_db().find(query)
def testUpdateSectionForExistingTrip(self): result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) SectionColl = get_section_db() storedSections = SectionColl.find({'user_id': self.testUUID}) self.assertEquals(storedSections.count(), 21) # Trip 20140407T175709-0700 has two sections storedTripSections = SectionColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTripSections.count(), 2) TripColl = get_trip_db() storedTrips = TripColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTrips.count(), 1) for trip in storedTrips: self.assertEquals(len(trip['sections']), 2) selTripFromMoves = None for i, seg in enumerate(result[0]['segments']): if seg['startTime'] == '20140407T175709-0700': selTripFromMoves = seg copiedTripSections = [] for i, act in enumerate(selTripFromMoves['activities']): act['startTime'] = '20140407T18%s039-0700' % (i + 2) copiedTripSections.append(act) self.assertEquals(len(copiedTripSections), 2) [ selTripFromMoves['activities'].append(act) for act in copiedTripSections ] self.assertEquals(len(selTripFromMoves['activities']), 4) collect.processResult(self.testUUID, result) storedTripSections = SectionColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTripSections.count(), 4) storedTrips = TripColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTrips.count(), 1) # This is actually a bug in the existing code. Need to fix it. for trip in storedTrips: self.assertEquals(len(trip['sections']), 2)
def testStoreSensedTrips(self): fakeEmail = "*****@*****.**" fakeUUID = User.fromEmail(fakeEmail).uuid trip_array = json.load(open("emission/tests/data/sensed_trips.json")) self.assertEqual(len(trip_array), 2) tripManager.storeSensedTrips(fakeUUID, trip_array) insertedTrips = [ trip for trip in get_trip_db().find({"user_id": fakeUUID}) ] # We load two sections for each user in the setup. Here we only want to # look at sections that we added here. We distinguish between the two by looking # to see whether the predicted mode exists insertedSections = [ section for section in get_section_db().find({ "$and": [{ "user_id": fakeUUID }, { "predicted_mode": { "$exists": False } }] }) ] # insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})] self.assertEqual(len(insertedTrips), 2) self.assertEqual(insertedTrips[0]["type"], "place") self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500") # self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52)) self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500") # self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51)) startPlaceLocation = insertedTrips[0]["place"]["place_location"] self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866]) self.assertEqual(insertedTrips[1]["type"], "move") self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500") self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500") self.assertEqual(len(insertedSections), 2) walkingSection = insertedSections[0] walkingTrackPointArray = insertedSections[0]["track_points"] self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500") self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500") self.assertEqual(walkingSection["duration"], 180631) self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2) self.assertEqual(len(walkingTrackPointArray), 7) self.assertEqual( walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866])
def setUpClass(cls): db = edb.get_trip_db()
def getTrainingTrips(uid): return getTrainingTrips_Date(uid, 30) query = {'user_id':uid, 'type':'move'} return get_trip_db().find(query)
def processTripArray(user_uuid, trip_array): # First, we open a connection to the database Stage_Trips=get_trip_db() Stage_Sections=get_section_db() Modes=get_mode_db() number_of_trips=len(trip_array) logging.info("number of trips = %s" % number_of_trips) for trip in range(number_of_trips): seg_note=trip_array[trip] trip_id=seg_note["startTime"] _id_trip=str(user_uuid)+'_'+seg_note["startTime"] #logging.debug(json.dumps(seg_note)) if "activities" in seg_note: number_of_sections=len(seg_note["activities"]) logging.debug("number of sections = %s" % number_of_sections) for sectionindex in range(number_of_sections): seg_act_note=seg_note["activities"][sectionindex] # if the section is missing some data that we access later, then we skip it if Stage_Sections.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id},{"section_id": sectionindex}]}).count()==0: try: _id_section = str(user_uuid)+'_'+seg_act_note["startTime"]+'_'+str(sectionindex) _mode = convertModeNameToIndex(Modes, seg_act_note["activity"]) isManual = seg_act_note["manual"] if "manual" in seg_act_note else False sections_todo={'source':'Shankari', '_id':_id_section, 'user_id': user_uuid, 'trip_id':trip_id, 'type':seg_note["type"], 'section_id':sectionindex, 'mode' : _mode, # SHANKARI: what does seg_act_note["manual"] mean? 'confirmed_mode' :_mode if isManual else '', # 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})]))) # if "group" in seg_act_note else '', } fillSectionWithMovesData(seg_act_note, sections_todo) label_filtered_section(sections_todo) # Now that we have created this section, let's insert it into the database try: logging.info("About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " % (trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"])) Stage_Sections.insert(sections_todo) except DuplicateKeyError: logging.warning("DuplicateKeyError, skipping insert %s" % sections_todo) logging.warning("Existing section is %s" % Stage_Sections.find_one({"_id": _id_section})) except KeyError, e: logging.warning("Missing key %s, skipping section insert %s" % (e, seg_act_note)) insertedSectionCount = Stage_Sections.find({"$and" : [{"user_id": user_uuid}, {"trip_id": trip_id}, {"section_id": sectionindex}]}).count() if insertedSectionCount == 0: logging.error("Insert appears to have FAILED. No entry for %s, %s, %s found" % (user_uuid, trip_id, sectionindex)) else: logging.debug("Found existing matching entry for %s, %s, %s, skipping entry" % (user_uuid, trip_id, sectionindex)) # Insert a trip if it doesn't already exist # SHANKARI: What if we get other sections for a trip later? When do we update the trip? # Do we even need to keep this linkage, with the concomittant # management cost if we can just find all sections by trip_id # instead? How expensive is the query? if Stage_Trips.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id}]}).count()==0: trips_todo={ 'source':'Shankari', '_id':_id_trip, 'user_id': user_uuid, 'trip_id':trip_id, 'sections':[sections['section_id'] for sections in Stage_Sections.find({"$and":[{"user_id":user_uuid}, {"trip_id":trip_id}]})]} fillTripWithMovesData(seg_note, trips_todo) logging.info("About to insert trip with trip_id = %s " % (trip_id)) Stage_Trips.insert(trips_todo) else: logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def getTrainingTrips_Date(uid, dys): d = datetime.datetime.now() - datetime.timedelta(days=dys) query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}, "pipelineFlags":{"$exists":True}} #query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}} #print get_trip_db().find(query).count() return get_trip_db().find(query)