def getResult(user_uuid): # This is in here, as opposed to the top level as recommended by the PEP # because then we don't have to worry about loading bottle in the unit tests from bottle import template original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}}) if original_trip is None: return template("clients/recommendation/no_recommendation.html") del original_trip['trip_start_datetime'] del original_trip['trip_end_datetime'] del original_trip['user_id'] del original_trip['pipelineFlags'] del original_trip['recommended_alternative']['user_id'] recommended_trip = original_trip['recommended_alternative'] original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']})) for section in original_sections: del section['user_id'] del section['section_start_datetime'] del section['section_end_datetime'] del section['retained'] del section['manual'] del section['commute'] renderedTemplate = template("clients/recommendation/result_template.html", originalSections = ast.literal_eval(json.dumps(original_sections)), recommendedTrip = ast.literal_eval(json.dumps(recommended_trip))) return renderedTemplate
def testPlaceLoad(self): result = self.loadTestJSON("tests/data/test20140410") collect.processResult(self.testUUID, result) # Check that the trips are loaded correctly TripColl = get_trip_db() firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140409T191531-0700'}]}) logging.debug("selected trip = %s" % firstStoredTrip) # For some reason, the place is always "unknown", at least for this set of test trips. # Maybe it is related to the fact that they haven't been tagged in FourSquare self.assertEqual(firstStoredTrip['type'], 'place') self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700') self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700") self.assertIn('place_location', firstStoredTrip['place']) self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point', 'coordinates': [-122.08632, 37.391]}) # Now, check that we have the sections as well. The previous trip did not # have any sections. This one does tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140410T071320-0700'}]}) self.assertNotEqual(tripWithSections, None) self.assertEqual(tripWithSections['sections'], [0]) SectionColl = get_section_db() sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140410T071320-0700', 'section_id': 0}]}) self.assertNotEqual(sectionForTrip, None)
def getNoAlternatives(uid): # If pipelineFlags exists then we have started alternatives, and so have # already scheduled the query. No need to reschedule unless the query fails. # TODO: If the query fails, then remove the pipelineFlags so that we will # reschedule. query = {'user_id':uid, 'type':'move', 'pipelineFlags': {'$exists': False}} return get_trip_db().find(query)
def getResult(user_uuid): # This is in here, as opposed to the top level as recommended by the PEP # because then we don't have to worry about loading bottle in the unit tests from bottle import template original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}}) if original_trip is None: return template("clients/recommendation/no_recommendation.html") del original_trip['trip_start_datetime'] del original_trip['trip_end_datetime'] del original_trip['user_id'] del original_trip['pipelineFlags'] del original_trip['recommended_alternative']['user_id'] recommended_trip = original_trip['recommended_alternative'] original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']})) for section in original_sections: del section['user_id'] del section['section_start_datetime'] del section['section_end_datetime'] if 'retained' in section: del section['retained'] del section['manual'] del section['commute'] renderedTemplate = template("clients/recommendation/result_template.html", originalSections = ast.literal_eval(json.dumps(original_sections)), recommendedTrip = ast.literal_eval(json.dumps(recommended_trip))) return renderedTemplate
def testPlaceLoad(self): result = self.loadTestJSON("tests/data/test20140410") collect.processResult(self.testUUID, result) # Check that the trips are loaded correctly TripColl = get_trip_db() firstStoredTrip = TripColl.find_one({ '$and': [{ 'user_id': self.testUUID, 'trip_id': '20140409T191531-0700' }] }) logging.debug("selected trip = %s" % firstStoredTrip) # For some reason, the place is always "unknown", at least for this set of test trips. # Maybe it is related to the fact that they haven't been tagged in FourSquare self.assertEqual(firstStoredTrip['type'], 'place') self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700') self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700") self.assertIn('place_location', firstStoredTrip['place']) self.assertEqual(firstStoredTrip['place']['place_location'], { 'type': 'Point', 'coordinates': [-122.08632, 37.391] }) # Now, check that we have the sections as well. The previous trip did not # have any sections. This one does tripWithSections = TripColl.find_one({ '$and': [{ 'user_id': self.testUUID, 'trip_id': '20140410T071320-0700' }] }) self.assertNotEqual(tripWithSections, None) self.assertEqual(tripWithSections['sections'], [0]) SectionColl = get_section_db() sectionForTrip = SectionColl.find_one({ '$and': [{ 'user_id': self.testUUID, 'trip_id': '20140410T071320-0700', 'section_id': 0 }] }) self.assertNotEqual(sectionForTrip, None)
def testUpdateSectionForExistingTrip(self): result = self.loadTestJSON("tests/data/missing_trip") collect.processResult(self.testUUID, result) SectionColl = get_section_db() storedSections = SectionColl.find({'user_id': self.testUUID}) self.assertEquals(storedSections.count(), 21) # Trip 20140407T175709-0700 has two sections storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTripSections.count(), 2) TripColl = get_trip_db() storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTrips.count(), 1) for trip in storedTrips: self.assertEquals(len(trip['sections']), 2) selTripFromMoves = None for i, seg in enumerate(result[0]['segments']): if seg['startTime'] == '20140407T175709-0700': selTripFromMoves = seg copiedTripSections = [] for i, act in enumerate(selTripFromMoves['activities']): act['startTime'] = '20140407T18%s039-0700' % (i + 2) copiedTripSections.append(act) self.assertEquals(len(copiedTripSections), 2) [selTripFromMoves['activities'].append(act) for act in copiedTripSections] self.assertEquals(len(selTripFromMoves['activities']), 4) collect.processResult(self.testUUID, result) storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTripSections.count(), 4) storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID}, {'trip_id': '20140407T175709-0700'}]}) self.assertEquals(storedTrips.count(), 1) # This is actually a bug in the existing code. Need to fix it. for trip in storedTrips: self.assertEquals(len(trip['sections']), 2)
def testStoreSensedTrips(self): fakeEmail = "*****@*****.**" fakeUUID = User.fromEmail(fakeEmail).uuid trip_array = json.load(open("tests/data/sensed_trips.json")) self.assertEqual(len(trip_array), 2) tripManager.storeSensedTrips(fakeUUID, trip_array) insertedTrips = [trip for trip in get_trip_db().find({"user_id": fakeUUID})] # We load two sections for each user in the setup. Here we only want to # look at sections that we added here. We distinguish between the two by looking # to see whether the predicted mode exists insertedSections = [section for section in get_section_db().find({"$and": [{"user_id": fakeUUID}, {"predicted_mode": {"$exists": False}}]})] # insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})] self.assertEqual(len(insertedTrips), 2) self.assertEqual(insertedTrips[0]["type"], "place") self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500") # self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52)) self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500") # self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51)) startPlaceLocation = insertedTrips[0]["place"]["place_location"] self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866]) self.assertEqual(insertedTrips[1]["type"], "move") self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500") self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500") self.assertEqual(len(insertedSections), 2) walkingSection = insertedSections[0] walkingTrackPointArray = insertedSections[0]["track_points"] self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500") self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500") self.assertEqual(walkingSection["duration"], 180631) self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2) self.assertEqual(len(walkingTrackPointArray), 10) self.assertEqual(walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866]) self.assertEqual(walkingTrackPointArray[8]["track_location"]["coordinates"], [-122.078265, 37.385461])
def testStoreSensedTrips(self): fakeEmail = "*****@*****.**" fakeUUID = User.fromEmail(fakeEmail).uuid trip_array = json.load(open("tests/data/sensed_trips.json")) self.assertEqual(len(trip_array), 2) tripManager.storeSensedTrips(fakeUUID, trip_array) insertedTrips = [trip for trip in get_trip_db().find({"user_id": fakeUUID})] # We load two sections for each user in the setup. Here we only want to # look at sections that we added here. We distinguish between the two by looking # to see whether the predicted mode exists insertedSections = [section for section in get_section_db().find({"$and": [{"user_id": fakeUUID}, {"predicted_mode": {"$exists": False}}]})] # insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})] self.assertEqual(len(insertedTrips), 2) self.assertEqual(insertedTrips[0]["type"], "place") self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500") # self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52)) self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500") # self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51)) startPlaceLocation = insertedTrips[0]["place"]["place_location"] self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866]) self.assertEqual(insertedTrips[1]["type"], "move") self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500") self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500") self.assertEqual(len(insertedSections), 2) walkingSection = insertedSections[0] walkingTrackPointArray = insertedSections[0]["track_points"] self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500") self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500") self.assertEqual(walkingSection["duration"], 180631) self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2) self.assertEqual(len(walkingTrackPointArray), 7) self.assertEqual(walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866])
def processTripArray(user_uuid, trip_array): # First, we open a connection to the database Stage_Trips = get_trip_db() Stage_Sections = get_section_db() Modes = get_mode_db() number_of_trips = len(trip_array) logging.info("number of trips = %s" % number_of_trips) for trip in range(number_of_trips): seg_note = trip_array[trip] trip_id = seg_note["startTime"] _id_trip = str(user_uuid) + '_' + seg_note["startTime"] #logging.debug(json.dumps(seg_note)) if "activities" in seg_note: number_of_sections = len(seg_note["activities"]) logging.debug("number of sections = %s" % number_of_sections) for sectionindex in range(number_of_sections): seg_act_note = seg_note["activities"][sectionindex] # if the section is missing some data that we access later, then we skip it if Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }, { "section_id": sectionindex }] }).count() == 0: try: _id_section = str(user_uuid) + '_' + seg_act_note[ "startTime"] + '_' + str(sectionindex) _mode = convertModeNameToIndex( Modes, seg_act_note["activity"]) isManual = seg_act_note[ "manual"] if "manual" in seg_act_note else False sections_todo = { 'source': 'Shankari', '_id': _id_section, 'user_id': user_uuid, 'trip_id': trip_id, 'type': seg_note["type"], 'section_id': sectionindex, 'mode': _mode, # SHANKARI: what does seg_act_note["manual"] mean? 'confirmed_mode': _mode if isManual else '', # 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})]))) # if "group" in seg_act_note else '', } fillSectionWithMovesData(seg_act_note, sections_todo) # Now that we have created this section, let's insert it into the database try: logging.info( "About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " % (trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"])) Stage_Sections.insert(sections_todo) except DuplicateKeyError: logging.warning( "DuplicateKeyError, skipping insert %s" % sections_todo) logging.warning( "Existing section is %s" % Stage_Sections.find_one({"_id": _id_section})) except KeyError, e: logging.warning( "Missing key %s, skipping section insert %s" % (e, seg_act_note)) insertedSectionCount = Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }, { "section_id": sectionindex }] }).count() if insertedSectionCount == 0: logging.error( "Insert appears to have FAILED. No entry for %s, %s, %s found" % (user_uuid, trip_id, sectionindex)) else: logging.debug( "Found existing matching entry for %s, %s, %s, skipping entry" % (user_uuid, trip_id, sectionindex)) # Insert a trip if it doesn't already exist # SHANKARI: What if we get other sections for a trip later? When do we update the trip? # Do we even need to keep this linkage, with the concomittant # management cost if we can just find all sections by trip_id # instead? How expensive is the query? if Stage_Trips.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }] }).count() == 0: trips_todo = { 'source': 'Shankari', '_id': _id_trip, 'user_id': user_uuid, 'trip_id': trip_id, 'sections': [ sections['section_id'] for sections in Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }] }) ] } fillTripWithMovesData(seg_note, trips_todo) logging.info("About to insert trip with trip_id = %s " % (trip_id)) Stage_Trips.insert(trips_todo) else: logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def testUpdateSectionForExistingTrip(self): result = self.loadTestJSON("tests/data/missing_trip") collect.processResult(self.testUUID, result) SectionColl = get_section_db() storedSections = SectionColl.find({'user_id': self.testUUID}) self.assertEquals(storedSections.count(), 21) # Trip 20140407T175709-0700 has two sections storedTripSections = SectionColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTripSections.count(), 2) TripColl = get_trip_db() storedTrips = TripColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTrips.count(), 1) for trip in storedTrips: self.assertEquals(len(trip['sections']), 2) selTripFromMoves = None for i, seg in enumerate(result[0]['segments']): if seg['startTime'] == '20140407T175709-0700': selTripFromMoves = seg copiedTripSections = [] for i, act in enumerate(selTripFromMoves['activities']): act['startTime'] = '20140407T18%s039-0700' % (i + 2) copiedTripSections.append(act) self.assertEquals(len(copiedTripSections), 2) [ selTripFromMoves['activities'].append(act) for act in copiedTripSections ] self.assertEquals(len(selTripFromMoves['activities']), 4) collect.processResult(self.testUUID, result) storedTripSections = SectionColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTripSections.count(), 4) storedTrips = TripColl.find({ '$and': [{ 'user_id': self.testUUID }, { 'trip_id': '20140407T175709-0700' }] }) self.assertEquals(storedTrips.count(), 1) # This is actually a bug in the existing code. Need to fix it. for trip in storedTrips: self.assertEquals(len(trip['sections']), 2)
def getAllTrips(uid): #trips = list(get_trip_db().find({"user_id":uid, "type":"move"})) query = {'user_id':uid, 'type':'move'} return get_trip_db().find(query)
def getTrainingTrips_Date(uid, dys): d = datetime.datetime.now() - datetime.timedelta(days=dys) query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}, "pipelineFlags":{"$exists":True}} #query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}} #print get_trip_db().find(query).count() return get_trip_db().find(query)
def getTrainingTrips(uid): return getTrainingTrips_Date(uid, 30) query = {'user_id':uid, 'type':'move'} return get_trip_db().find(query)
def getNoAlternativesPastMonth(uid): d = datetime.datetime.now() - datetime.timedelta(days=30) query = {'user_id':uid, 'type':'move', 'trip_start_datetime':{"$gt":d}, 'pipelineFlags': {'$exists': False}} return get_trip_db().find(query)
def getAllTrips_Date(uid, dys): #trips = list(get_trip_db().find({"user_id":uid, "type":"move"})) d = datetime.datetime.now() - datetime.timedelta(days=dys) query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}} return get_trip_db().find(query)
def processTripArray(user_uuid, trip_array): # First, we open a connection to the database Stage_Trips=get_trip_db() Stage_Sections=get_section_db() Modes=get_mode_db() number_of_trips=len(trip_array) logging.info("number of trips = %s" % number_of_trips) for trip in range(number_of_trips): seg_note=trip_array[trip] trip_id=seg_note["startTime"] _id_trip=str(user_uuid)+'_'+seg_note["startTime"] #logging.debug(json.dumps(seg_note)) if "activities" in seg_note: number_of_sections=len(seg_note["activities"]) logging.debug("number of sections = %s" % number_of_sections) for sectionindex in range(number_of_sections): seg_act_note=seg_note["activities"][sectionindex] # if the section is missing some data that we access later, then we skip it if Stage_Sections.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id},{"section_id": sectionindex}]}).count()==0: try: _id_section = str(user_uuid)+'_'+seg_act_note["startTime"]+'_'+str(sectionindex) _mode = convertModeNameToIndex(Modes, seg_act_note["activity"]) isManual = seg_act_note["manual"] if "manual" in seg_act_note else False sections_todo={'source':'Shankari', '_id':_id_section, 'user_id': user_uuid, 'trip_id':trip_id, 'type':seg_note["type"], 'section_id':sectionindex, 'mode' : _mode, # SHANKARI: what does seg_act_note["manual"] mean? 'confirmed_mode' :_mode if isManual else '', # 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})]))) # if "group" in seg_act_note else '', } fillSectionWithMovesData(seg_act_note, sections_todo) label_filtered_section(sections_todo) # Now that we have created this section, let's insert it into the database try: logging.info("About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " % (trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"])) Stage_Sections.insert(sections_todo) except DuplicateKeyError: logging.warning("DuplicateKeyError, skipping insert %s" % sections_todo) logging.warning("Existing section is %s" % Stage_Sections.find_one({"_id": _id_section})) except KeyError, e: logging.warning("Missing key %s, skipping section insert %s" % (e, seg_act_note)) insertedSectionCount = Stage_Sections.find({"$and" : [{"user_id": user_uuid}, {"trip_id": trip_id}, {"section_id": sectionindex}]}).count() if insertedSectionCount == 0: logging.error("Insert appears to have FAILED. No entry for %s, %s, %s found" % (user_uuid, trip_id, sectionindex)) else: logging.debug("Found existing matching entry for %s, %s, %s, skipping entry" % (user_uuid, trip_id, sectionindex)) # Insert a trip if it doesn't already exist # SHANKARI: What if we get other sections for a trip later? When do we update the trip? # Do we even need to keep this linkage, with the concomittant # management cost if we can just find all sections by trip_id # instead? How expensive is the query? if Stage_Trips.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id}]}).count()==0: trips_todo={ 'source':'Shankari', '_id':_id_trip, 'user_id': user_uuid, 'trip_id':trip_id, 'sections':[sections['section_id'] for sections in Stage_Sections.find({"$and":[{"user_id":user_uuid}, {"trip_id":trip_id}]})]} fillTripWithMovesData(seg_note, trips_todo) logging.info("About to insert trip with trip_id = %s " % (trip_id)) Stage_Trips.insert(trips_todo) else: logging.debug("Found existing trip with trip_id = %s " % (trip_id))