def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}})
if original_trip is None:
return template("clients/recommendation/no_recommendation.html")
del original_trip['trip_start_datetime']
del original_trip['trip_end_datetime']
del original_trip['user_id']
del original_trip['pipelineFlags']
del original_trip['recommended_alternative']['user_id']
recommended_trip = original_trip['recommended_alternative']
original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']}))
for section in original_sections:
del section['user_id']
del section['section_start_datetime']
del section['section_end_datetime']
del section['retained']
del section['manual']
del section['commute']
renderedTemplate = template("clients/recommendation/result_template.html",
originalSections = ast.literal_eval(json.dumps(original_sections)),
recommendedTrip = ast.literal_eval(json.dumps(recommended_trip)))
return renderedTemplate
def testPlaceLoad(self):
result = self.loadTestJSON("tests/data/test20140410")
collect.processResult(self.testUUID, result)
# Check that the trips are loaded correctly
TripColl = get_trip_db()
firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140409T191531-0700'}]})
logging.debug("selected trip = %s" % firstStoredTrip)
# For some reason, the place is always "unknown", at least for this set of test trips.
# Maybe it is related to the fact that they haven't been tagged in FourSquare
self.assertEqual(firstStoredTrip['type'], 'place')
self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700')
self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700")
self.assertIn('place_location', firstStoredTrip['place'])
self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point',
'coordinates': [-122.08632, 37.391]})
# Now, check that we have the sections as well. The previous trip did not
# have any sections. This one does
tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700'}]})
self.assertNotEqual(tripWithSections, None)
self.assertEqual(tripWithSections['sections'], [0])
SectionColl = get_section_db()
sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700',
'section_id': 0}]})
self.assertNotEqual(sectionForTrip, None)
def getNoAlternatives(uid):
# If pipelineFlags exists then we have started alternatives, and so have
# already scheduled the query. No need to reschedule unless the query fails.
# TODO: If the query fails, then remove the pipelineFlags so that we will
# reschedule.
query = {'user_id':uid, 'type':'move', 'pipelineFlags': {'$exists': False}}
return get_trip_db().find(query)
def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}})
if original_trip is None:
return template("clients/recommendation/no_recommendation.html")
del original_trip['trip_start_datetime']
del original_trip['trip_end_datetime']
del original_trip['user_id']
del original_trip['pipelineFlags']
del original_trip['recommended_alternative']['user_id']
recommended_trip = original_trip['recommended_alternative']
original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']}))
for section in original_sections:
del section['user_id']
del section['section_start_datetime']
del section['section_end_datetime']
if 'retained' in section:
del section['retained']
del section['manual']
del section['commute']
renderedTemplate = template("clients/recommendation/result_template.html",
originalSections = ast.literal_eval(json.dumps(original_sections)),
recommendedTrip = ast.literal_eval(json.dumps(recommended_trip)))
return renderedTemplate
def testPlaceLoad(self):
result = self.loadTestJSON("tests/data/test20140410")
collect.processResult(self.testUUID, result)
# Check that the trips are loaded correctly
TripColl = get_trip_db()
firstStoredTrip = TripColl.find_one({
'$and': [{
'user_id': self.testUUID,
'trip_id': '20140409T191531-0700'
}]
})
logging.debug("selected trip = %s" % firstStoredTrip)
# For some reason, the place is always "unknown", at least for this set of test trips.
# Maybe it is related to the fact that they haven't been tagged in FourSquare
self.assertEqual(firstStoredTrip['type'], 'place')
self.assertEqual(firstStoredTrip['trip_start_time'],
'20140409T191531-0700')
self.assertEqual(firstStoredTrip['trip_end_time'],
"20140410T065227-0700")
self.assertIn('place_location', firstStoredTrip['place'])
self.assertEqual(firstStoredTrip['place']['place_location'], {
'type': 'Point',
'coordinates': [-122.08632, 37.391]
})
# Now, check that we have the sections as well. The previous trip did not
# have any sections. This one does
tripWithSections = TripColl.find_one({
'$and': [{
'user_id': self.testUUID,
'trip_id': '20140410T071320-0700'
}]
})
self.assertNotEqual(tripWithSections, None)
self.assertEqual(tripWithSections['sections'], [0])
SectionColl = get_section_db()
sectionForTrip = SectionColl.find_one({
'$and': [{
'user_id': self.testUUID,
'trip_id': '20140410T071320-0700',
'section_id': 0
}]
})
self.assertNotEqual(sectionForTrip, None)
def testUpdateSectionForExistingTrip(self):
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
storedSections = SectionColl.find({'user_id': self.testUUID})
self.assertEquals(storedSections.count(), 21)
# Trip 20140407T175709-0700 has two sections
storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTripSections.count(), 2)
TripColl = get_trip_db()
storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTrips.count(), 1)
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
selTripFromMoves = None
for i, seg in enumerate(result[0]['segments']):
if seg['startTime'] == '20140407T175709-0700':
selTripFromMoves = seg
copiedTripSections = []
for i, act in enumerate(selTripFromMoves['activities']):
act['startTime'] = '20140407T18%s039-0700' % (i + 2)
copiedTripSections.append(act)
self.assertEquals(len(copiedTripSections), 2)
[selTripFromMoves['activities'].append(act) for act in copiedTripSections]
self.assertEquals(len(selTripFromMoves['activities']), 4)
collect.processResult(self.testUUID, result)
storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTripSections.count(), 4)
storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTrips.count(), 1)
# This is actually a bug in the existing code. Need to fix it.
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
def testStoreSensedTrips(self):
fakeEmail = "*****@*****.**"
fakeUUID = User.fromEmail(fakeEmail).uuid
trip_array = json.load(open("tests/data/sensed_trips.json"))
self.assertEqual(len(trip_array), 2)
tripManager.storeSensedTrips(fakeUUID, trip_array)
insertedTrips = [trip for trip in get_trip_db().find({"user_id": fakeUUID})]
# We load two sections for each user in the setup. Here we only want to
# look at sections that we added here. We distinguish between the two by looking
# to see whether the predicted mode exists
insertedSections = [section for section in get_section_db().find({"$and":
[{"user_id": fakeUUID}, {"predicted_mode": {"$exists": False}}]})]
# insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})]
self.assertEqual(len(insertedTrips), 2)
self.assertEqual(insertedTrips[0]["type"], "place")
self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500")
# self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52))
self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500")
# self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51))
startPlaceLocation = insertedTrips[0]["place"]["place_location"]
self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866])
self.assertEqual(insertedTrips[1]["type"], "move")
self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500")
self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500")
self.assertEqual(len(insertedSections), 2)
walkingSection = insertedSections[0]
walkingTrackPointArray = insertedSections[0]["track_points"]
self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500")
self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500")
self.assertEqual(walkingSection["duration"], 180631)
self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2)
self.assertEqual(len(walkingTrackPointArray), 10)
self.assertEqual(walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866])
self.assertEqual(walkingTrackPointArray[8]["track_location"]["coordinates"], [-122.078265, 37.385461])
def testStoreSensedTrips(self):
fakeEmail = "*****@*****.**"
fakeUUID = User.fromEmail(fakeEmail).uuid
trip_array = json.load(open("tests/data/sensed_trips.json"))
self.assertEqual(len(trip_array), 2)
tripManager.storeSensedTrips(fakeUUID, trip_array)
insertedTrips = [trip for trip in get_trip_db().find({"user_id": fakeUUID})]
# We load two sections for each user in the setup. Here we only want to
# look at sections that we added here. We distinguish between the two by looking
# to see whether the predicted mode exists
insertedSections = [section for section in get_section_db().find({"$and":
[{"user_id": fakeUUID}, {"predicted_mode": {"$exists": False}}]})]
# insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})]
self.assertEqual(len(insertedTrips), 2)
self.assertEqual(insertedTrips[0]["type"], "place")
self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500")
# self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52))
self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500")
# self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51))
startPlaceLocation = insertedTrips[0]["place"]["place_location"]
self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866])
self.assertEqual(insertedTrips[1]["type"], "move")
self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500")
self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500")
self.assertEqual(len(insertedSections), 2)
walkingSection = insertedSections[0]
walkingTrackPointArray = insertedSections[0]["track_points"]
self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500")
self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500")
self.assertEqual(walkingSection["duration"], 180631)
self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2)
self.assertEqual(len(walkingTrackPointArray), 7)
self.assertEqual(walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866])
def processTripArray(user_uuid, trip_array):
# First, we open a connection to the database
Stage_Trips = get_trip_db()
Stage_Sections = get_section_db()
Modes = get_mode_db()
number_of_trips = len(trip_array)
logging.info("number of trips = %s" % number_of_trips)
for trip in range(number_of_trips):
seg_note = trip_array[trip]
trip_id = seg_note["startTime"]
_id_trip = str(user_uuid) + '_' + seg_note["startTime"]
#logging.debug(json.dumps(seg_note))
if "activities" in seg_note:
number_of_sections = len(seg_note["activities"])
logging.debug("number of sections = %s" % number_of_sections)
for sectionindex in range(number_of_sections):
seg_act_note = seg_note["activities"][sectionindex]
# if the section is missing some data that we access later, then we skip it
if Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}, {
"section_id": sectionindex
}]
}).count() == 0:
try:
_id_section = str(user_uuid) + '_' + seg_act_note[
"startTime"] + '_' + str(sectionindex)
_mode = convertModeNameToIndex(
Modes, seg_act_note["activity"])
isManual = seg_act_note[
"manual"] if "manual" in seg_act_note else False
sections_todo = {
'source': 'Shankari',
'_id': _id_section,
'user_id': user_uuid,
'trip_id': trip_id,
'type': seg_note["type"],
'section_id': sectionindex,
'mode': _mode,
# SHANKARI: what does seg_act_note["manual"] mean?
'confirmed_mode': _mode if isManual else '',
# 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})])))
# if "group" in seg_act_note else '',
}
fillSectionWithMovesData(seg_act_note, sections_todo)
# Now that we have created this section, let's insert it into the database
try:
logging.info(
"About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s "
% (trip_id, sectionindex,
sections_todo['section_start_time'],
seg_note["type"], seg_act_note["activity"]))
Stage_Sections.insert(sections_todo)
except DuplicateKeyError:
logging.warning(
"DuplicateKeyError, skipping insert %s" %
sections_todo)
logging.warning(
"Existing section is %s" %
Stage_Sections.find_one({"_id": _id_section}))
except KeyError, e:
logging.warning(
"Missing key %s, skipping section insert %s" %
(e, seg_act_note))
insertedSectionCount = Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}, {
"section_id": sectionindex
}]
}).count()
if insertedSectionCount == 0:
logging.error(
"Insert appears to have FAILED. No entry for %s, %s, %s found"
% (user_uuid, trip_id, sectionindex))
else:
logging.debug(
"Found existing matching entry for %s, %s, %s, skipping entry"
% (user_uuid, trip_id, sectionindex))
# Insert a trip if it doesn't already exist
# SHANKARI: What if we get other sections for a trip later? When do we update the trip?
# Do we even need to keep this linkage, with the concomittant
# management cost if we can just find all sections by trip_id
# instead? How expensive is the query?
if Stage_Trips.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}]
}).count() == 0:
trips_todo = {
'source':
'Shankari',
'_id':
_id_trip,
'user_id':
user_uuid,
'trip_id':
trip_id,
'sections': [
sections['section_id']
for sections in Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}]
})
]
}
fillTripWithMovesData(seg_note, trips_todo)
logging.info("About to insert trip with trip_id = %s " % (trip_id))
Stage_Trips.insert(trips_todo)
else:
logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def testUpdateSectionForExistingTrip(self):
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
storedSections = SectionColl.find({'user_id': self.testUUID})
self.assertEquals(storedSections.count(), 21)
# Trip 20140407T175709-0700 has two sections
storedTripSections = SectionColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTripSections.count(), 2)
TripColl = get_trip_db()
storedTrips = TripColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTrips.count(), 1)
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
selTripFromMoves = None
for i, seg in enumerate(result[0]['segments']):
if seg['startTime'] == '20140407T175709-0700':
selTripFromMoves = seg
copiedTripSections = []
for i, act in enumerate(selTripFromMoves['activities']):
act['startTime'] = '20140407T18%s039-0700' % (i + 2)
copiedTripSections.append(act)
self.assertEquals(len(copiedTripSections), 2)
[
selTripFromMoves['activities'].append(act)
for act in copiedTripSections
]
self.assertEquals(len(selTripFromMoves['activities']), 4)
collect.processResult(self.testUUID, result)
storedTripSections = SectionColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTripSections.count(), 4)
storedTrips = TripColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTrips.count(), 1)
# This is actually a bug in the existing code. Need to fix it.
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
def getAllTrips(uid):
#trips = list(get_trip_db().find({"user_id":uid, "type":"move"}))
query = {'user_id':uid, 'type':'move'}
return get_trip_db().find(query)
def getTrainingTrips_Date(uid, dys):
d = datetime.datetime.now() - datetime.timedelta(days=dys)
query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}, "pipelineFlags":{"$exists":True}}
#query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}}
#print get_trip_db().find(query).count()
return get_trip_db().find(query)
def getTrainingTrips(uid):
return getTrainingTrips_Date(uid, 30)
query = {'user_id':uid, 'type':'move'}
return get_trip_db().find(query)
def getNoAlternativesPastMonth(uid):
d = datetime.datetime.now() - datetime.timedelta(days=30)
query = {'user_id':uid, 'type':'move',
'trip_start_datetime':{"$gt":d},
'pipelineFlags': {'$exists': False}}
return get_trip_db().find(query)
def getAllTrips_Date(uid, dys):
#trips = list(get_trip_db().find({"user_id":uid, "type":"move"}))
d = datetime.datetime.now() - datetime.timedelta(days=dys)
query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}}
return get_trip_db().find(query)
def processTripArray(user_uuid, trip_array):
# First, we open a connection to the database
Stage_Trips=get_trip_db()
Stage_Sections=get_section_db()
Modes=get_mode_db()
number_of_trips=len(trip_array)
logging.info("number of trips = %s" % number_of_trips)
for trip in range(number_of_trips):
seg_note=trip_array[trip]
trip_id=seg_note["startTime"]
_id_trip=str(user_uuid)+'_'+seg_note["startTime"]
#logging.debug(json.dumps(seg_note))
if "activities" in seg_note:
number_of_sections=len(seg_note["activities"])
logging.debug("number of sections = %s" % number_of_sections)
for sectionindex in range(number_of_sections):
seg_act_note=seg_note["activities"][sectionindex]
# if the section is missing some data that we access later, then we skip it
if Stage_Sections.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id},{"section_id": sectionindex}]}).count()==0:
try:
_id_section = str(user_uuid)+'_'+seg_act_note["startTime"]+'_'+str(sectionindex)
_mode = convertModeNameToIndex(Modes, seg_act_note["activity"])
isManual = seg_act_note["manual"] if "manual" in seg_act_note else False
sections_todo={'source':'Shankari',
'_id':_id_section,
'user_id': user_uuid,
'trip_id':trip_id,
'type':seg_note["type"],
'section_id':sectionindex,
'mode' : _mode,
# SHANKARI: what does seg_act_note["manual"] mean?
'confirmed_mode' :_mode if isManual else '',
# 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})])))
# if "group" in seg_act_note else '',
}
fillSectionWithMovesData(seg_act_note, sections_todo)
label_filtered_section(sections_todo)
# Now that we have created this section, let's insert it into the database
try:
logging.info("About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " %
(trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"]))
Stage_Sections.insert(sections_todo)
except DuplicateKeyError:
logging.warning("DuplicateKeyError, skipping insert %s" % sections_todo)
logging.warning("Existing section is %s" % Stage_Sections.find_one({"_id": _id_section}))
except KeyError, e:
logging.warning("Missing key %s, skipping section insert %s" % (e, seg_act_note))
insertedSectionCount = Stage_Sections.find({"$and" : [{"user_id": user_uuid},
{"trip_id": trip_id},
{"section_id": sectionindex}]}).count()
if insertedSectionCount == 0:
logging.error("Insert appears to have FAILED. No entry for %s, %s, %s found" %
(user_uuid, trip_id, sectionindex))
else:
logging.debug("Found existing matching entry for %s, %s, %s, skipping entry" %
(user_uuid, trip_id, sectionindex))
# Insert a trip if it doesn't already exist
# SHANKARI: What if we get other sections for a trip later? When do we update the trip?
# Do we even need to keep this linkage, with the concomittant
# management cost if we can just find all sections by trip_id
# instead? How expensive is the query?
if Stage_Trips.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id}]}).count()==0:
trips_todo={ 'source':'Shankari',
'_id':_id_trip,
'user_id': user_uuid,
'trip_id':trip_id,
'sections':[sections['section_id'] for sections in Stage_Sections.find({"$and":[{"user_id":user_uuid}, {"trip_id":trip_id}]})]}
fillTripWithMovesData(seg_note, trips_todo)
logging.info("About to insert trip with trip_id = %s " % (trip_id))
Stage_Trips.insert(trips_todo)
else:
logging.debug("Found existing trip with trip_id = %s " % (trip_id))
