def setUp(self):
self.clearRelatedDb()
edb.get_trip_db().remove()
edb.get_section_db().remove()
edb.get_trip_new_db().remove()
edb.get_section_new_db().remove()
etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-21")
eaicf.filter_accuracy(self.testUUID)
estfm.move_all_filters_to_data()
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
def read_data(uuid=None, size=None, old=True):
db = edb.get_trip_db()
if not old:
logging.debug("not old")
trips = esda.get_entries(esda.RAW_TRIP_KEY, uuid,
time_query=None, geo_query=None)
return trips
if old:
data = []
trip_db = db
if uuid:
trips = trip_db.find({'user_id' : uuid, 'type' : 'move'})
else:
trips = trip_db.find({'type' : 'move'})
for t in trips:
try:
trip = Trip.trip_from_json(t)
except:
continue
if not (trip.trip_start_location and trip.trip_end_location and trip.start_time):
continue
data.append(trip)
if size:
if len(data) == size:
break
return data
def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}})
if original_trip is None:
return template("clients/recommendation/no_recommendation.html")
del original_trip['trip_start_datetime']
del original_trip['trip_end_datetime']
del original_trip['user_id']
del original_trip['pipelineFlags']
del original_trip['recommended_alternative']['user_id']
recommended_trip = original_trip['recommended_alternative']
original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']}))
for section in original_sections:
del section['user_id']
del section['section_start_datetime']
del section['section_end_datetime']
if 'retained' in section:
del section['retained']
del section['manual']
del section['commute']
logging.debug("original sections = %s" % original_sections)
logging.debug("recommended trip = %s" % recommended_trip)
renderedTemplate = template("clients/recommendation/result_template.html",
originalSections = json.dumps(original_sections),
recommendedTrip = json.dumps(recommended_trip))
return renderedTemplate
def getNoAlternatives(uid):
# If pipelineFlags exists then we have started alternatives, and so have
# already scheduled the query. No need to reschedule unless the query fails.
# TODO: If the query fails, then remove the pipelineFlags so that we will
# reschedule.
query = {'user_id':uid, 'type':'move', 'pipelineFlags': {'$exists': False}}
return get_trip_db().find(query)
def testPlaceLoad(self):
result = self.loadTestJSON("emission/tests/data/test20140410")
collect.processResult(self.testUUID, result)
# Check that the trips are loaded correctly
TripColl = get_trip_db()
firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140409T191531-0700'}]})
logging.debug("selected trip = %s" % firstStoredTrip)
# For some reason, the place is always "unknown", at least for this set of test trips.
# Maybe it is related to the fact that they haven't been tagged in FourSquare
self.assertEqual(firstStoredTrip['type'], 'place')
self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700')
self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700")
self.assertIn('place_location', firstStoredTrip['place'])
self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point',
'coordinates': [-122.08632, 37.391]})
# Now, check that we have the sections as well. The previous trip did not
# have any sections. This one does
tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700'}]})
self.assertNotEqual(tripWithSections, None)
self.assertEqual(tripWithSections['sections'], [0])
SectionColl = get_section_db()
sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700',
'section_id': 0}]})
self.assertNotEqual(sectionForTrip, None)
def obtain_alternatives(trip_id, user_id):
db = edb.get_trip_db()
trip = ecwt.E_Mission_Trip.trip_from_json(db.find_one({"trip_id": trip_id, "user_id": user_id}))
logging.debug(trip.sections)
start_coord = trip.trip_start_location.maps_coordinate()
end_coord = trip.trip_end_location.maps_coordinate()
logging.debug("Start: %s " % start_coord)
logging.debug("End: %s " % end_coord)
curr_time = datetime.datetime.now()
curr_year = curr_time.year
curr_month = curr_time.month
curr_day = curr_time.day
curr_hour = curr_time.hour
curr_minute = curr_time.minute
otp_modes = ['CAR', 'WALK', 'BICYCLE', 'TRANSIT']
for mode in otp_modes:
try:
otp_trip = otp.OTP(start_coord, end_coord, mode, write_day(curr_month, curr_day, curr_year), write_time(curr_hour, curr_minute), False)
otp_trip = otp_trip.turn_into_trip(None, user_id, trip_id)
otp_trip.save_to_db()
except otp.PathNotFoundException as e:
#modes = ['driving', 'walking', 'bicycling', 'transit']
logging.debug("Got error %s from OTP, defaulting to Google Maps" % e)
otp_to_google_mode = {"CAR":"driving", "WALK":"walking", "BICYCLE":"bicycling", "TRANSIT":"transit"}
mode = otp_to_google_mode[mode]
gmaps = gmaps_lib.googlemaps.GoogleMaps('AIzaSyBEkw4PXVv_bsAdUmrFwatEyS6xLw3Bd9c')
result = gmaps.directions(origin=start_coord, destination=end_coord, mode=mode)
gmaps_trip = gmaps_lib.common.google_maps_to_our_trip(result, None, user_id, trip_id, mode, curr_time)
gmaps_trip.save_to_db()
'''
def save_trip_to_db(trip):
db = edb.get_trip_db()
print "start loc = %s" % trip.trip_start_location.coordinate_list()
print "end loc = %s" % trip.trip_end_location.coordinate_list()
db.insert({"_id": trip._id, "user_id": trip.user_id, "trip_id": trip.trip_id, "type" : "move", "sections": range(len(trip.sections)), "trip_start_datetime": trip.start_time,
"trip_end_datetime": trip.end_time, "trip_start_location": trip.trip_start_location.coordinate_list(),
"trip_end_location": trip.trip_end_location.coordinate_list(), "mode_list": trip.mode_list})
for section in trip.sections:
save_section_to_db(section)
def testCreation(self):
# This is mostly just a sanity check
db = get_trip_db()
db.remove()
create_fake_trips()
list_of_cluster_data = eamtcp.main()
tm = create_tour_model('test_user', list_of_cluster_data)
self.assertEquals(len(tm.get_top_trips(1)), 1)
tour = tm.build_tour_model()
self.assertEquals(len(tour), 7)
def save_to_db(self):
db = edb.get_trip_db()
result = db.update({"_id": self._id},
{"$set": {"mode" : self.mode_list, "confirmed_mode" : self.confirmed_mode_list}},
upsert=False,multi=False)
print result
if not result["updatedExisting"]:
self._create_new(db)
self._save_alternatives(self.alternatives)
self._save_perturbed(self.perturbed_trips)
def loadPipelineFlags(self, _id):
db = edb.get_trip_db()
json_object = db.find_one({'_id': _id})
if json_object:
tf = json_object.get('pipelineFlags')
if tf:
if tf['alternativesStarted'] == 'True':
self.alternativesStarted = True
if tf['alternativesFinished'] == 'True':
self.alternativesFinished = True
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
#self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
# "*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
self.testUsers = []
#for userEmail in self.testUserEmails:
# User.register(userEmail)
# self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = edb.get_mode_db()
self.ModesColl.remove()
edb.get_trip_db().remove()
edb.get_section_db().remove()
edb.get_alternatives_db().remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("emission/tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
# register each of the users and add sample trips to each user
result = self.loadTestJSON("emission/tests/data/missing_trip")
collect.processResult(self.testUUID, result)
for trip in edb.get_trip_db().find():
trip['trip_start_datetime'] = pydt.datetime.now() + pydt.timedelta(hours=-5)
trip['trip_end_datetime'] = pydt.datetime.now()
edb.get_trip_db().update({"_id": trip["_id"]}, trip)
for section in edb.get_section_db().find():
section['section_start_datetime'] = pydt.datetime.now() + pydt.timedelta(hours=-5)
section['section_end_datetime'] = pydt.datetime.now()
edb.get_section_db().update({"_id": section["_id"]}, section)
self.pipeline = AlternativeTripsPipeline()
def save_trip_to_db(trip):
print("saving trip to db")
print(trip.user_id)
db = edb.get_trip_db()
print("RHRHRH")
print("start loc = %s" % trip.trip_start_location.coordinate_list())
print("end loc = %s" % trip.trip_end_location.coordinate_list())
db.insert({"_id": trip._id, "user_id": trip.user_id, "trip_id": trip.trip_id, "type" : "move", "sections": list(range(len(trip.sections))), "trip_start_datetime": trip.start_time.datetime,
"trip_end_datetime": trip.end_time.datetime, "trip_start_location": trip.trip_start_location.coordinate_list(),
"trip_end_location": trip.trip_end_location.coordinate_list(), "mode_list": trip.mode_list})
print("len(trip.sections) in trip gen is %s" % len(trip.sections))
for section in trip.sections:
save_section_to_db(section)
def testUpdateSectionForExistingTrip(self):
result = self.loadTestJSON("emission/tests/data/missing_trip")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
storedSections = SectionColl.find({'user_id': self.testUUID})
self.assertEquals(storedSections.count(), 21)
# Trip 20140407T175709-0700 has two sections
storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTripSections.count(), 2)
TripColl = get_trip_db()
storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTrips.count(), 1)
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
selTripFromMoves = None
for i, seg in enumerate(result[0]['segments']):
if seg['startTime'] == '20140407T175709-0700':
selTripFromMoves = seg
copiedTripSections = []
for i, act in enumerate(selTripFromMoves['activities']):
act['startTime'] = '20140407T18%s039-0700' % (i + 2)
copiedTripSections.append(act)
self.assertEquals(len(copiedTripSections), 2)
[selTripFromMoves['activities'].append(act) for act in copiedTripSections]
self.assertEquals(len(selTripFromMoves['activities']), 4)
collect.processResult(self.testUUID, result)
storedTripSections = SectionColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTripSections.count(), 4)
storedTrips = TripColl.find({'$and': [{'user_id': self.testUUID},
{'trip_id': '20140407T175709-0700'}]})
self.assertEquals(storedTrips.count(), 1)
# This is actually a bug in the existing code. Need to fix it.
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
def mark_recommended(self, alternative):
db = edb.get_trip_db()
'''
point_list = []
for section in self.sections:
point_list.append([{'coordinates':[point.lon, point.lat]}
for point in section.points])
'''
alternative_json = {"user_id": alternative.user_id, "trip_id": alternative.trip_id,
"trip_start_time": alternative.start_time.strftime(DATE_FORMAT),
"trip_end_time": alternative.end_time.strftime(DATE_FORMAT),
"trip_start_location": alternative.trip_start_location.coordinate_list(),
"trip_end_location": alternative.trip_end_location.coordinate_list(),
"mode_list": alternative.mode_list,
"track_points": alternative.track_points}
print "recommending"
result = db.update({"trip_id": self.trip_id, "user_id": self.user_id},
{"$set": {"recommended_alternative" : alternative_json}},
upsert=False,multi=False)
def set_up(_id):
#fake_alternative = open("testTrip.txt", "r")
g = sdk.GoogleMaps('AIzaSyBEkw4PXVv_bsAdUmrFwatEyS6xLw3Bd9c')
#fake_alternative = json.loads(fake_alternative.read())
#t1 = google_maps_to_our_trip(g.directions('2703 Hallmark Dr Belmont CA', '2341 Ellsworth Berkeley CA', 0, 0, 0, 0)
#t2 = google_maps_to_our_trip(g.directions('1114 Madera way Belmont CA', '2510 Bancroft st Berkeley CA', 0, 0, 0, 0)
db = edb.get_trip_db()
our_trip = et.E_Mission_Trip.trip_from_json(db.find_one({'trip_id':_id}))
print "our_trip duration: " , our_trip.get_duration()
pt = ep.find_perturbed_trips(our_trip)
new_trip = pt[0]
#print our_trip.trip_start_location
#print our_trip.trip_end_location
modes = ['driving', 'walking', 'bicycling', 'transit']
for mode in modes:
thing = g.directions(our_trip.trip_start_location.maps_coordinate(), our_trip.trip_end_location.maps_coordinate(), mode=mode)
alt = ec.google_maps_to_our_trip(thing, random.randint(1,10), 0, _id, 0, our_trip.start_time)
print type(alt)
alt.save_to_db()
print "alternative duration: " , alt.get_duration()
def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
original_trip = get_trip_db().find_one({
'user_id': user_uuid,
'recommended_alternative': {
'$exists': True
}
})
if original_trip is None:
return template("clients/recommendation/no_recommendation.html")
del original_trip['trip_start_datetime']
del original_trip['trip_end_datetime']
del original_trip['user_id']
del original_trip['pipelineFlags']
del original_trip['recommended_alternative']['user_id']
recommended_trip = original_trip['recommended_alternative']
original_sections = list(get_section_db().find(
{'trip_id': original_trip['trip_id']}))
for section in original_sections:
del section['user_id']
del section['section_start_datetime']
del section['section_end_datetime']
if 'retained' in section:
del section['retained']
del section['manual']
del section['commute']
logging.debug("original sections = %s" % original_sections)
logging.debug("recommended trip = %s" % recommended_trip)
renderedTemplate = template("clients/recommendation/result_template.html",
originalSections=json.dumps(original_sections),
recommendedTrip=json.dumps(recommended_trip))
return renderedTemplate
def testStoreSensedTrips(self):
fakeEmail = "*****@*****.**"
fakeUUID = User.fromEmail(fakeEmail).uuid
trip_array = json.load(open("emission/tests/data/sensed_trips.json"))
self.assertEqual(len(trip_array), 2)
tripManager.storeSensedTrips(fakeUUID, trip_array)
insertedTrips = [trip for trip in get_trip_db().find({"user_id": fakeUUID})]
# We load two sections for each user in the setup. Here we only want to
# look at sections that we added here. We distinguish between the two by looking
# to see whether the predicted mode exists
insertedSections = [section for section in get_section_db().find({"$and":
[{"user_id": fakeUUID}, {"predicted_mode": {"$exists": False}}]})]
# insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})]
self.assertEqual(len(insertedTrips), 2)
self.assertEqual(insertedTrips[0]["type"], "place")
self.assertEqual(insertedTrips[0]["trip_start_time"], "20150101T000153-0500")
# self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52))
self.assertEqual(insertedTrips[0]["trip_end_time"], "20150102T000252-0500")
# self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51))
startPlaceLocation = insertedTrips[0]["place"]["place_location"]
self.assertEqual(startPlaceLocation["coordinates"], [-122.086945, 37.380866])
self.assertEqual(insertedTrips[1]["type"], "move")
self.assertEqual(insertedTrips[1]["trip_start_time"], "20150102T000252-0500")
self.assertEqual(insertedTrips[1]["trip_end_time"], "20150102T000252-0500")
self.assertEqual(len(insertedSections), 2)
walkingSection = insertedSections[0]
walkingTrackPointArray = insertedSections[0]["track_points"]
self.assertEqual(walkingSection["section_start_time"], "20150102T000252-0500")
self.assertEqual(walkingSection["section_end_time"], "20150102T000253-0500")
self.assertEqual(walkingSection["duration"], 180631)
self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2)
self.assertEqual(len(walkingTrackPointArray), 7)
self.assertEqual(walkingTrackPointArray[0]["track_location"]["coordinates"], [-122.086945, 37.380866])
def save_trip_to_db(trip):
print "saving trip to db"
print trip.user_id
db = edb.get_trip_db()
print "RHRHRH"
print "start loc = %s" % trip.trip_start_location.coordinate_list()
print "end loc = %s" % trip.trip_end_location.coordinate_list()
db.insert({
"_id": trip._id,
"user_id": trip.user_id,
"trip_id": trip.trip_id,
"type": "move",
"sections": range(len(trip.sections)),
"trip_start_datetime": trip.start_time,
"trip_end_datetime": trip.end_time,
"trip_start_location": trip.trip_start_location.coordinate_list(),
"trip_end_location": trip.trip_end_location.coordinate_list(),
"mode_list": trip.mode_list
})
print "len(trip.sections) in trip gen is %s" % len(trip.sections)
for section in trip.sections:
save_section_to_db(section)
def read_data(uuid=None,size=None):
data = []
db = edb.get_trip_db()
if uuid:
trips = db.find({'user_id' : uuid, 'type' : 'move'})
else:
trips = db.find({'type' : 'move'})
if trips.count() == 0:
return []
for t in trips:
try:
trip = Trip.trip_from_json(t)
except:
continue
if not (trip.trip_start_location and trip.trip_end_location and trip.start_time):
continue
data.append(trip)
if size:
if len(data) == size:
break
if len(data) == 0:
return []
return data
def read_data(uuid=None, size=None, old=True):
data = []
trip_db = edb.get_trip_db()
if not old:
trip_db = edb.get_trip_new_db()
trips = trip_db.find({"user_id" : uuid})
else:
if uuid:
trips = trip_db.find({'user_id' : uuid, 'type' : 'move'})
else:
trips = trip_db.find({'type' : 'move'})
for t in trips:
try:
trip = Trip.trip_from_json(t)
except:
continue
if not (trip.trip_start_location and trip.trip_end_location and trip.start_time):
continue
data.append(trip)
if size:
if len(data) == size:
break
return data
return [ecwt.Trip(trip) for trip in trips]
def get_training_uuid_list():
return edb.get_trip_db().find({}).distinct("user_id")
def get_recommender_uuid_list():
return edb.get_trip_db().find({}).distinct("user_id")
def processTripArray(user_uuid, trip_array):
# First, we open a connection to the database
Stage_Trips = get_trip_db()
Stage_Sections = get_section_db()
Modes = get_mode_db()
number_of_trips = len(trip_array)
logging.info("number of trips = %s" % number_of_trips)
for trip in range(number_of_trips):
seg_note = trip_array[trip]
trip_id = seg_note["startTime"]
_id_trip = str(user_uuid) + '_' + seg_note["startTime"]
#logging.debug(json.dumps(seg_note))
if "activities" in seg_note:
number_of_sections = len(seg_note["activities"])
logging.debug("number of sections = %s" % number_of_sections)
for sectionindex in range(number_of_sections):
seg_act_note = seg_note["activities"][sectionindex]
# if the section is missing some data that we access later, then we skip it
if Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}, {
"section_id": sectionindex
}]
}).count() == 0:
try:
_id_section = str(user_uuid) + '_' + seg_act_note[
"startTime"] + '_' + str(sectionindex)
_mode = convertModeNameToIndex(
Modes, seg_act_note["activity"])
isManual = seg_act_note[
"manual"] if "manual" in seg_act_note else False
sections_todo = {
'source': 'Shankari',
'_id': _id_section,
'user_id': user_uuid,
'trip_id': trip_id,
'type': seg_note["type"],
'section_id': sectionindex,
'mode': _mode,
# SHANKARI: what does seg_act_note["manual"] mean?
'confirmed_mode': _mode if isManual else '',
# 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})])))
# if "group" in seg_act_note else '',
}
fillSectionWithMovesData(seg_act_note, sections_todo)
label_filtered_section(sections_todo)
# Now that we have created this section, let's insert it into the database
try:
logging.info(
"About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s "
% (trip_id, sectionindex,
sections_todo['section_start_time'],
seg_note["type"], seg_act_note["activity"]))
Stage_Sections.insert(sections_todo)
except DuplicateKeyError:
logging.warning(
"DuplicateKeyError, skipping insert %s" %
sections_todo)
logging.warning(
"Existing section is %s" %
Stage_Sections.find_one({"_id": _id_section}))
except KeyError, e:
logging.warning(
"Missing key %s, skipping section insert %s" %
(e, seg_act_note))
insertedSectionCount = Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}, {
"section_id": sectionindex
}]
}).count()
if insertedSectionCount == 0:
logging.error(
"Insert appears to have FAILED. No entry for %s, %s, %s found"
% (user_uuid, trip_id, sectionindex))
else:
logging.debug(
"Found existing matching entry for %s, %s, %s, skipping entry"
% (user_uuid, trip_id, sectionindex))
# Insert a trip if it doesn't already exist
# SHANKARI: What if we get other sections for a trip later? When do we update the trip?
# Do we even need to keep this linkage, with the concomittant
# management cost if we can just find all sections by trip_id
# instead? How expensive is the query?
if Stage_Trips.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}]
}).count() == 0:
trips_todo = {
'source':
'Shankari',
'_id':
_id_trip,
'user_id':
user_uuid,
'trip_id':
trip_id,
'sections': [
sections['section_id']
for sections in Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}]
})
]
}
fillTripWithMovesData(seg_note, trips_todo)
logging.info("About to insert trip with trip_id = %s " % (trip_id))
Stage_Trips.insert(trips_todo)
else:
logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def getAllTrips_Date(uid, dys):
#trips = list(get_trip_db().find({"user_id":uid, "type":"move"}))
d = datetime.datetime.now() - datetime.timedelta(days=dys)
query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}}
return get_trip_db().find(query)
def getAllTrips(uid):
#trips = list(get_trip_db().find({"user_id":uid, "type":"move"}))
query = {'user_id':uid, 'type':'move'}
return get_trip_db().find(query)
def savePipelineFlags(self):
db = edb.get_trip_db()
db.update({"_id": self._id},
{"$set": {"pipelineFlags" : {'alternativesStarted': self.alternativesStarted, 'alternativesFinished': self.alternativesFinished}}},
multi=False, upsert=False)
def get_recommender_uuid_list():
return edb.get_trip_db().find({}).distinct("user_id")
def getNoAlternativesPastMonth(uid):
d = datetime.datetime.now() - datetime.timedelta(days=30)
query = {'user_id':uid, 'type':'move',
'trip_start_datetime':{"$gt":d},
'pipelineFlags': {'$exists': False}}
return get_trip_db().find(query)
def getNoAlternativesPastMonth(uid):
d = datetime.datetime.now() - datetime.timedelta(days=30)
query = {'user_id':uid, 'type':'move',
'trip_start_datetime':{"$gt":d},
'pipelineFlags': {'$exists': False}}
return get_trip_db().find(query)
def testUpdateSectionForExistingTrip(self):
result = self.loadTestJSON("emission/tests/data/missing_trip")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
storedSections = SectionColl.find({'user_id': self.testUUID})
self.assertEquals(storedSections.count(), 21)
# Trip 20140407T175709-0700 has two sections
storedTripSections = SectionColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTripSections.count(), 2)
TripColl = get_trip_db()
storedTrips = TripColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTrips.count(), 1)
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
selTripFromMoves = None
for i, seg in enumerate(result[0]['segments']):
if seg['startTime'] == '20140407T175709-0700':
selTripFromMoves = seg
copiedTripSections = []
for i, act in enumerate(selTripFromMoves['activities']):
act['startTime'] = '20140407T18%s039-0700' % (i + 2)
copiedTripSections.append(act)
self.assertEquals(len(copiedTripSections), 2)
[
selTripFromMoves['activities'].append(act)
for act in copiedTripSections
]
self.assertEquals(len(selTripFromMoves['activities']), 4)
collect.processResult(self.testUUID, result)
storedTripSections = SectionColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTripSections.count(), 4)
storedTrips = TripColl.find({
'$and': [{
'user_id': self.testUUID
}, {
'trip_id': '20140407T175709-0700'
}]
})
self.assertEquals(storedTrips.count(), 1)
# This is actually a bug in the existing code. Need to fix it.
for trip in storedTrips:
self.assertEquals(len(trip['sections']), 2)
def testStoreSensedTrips(self):
fakeEmail = "*****@*****.**"
fakeUUID = User.fromEmail(fakeEmail).uuid
trip_array = json.load(open("emission/tests/data/sensed_trips.json"))
self.assertEqual(len(trip_array), 2)
tripManager.storeSensedTrips(fakeUUID, trip_array)
insertedTrips = [
trip for trip in get_trip_db().find({"user_id": fakeUUID})
]
# We load two sections for each user in the setup. Here we only want to
# look at sections that we added here. We distinguish between the two by looking
# to see whether the predicted mode exists
insertedSections = [
section for section in get_section_db().find({
"$and": [{
"user_id": fakeUUID
}, {
"predicted_mode": {
"$exists": False
}
}]
})
]
# insertedSections = [section["predicted_mode"] for section in get_section_db().find({"user_id": fakeUUID})]
self.assertEqual(len(insertedTrips), 2)
self.assertEqual(insertedTrips[0]["type"], "place")
self.assertEqual(insertedTrips[0]["trip_start_time"],
"20150101T000153-0500")
# self.assertEqual(insertedTrips[0]["trip_start_datetime"], datetime(2014,12,31,17,31,52))
self.assertEqual(insertedTrips[0]["trip_end_time"],
"20150102T000252-0500")
# self.assertEqual(insertedTrips[0]["trip_end_datetime"], datetime(2015,01,02,04,01,51))
startPlaceLocation = insertedTrips[0]["place"]["place_location"]
self.assertEqual(startPlaceLocation["coordinates"],
[-122.086945, 37.380866])
self.assertEqual(insertedTrips[1]["type"], "move")
self.assertEqual(insertedTrips[1]["trip_start_time"],
"20150102T000252-0500")
self.assertEqual(insertedTrips[1]["trip_end_time"],
"20150102T000252-0500")
self.assertEqual(len(insertedSections), 2)
walkingSection = insertedSections[0]
walkingTrackPointArray = insertedSections[0]["track_points"]
self.assertEqual(walkingSection["section_start_time"],
"20150102T000252-0500")
self.assertEqual(walkingSection["section_end_time"],
"20150102T000253-0500")
self.assertEqual(walkingSection["duration"], 180631)
self.assertAlmostEqual(walkingSection["distance"], 1311.125, places=2)
self.assertEqual(len(walkingTrackPointArray), 7)
self.assertEqual(
walkingTrackPointArray[0]["track_location"]["coordinates"],
[-122.086945, 37.380866])
def setUpClass(cls):
db = edb.get_trip_db()
def getTrainingTrips(uid):
return getTrainingTrips_Date(uid, 30)
query = {'user_id':uid, 'type':'move'}
return get_trip_db().find(query)
def getTrainingTrips(uid):
return getTrainingTrips_Date(uid, 30)
query = {'user_id':uid, 'type':'move'}
return get_trip_db().find(query)
def processTripArray(user_uuid, trip_array):
# First, we open a connection to the database
Stage_Trips=get_trip_db()
Stage_Sections=get_section_db()
Modes=get_mode_db()
number_of_trips=len(trip_array)
logging.info("number of trips = %s" % number_of_trips)
for trip in range(number_of_trips):
seg_note=trip_array[trip]
trip_id=seg_note["startTime"]
_id_trip=str(user_uuid)+'_'+seg_note["startTime"]
#logging.debug(json.dumps(seg_note))
if "activities" in seg_note:
number_of_sections=len(seg_note["activities"])
logging.debug("number of sections = %s" % number_of_sections)
for sectionindex in range(number_of_sections):
seg_act_note=seg_note["activities"][sectionindex]
# if the section is missing some data that we access later, then we skip it
if Stage_Sections.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id},{"section_id": sectionindex}]}).count()==0:
try:
_id_section = str(user_uuid)+'_'+seg_act_note["startTime"]+'_'+str(sectionindex)
_mode = convertModeNameToIndex(Modes, seg_act_note["activity"])
isManual = seg_act_note["manual"] if "manual" in seg_act_note else False
sections_todo={'source':'Shankari',
'_id':_id_section,
'user_id': user_uuid,
'trip_id':trip_id,
'type':seg_note["type"],
'section_id':sectionindex,
'mode' : _mode,
# SHANKARI: what does seg_act_note["manual"] mean?
'confirmed_mode' :_mode if isManual else '',
# 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})])))
# if "group" in seg_act_note else '',
}
fillSectionWithMovesData(seg_act_note, sections_todo)
label_filtered_section(sections_todo)
# Now that we have created this section, let's insert it into the database
try:
logging.info("About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " %
(trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"]))
Stage_Sections.insert(sections_todo)
except DuplicateKeyError:
logging.warning("DuplicateKeyError, skipping insert %s" % sections_todo)
logging.warning("Existing section is %s" % Stage_Sections.find_one({"_id": _id_section}))
except KeyError, e:
logging.warning("Missing key %s, skipping section insert %s" % (e, seg_act_note))
insertedSectionCount = Stage_Sections.find({"$and" : [{"user_id": user_uuid},
{"trip_id": trip_id},
{"section_id": sectionindex}]}).count()
if insertedSectionCount == 0:
logging.error("Insert appears to have FAILED. No entry for %s, %s, %s found" %
(user_uuid, trip_id, sectionindex))
else:
logging.debug("Found existing matching entry for %s, %s, %s, skipping entry" %
(user_uuid, trip_id, sectionindex))
# Insert a trip if it doesn't already exist
# SHANKARI: What if we get other sections for a trip later? When do we update the trip?
# Do we even need to keep this linkage, with the concomittant
# management cost if we can just find all sections by trip_id
# instead? How expensive is the query?
if Stage_Trips.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id}]}).count()==0:
trips_todo={ 'source':'Shankari',
'_id':_id_trip,
'user_id': user_uuid,
'trip_id':trip_id,
'sections':[sections['section_id'] for sections in Stage_Sections.find({"$and":[{"user_id":user_uuid}, {"trip_id":trip_id}]})]}
fillTripWithMovesData(seg_note, trips_todo)
logging.info("About to insert trip with trip_id = %s " % (trip_id))
Stage_Trips.insert(trips_todo)
else:
logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def getTrainingTrips_Date(uid, dys):
d = datetime.datetime.now() - datetime.timedelta(days=dys)
query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}, "pipelineFlags":{"$exists":True}}
#query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}}
#print get_trip_db().find(query).count()
return get_trip_db().find(query)
def get_training_uuid_list():
return edb.get_trip_db().find({}).distinct("user_id")
def getAllTrips(uid):
#trips = list(get_trip_db().find({"user_id":uid, "type":"move"}))
query = {'user_id':uid, 'type':'move'}
return get_trip_db().find(query)
def getAllTrips_Date(uid, dys):
#trips = list(get_trip_db().find({"user_id":uid, "type":"move"}))
d = datetime.datetime.now() - datetime.timedelta(days=dys)
query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}}
return get_trip_db().find(query)
def getTrainingTrips_Date(uid, dys):
d = datetime.datetime.now() - datetime.timedelta(days=dys)
query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}, "pipelineFlags":{"$exists":True}}
#query = {'user_id':uid, 'type':'move','trip_start_datetime':{"$gt":d}}
#print get_trip_db().find(query).count()
return get_trip_db().find(query)