def createDummySection(startTime,
endTime,
startLoc,
endLoc,
predictedMode=None,
confirmedMode=None):
section = {
'source': 'Shankari',
'section_start_datetime': startTime,
'section_end_datetime': endTime,
'section_start_time': startTime.isoformat(),
'section_end_time': endTime.isoformat(),
'section_start_point': {
'type': 'Point',
'coordinates': startLoc
},
'section_end_point': {
'type': 'Point',
'coordinates': endLoc
},
}
if predictedMode != None:
section['predicted_mode'] = predictedMode
if confirmedMode != None:
section['confirmed_mode'] = confirmedMode
get_section_db().insert(section)
return section
def testClientSpecificSettersWithOverride(self):
fakeEmail = "*****@*****.**"
client = Client("testclient")
client.update(createKey = False)
common.makeValid(client)
(resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail)
studyList = Client.getPendingClientRegs(fakeEmail)
self.assertEqual(studyList, ["testclient"])
user = User.register("*****@*****.**")
self.assertEqual(user.getFirstStudy(), 'testclient')
dummyPredModeMap = {'walking': 1.0}
dummySection = common.createDummySection(startTime = datetime.now() - timedelta(seconds = 60 * 60),
endTime = datetime.now(),
startLoc = [-122, 34],
endLoc = [-122, 35],
predictedMode = dummyPredModeMap)
clientSetQuery = client.clientSpecificSetters(user.uuid, dummySection['_id'], dummyPredModeMap)
self.assertEqual(clientSetQuery, {'$set': {'test_auto_confirmed': {'mode': 1, 'prob': 1.0}}})
# Apply the change
get_section_db().update({'_id': dummySection['_id']}, clientSetQuery)
retrievedSection = get_section_db().find_one({'_id': dummySection['_id']})
self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
def testClientSpecificSettersWithOverride(self):
fakeEmail = "*****@*****.**"
client = Client("testclient")
client.update(createKey = False)
common.makeValid(client)
(resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail)
studyList = Client.getPendingClientRegs(fakeEmail)
self.assertEqual(studyList, ["testclient"])
user = User.register("*****@*****.**")
self.assertEqual(user.getFirstStudy(), 'testclient')
dummyPredModeMap = {'walking': 1.0}
dummySection = common.createDummySection(startTime = datetime.now() - timedelta(seconds = 60 * 60),
endTime = datetime.now(),
startLoc = [-122, 34],
endLoc = [-122, 35],
predictedMode = dummyPredModeMap)
clientSetQuery = client.clientSpecificSetters(user.uuid, dummySection['_id'], dummyPredModeMap)
self.assertEqual(clientSetQuery, {'$set': {'test_auto_confirmed': {'mode': 1, 'prob': 1.0}}})
# Apply the change
get_section_db().update({'_id': dummySection['_id']}, clientSetQuery)
retrievedSection = get_section_db().find_one({'_id': dummySection['_id']})
self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
def testConfirmationModeQueryNeither(self):
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(1))
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(4))
self.assertEqual(retrieveByQuery.count(), 0)
def get_trip_before(section_id):
""" Return the trip just before the one that this section belongs to.
"""
section = rt.Section.section_from_json(get_section_db().find_one(
{'_id': section_id}))
logging.debug("Found section %s" % section)
firstSection = rt.Section.section_from_json(get_section_db().find_one({
"trip_id":
section.trip_id,
"section_id":
0
}))
logging.debug("First section %s" % firstSection)
# First, try to find the seection assuming that data collection was continuous
prevPlace = rt.Section.section_from_json(get_section_db().find_one(
{"section_end_datetime": firstSection.start_time}))
logging.debug("prevPlace %s" % prevPlace)
# This should be the "place" trip
if prevPlace is not None:
logging.debug("prevPlace.section_type = %s" % prevPlace.section_type)
if prevPlace.section_type != "place":
return None
else:
prevTrip = get_section_db().find_one(
{"section_end_datetime": prevPlace.start_time})
return prevTrip
else:
assert (False)
return allSections
def testConfirmationModeQueryCorrectedManualAndAuto(self):
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
clientSetQuery = Client(user.getFirstStudy()).clientSpecificSetters(user.uuid, dummySection, dummyPredModeMap)
# Apply the change
get_database.get_section_db().update({'_id': dummySection['_id']}, clientSetQuery)
retrievedSection = get_database.get_section_db().find_one({'_id': dummySection['_id']})
self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': {'confirmed_mode': 4}})
get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': {'corrected_mode': 9}})
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(1))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(4))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(9))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 1)
def get_all_sections(section_id):
""" Return all sections in the trip that the specified section is a part of
For example, if this is the section to go to the train station, return all
sections for the same trip.
The input is the _id field of the section
"""
section = rt.Section.section_from_json(get_section_db().find_one({'_id': section_id}))
allSections = get_section_db().find({"trip_id": section.trip_id})
return list(allSections)
def testConfirmationModeQueryNeither(self):
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
retrieveByQuery = get_database.get_section_db().find(
common.getConfirmationModeQuery(1))
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_database.get_section_db().find(
common.getConfirmationModeQuery(4))
self.assertEqual(retrieveByQuery.count(), 0)
def testConfirmationModeQueryManualNotAuto(self):
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': {'confirmed_mode': 4}})
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(1))
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(4))
self.assertEqual(retrieveByQuery.count(), 1)
def get_all_sections(section_id):
""" Return all sections in the trip that the specified section is a part of
For example, if this is the section to go to the train station, return all
sections for the same trip.
The input is the _id field of the section
"""
section = rt.Section.section_from_json(get_section_db().find_one(
{'_id': section_id}))
allSections = get_section_db().find({"trip_id": section.trip_id})
return list(allSections)
def tearDown(self):
get_section_db().remove({"_id": "foo_1"})
get_section_db().remove({"_id": "foo_2"})
get_section_db().remove({"_id": "foo_3"})
self.assertEqual(get_section_db().find({'_id': 'foo_1'}).count(), 0)
self.assertEqual(get_section_db().find({'_id': 'foo_2'}).count(), 0)
self.assertEqual(get_section_db().find({'_id': 'foo_3'}).count(), 0)
def tearDown(self):
get_section_db().delete_one({"_id": "foo_1"})
get_section_db().delete_one({"_id": "foo_2"})
get_section_db().delete_one({"_id": "foo_3"})
self.assertEqual(get_section_db().count_documents({'_id': 'foo_1'}), 0)
self.assertEqual(get_section_db().count_documents({'_id': 'foo_2'}), 0)
self.assertEqual(get_section_db().count_documents({'_id': 'foo_3'}), 0)
def tearDown(self):
get_section_db().remove({"_id": "foo_1"})
get_section_db().remove({"_id": "foo_2"})
get_section_db().remove({"_id": "foo_3"})
self.assertEqual(get_section_db().find({'_id': 'foo_1'}).count(), 0)
self.assertEqual(get_section_db().find({'_id': 'foo_2'}).count(), 0)
self.assertEqual(get_section_db().find({'_id': 'foo_3'}).count(), 0)
def setUp(self):
self.clearRelatedDb()
edb.get_trip_db().remove()
edb.get_section_db().remove()
edb.get_trip_new_db().remove()
edb.get_section_new_db().remove()
etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-21")
eaicf.filter_accuracy(self.testUUID)
estfm.move_all_filters_to_data()
eaist.segment_current_trips(self.testUUID)
eaiss.segment_current_sections(self.testUUID)
def testConfirmationModeQueryManualNotAuto(self):
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
get_database.get_section_db().update({'_id': dummySection['_id']},
{'$set': {
'confirmed_mode': 4
}})
retrieveByQuery = get_database.get_section_db().find(
common.getConfirmationModeQuery(1))
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_database.get_section_db().find(
common.getConfirmationModeQuery(4))
self.assertEqual(retrieveByQuery.count(), 1)
def update_profiles(dummy_users=False):
if dummy_users:
user_list = ['1']
else:
user_list = get_section_db().distinct('user_id')
for user in user_list:
generate_route_clusters(user)
def testRunBackgroundTasksForDay(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
emission.tests.common.loadTable(self.serverName, "Stage_Modes",
"emission/tests/data/modes.json")
emission.tests.common.loadTable(self.serverName, "Stage_Sections",
"emission/tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.SectionsColl = get_section_db()
emission.tests.common.updateSections(self)
self.assertNotEqual(len(self.uuid_list), 0)
# Can access the zeroth element because we know that then length is greater than zero
# (see above)
test_uuid = self.uuid_list[0]
test_user = User.fromUUID(test_uuid)
self.assertNotIn('carbon_footprint', test_user.getProfile().keys())
data.runBackgroundTasks(test_user.uuid)
self.assertIn('carbon_footprint', test_user.getProfile().keys())
def setUp(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = edb.url
self.ModesColl = get_mode_db()
self.SectionsColl = get_section_db()
# Let's make sure that the users are registered so that they have profiles
user_objects = []
for userEmail in self.testUsers:
user_objects.append(User.register(userEmail))
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
for testUser in user_objects:
etc.purgeSectionData(self.SectionsColl, testUser.uuid)
if self.ModesColl.estimated_document_count() > 0:
self.ModesColl.delete_many({})
self.assertEqual(self.ModesColl.estimated_document_count(), 0)
self.assertEqual(self.SectionsColl.estimated_document_count(), 0)
MongoClient(edb.url).drop_database("Backup_database")
etc.loadTable(self.serverName, "Stage_Modes",
"emission/tests/data/modes.json")
etc.loadTable(self.serverName, "Stage_Sections",
"emission/tests/data/testModeInferSeedFile")
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours=1)
if (section['confirmed_mode'] == 5):
# We only cluster bus and train trips
# And our test data only has bus trips
section['section_start_point'] = {
u'type': u'Point',
u'coordinates': [-122.270039042, 37.8800285728]
}
section['section_end_point'] = {
u'type': u'Point',
u'coordinates': [-122.2690412952, 37.8739578595]
}
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
# Replace the user email with the UUID
section['user_id'] = User.fromEmail(section['user_id']).uuid
edb.save(self.SectionsColl, section)
self.pipeline = pipeline.ModeInferencePipelineMovesFormat()
self.testLoadTrainingData()
def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
emission.tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
emission.tests.common.updateSections(self)
def __init__(self):
self.featureLabels = [
"distance",
"duration",
"first filter mode",
"sectionId",
"avg speed",
"speed EV",
"speed variance",
"max speed",
"max accel",
"isCommute",
"heading change rate",
"stop rate",
"velocity change rate",
"start lat",
"start lng",
"stop lat",
"stop lng",
"start hour",
"end hour",
"close to bus stop",
"close to train stop",
"close to airport",
]
self.Sections = edb.get_section_db()
def get_section(section_id):
section_json = edb.get_section_db().find_one({"id": section_id})
if section_json is None:
logging.warning("Did not find match for section %s, returning None" %
section_id)
return None
return ad.AttrDict(section_json)
def label_filtered_section(section):
minimum_travel_time=120
minimum_travel_distance=200
Modes=get_mode_db()
Sections=get_section_db()
is_retained = False
# logging.debug("Appending %s" % json.dumps(section))
if section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])>=2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \
max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
elif section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])<2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
elif (section['section_start_time']==''or section['section_end_time']=='') and len(section['track_points'])>=2:
if max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
else:
section['type'] ='not complete information'
section['retained'] = is_retained
def save_section_to_db(section):
print "saving section to db"
db = edb.get_section_db()
db.insert({"user_id" : section.user_id, "trip_id" : section.trip_id, "distance" : section.distance, "type" : section.section_type,
"section_start_datetime" : section.start_time.datetime, "section_end_datetime" : section.end_time.datetime,
"section_start_point" : {"coordinates" : section.section_start_location.coordinate_list()},
"section_end_point" : {"coordinates" : section.section_end_location.coordinate_list()}, "mode" : section.mode, "confirmed_mode" : section.confirmed_mode})
def testBlankToday(self):
result = self.loadTestJSON("emission/tests/data/test2_blank_today")
collect.processResult(self.testUUID, [result[0]])
collect.processResult(self.testUUID, [result[1]])
SectionColl = get_section_db()
self.assertTrue(SectionColl.find({'user_id': self.testUUID}).count() > 0)
def save_section_to_db(section):
print("saving section to db")
db = edb.get_section_db()
db.insert({"user_id" : section.user_id, "trip_id" : section.trip_id, "distance" : section.distance, "type" : section.section_type,
"section_start_datetime" : section.start_time.datetime, "section_end_datetime" : section.end_time.datetime,
"section_start_point" : {"coordinates" : section.section_start_location.coordinate_list()},
"section_end_point" : {"coordinates" : section.section_end_location.coordinate_list()}, "mode" : section.mode, "confirmed_mode" : section.confirmed_mode})
def getRoute(section_id):
route=[]
Sections=edb.get_section_db()
section=Sections.find_one({'_id':section_id})
for point in section['track_points']:
route.append(point['track_location']['coordinates'])
return route
def testPlaceLoad(self):
result = self.loadTestJSON("emission/tests/data/test20140410")
collect.processResult(self.testUUID, result)
# Check that the trips are loaded correctly
TripColl = get_trip_db()
firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140409T191531-0700'}]})
logging.debug("selected trip = %s" % firstStoredTrip)
# For some reason, the place is always "unknown", at least for this set of test trips.
# Maybe it is related to the fact that they haven't been tagged in FourSquare
self.assertEqual(firstStoredTrip['type'], 'place')
self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700')
self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700")
self.assertIn('place_location', firstStoredTrip['place'])
self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point',
'coordinates': [-122.08632, 37.391]})
# Now, check that we have the sections as well. The previous trip did not
# have any sections. This one does
tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700'}]})
self.assertNotEqual(tripWithSections, None)
self.assertEqual(tripWithSections['sections'], [0])
SectionColl = get_section_db()
sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700',
'section_id': 0}]})
self.assertNotEqual(sectionForTrip, None)
def _init_sections(cls, user_id, trip_id, num_sections):
sections = []
db = edb.get_section_db()
json_object = db.find({'user_id': user_id, 'trip_id' : trip_id}, limit = num_sections)
for section_json in json_object:
sections.append(Section.section_from_json(section_json))
return sections
def getRoute(section_id):
route = []
Sections = edb.get_section_db()
section = Sections.find_one({'_id': section_id})
for point in section['track_points']:
route.append(point['track_location']['coordinates'])
return route
def setUp(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
emission.tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
emission.tests.common.loadTable(self.serverName, "Stage_Modes",
"emission/tests/data/modes.json")
emission.tests.common.loadTable(self.serverName, "Stage_Sections",
"emission/tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
emission.tests.common.updateSections(self)
def get_mode_share_by_count(list_idx):
Sections=get_section_db()
## takes a list of idx's
AllModeList = getAllModes()
MODE = {}
MODE2= {}
for mode in AllModeList:
MODE[mode['mode_id']]=0
for _id in list_idx:
section=Sections.find_one({'_id': _id})
mode_id = section['confirmed_mode']
try:
MODE[mode_id] += 1
except KeyError:
MODE[mode_id] = 1
# print(sum(MODE.values()))
if sum(MODE.values())==0:
for mode in AllModeList:
MODE2[mode['mode_id']]=0
# print(MODE2)
else:
for mode in AllModeList:
MODE2[mode['mode_id']]=MODE[mode['mode_id']]/sum(MODE.values())
return MODE2
def get_mode_share_by_count(list_idx):
Sections = get_section_db()
BackupSections = safmt.AbstractCollection(edb.pm_address,
"Backup_database",
"Stage_Sections", None)
## takes a list of idx's
AllModeList = getAllModes()
MODE = {}
MODE2 = {}
for mode in AllModeList:
MODE[mode['mode_id']] = 0
for _id in list_idx:
section = Sections.find_one({'_id': _id})
if section is None:
section = BackupSections.find_one({'id': _id})
mode_id = section['confirmed_mode']
try:
MODE[mode_id] += 1
except KeyError:
MODE[mode_id] = 1
# print(sum(MODE.values()))
if sum(MODE.values()) == 0:
for mode in AllModeList:
MODE2[mode['mode_id']] = 0
# print(MODE2)
else:
for mode in AllModeList:
MODE2[mode['mode_id']] = MODE[mode['mode_id']] / sum(MODE.values())
return MODE2
def get_clusters_info(uid):
c_db = get_routeCluster_db()
s_db = get_section_db()
clusterJson = c_db.find_one({"clusters":{"$exists":True}, "user": uid})
if clusterJson is None:
return []
c_info = []
clusterSectionLists= clusterJson["clusters"].values()
logging.debug( "Number of section lists for user %s is %s" % (uid, len(clusterSectionLists)))
for sectionList in clusterSectionLists:
first = True
logging.debug( "Number of sections in sectionList for user %s is %s" % (uid, len(sectionList)))
if (len(sectionList) == 0):
# There's no point in returning this cluster, let's move on
continue
distributionArrays = [[] for _ in range(5)]
for section in sectionList:
section_json = s_db.find_one({"_id":section})
if first:
representative_trip = section_json
first = False
appendIfPresent(distributionArrays[0], section_json, "section_start_datetime")
appendIfPresent(distributionArrays[1], section_json, "section_end_datetime")
appendIfPresent(distributionArrays[2], section_json, "section_start_point")
appendIfPresent(distributionArrays[3], section_json, "section_end_point")
appendIfPresent(distributionArrays[4], section_json, "confirmed_mode")
c_info.append((distributionArrays, representative_trip))
return c_info
def detect_daily_work_office(user_id,day):
# say should be from 1 to 5
Sections=edb.get_section_db()
office_candidate=[]
home=eamh.detect_home(user_id)
if home == 'N/A':
return 'N/A'
# print(list_first_pnt)
for section in Sections.find({"$and":[{"user_id": user_id},{ "section_start_point": { "$ne": None }}]}):
section_start_pnt=section['section_start_point']
section_end_pnt=section['section_end_point']
if ec.Is_date(section['section_start_time'],day)==True:
# parameter that the distance away from home:
away_home=400
if not ec.Is_place(section_start_pnt,home,away_home) and not ec.Is_place(section_end_pnt,home,away_home):
office_candidate.append(section['track_points'][0])
office_candidate.append(section['track_points'][-1])
elif ec.Is_place(section_start_pnt,home,away_home) and not ec.Is_place(section_end_pnt,home,away_home):
office_candidate.append(section['track_points'][-1])
elif not ec.Is_place(section_start_pnt,home,away_home) and ec.Is_place(section_end_pnt,home,away_home):
office_candidate.append(section['track_points'][0])
if len(office_candidate)>0:
office_candidate = sorted(office_candidate, key=lambda k: parser.parse(k['time']))
# print(office_candidate)
weighted_office_candidate=ec.get_static_pnts(office_candidate)
office_location=ec.most_common(weighted_office_candidate,200)
# print(len(office_candidate))
# print(len(weighted_office_candidate))
# print(ec.calculate_appearance_rate(office_candidate,office_location))
# print(ec.calculate_appearance_rate(weighted_office_candidate,office_location))
return office_location
else:
return 'N/A'
def get_mode_share_by_count(list_idx):
Sections = get_section_db()
## takes a list of idx's
AllModeList = getAllModes()
MODE = {}
MODE2 = {}
for mode in AllModeList:
MODE[mode['mode_id']] = 0
for _id in list_idx:
section = Sections.find_one({'_id': _id})
mode_id = section['confirmed_mode']
try:
MODE[mode_id] += 1
except KeyError:
MODE[mode_id] = 1
# print(sum(MODE.values()))
if sum(MODE.values()) == 0:
for mode in AllModeList:
MODE2[mode['mode_id']] = 0
# print(MODE2)
else:
for mode in AllModeList:
MODE2[mode['mode_id']] = MODE[mode['mode_id']] / sum(MODE.values())
return MODE2
def get_clusters_info(uid):
c_db = get_routeCluster_db()
s_db = get_section_db()
clusterJson = c_db.find_one({"clusters":{"$exists":True}, "user": uid})
if clusterJson is None:
return []
c_info = []
clusterSectionLists= clusterJson["clusters"].values()
logging.debug( "Number of section lists for user %s is %s" % (uid, len(clusterSectionLists)))
for sectionList in clusterSectionLists:
first = True
logging.debug( "Number of sections in sectionList for user %s is %s" % (uid, len(sectionList)))
if (len(sectionList) == 0):
# There's no point in returning this cluster, let's move on
continue
distributionArrays = [[] for _ in range(5)]
for section in sectionList:
section_json = s_db.find_one({"_id":section})
if first:
representative_trip = section_json
first = False
appendIfPresent(distributionArrays[0], section_json, "section_start_datetime")
appendIfPresent(distributionArrays[1], section_json, "section_end_datetime")
appendIfPresent(distributionArrays[2], section_json, "section_start_point")
appendIfPresent(distributionArrays[3], section_json, "section_end_point")
appendIfPresent(distributionArrays[4], section_json, "confirmed_mode")
c_info.append((distributionArrays, representative_trip))
return c_info
def update_profiles(dummy_users=False):
if dummy_users:
user_list = ['1']
else:
user_list = get_section_db().distinct('user_id')
for user in user_list:
generate_route_clusters(user)
def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}})
if original_trip is None:
return template("clients/recommendation/no_recommendation.html")
del original_trip['trip_start_datetime']
del original_trip['trip_end_datetime']
del original_trip['user_id']
del original_trip['pipelineFlags']
del original_trip['recommended_alternative']['user_id']
recommended_trip = original_trip['recommended_alternative']
original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']}))
for section in original_sections:
del section['user_id']
del section['section_start_datetime']
del section['section_end_datetime']
if 'retained' in section:
del section['retained']
del section['manual']
del section['commute']
logging.debug("original sections = %s" % original_sections)
logging.debug("recommended trip = %s" % recommended_trip)
renderedTemplate = template("clients/recommendation/result_template.html",
originalSections = json.dumps(original_sections),
recommendedTrip = json.dumps(recommended_trip))
return renderedTemplate
def setUp(self):
import emission.tests.common
from copy import copy
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
emission.tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
self.setupUserAndClient()
emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.airCarbon = 217.0/1609
self.driveCarbon = 278.0/1609
self.busOptimalCarbon = 92.0/1609
self.allDriveExpect = (self.busExpect * self.driveCarbon + self.walkExpect * self.driveCarbon)/1000
self.myFootprintExpect = float(self.busExpect * self.busCarbon)/1000
self.sb375GoalExpect = 40.142892/7
self.mineMinusOptimalExpect = 0
self.allDriveMinusMineExpect = float(self.allDriveExpect - self.myFootprintExpect)/self.allDriveExpect
self.sb375DailyGoalMinusMineExpect = float(self.sb375GoalExpect - self.myFootprintExpect)/self.sb375GoalExpect
self.now = datetime.now()
self.twodaysago = self.now - timedelta(days=2)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.twodaysago
section['section_end_datetime'] = self.twodaysago + timedelta(hours = 1)
section['predicted_mode'] = {'walking': 1.0}
if section['user_id'] == '*****@*****.**':
logging.debug("Setting user_id for section %s, %s = %s" %
(section['trip_id'], section['section_id'], self.user.uuid))
section['user_id'] = self.user.uuid
if section['confirmed_mode'] == 5:
airSection = copy(section)
airSection['confirmed_mode'] = 9
airSection['_id'] = section['_id'] + "_air"
self.SectionsColl.insert(airSection)
airSection['confirmed_mode'] = ''
airSection['_id'] = section['_id'] + "_unconf"
self.SectionsColl.insert(airSection)
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
self.SectionsColl.save(section)
def testRetrieveTripsToImproveWithClusters(self):
sectionList = list(get_section_db().find())
get_routeCluster_db().insert({"user": self.testUUID,
"clusters":
{"cluster1": [s["_id"] for s in sectionList[0:10]],
"cluster2": [s["_id"] for s in sectionList[10:20]]}})
trip_list = list(self.pipeline.get_trips_to_improve(self.testUUID))
self.assertEquals(len(trip_list), 2)
def drawSectionsFromList(sectionIdList, outPath):
sectionJsonList = [get_section_db().find_one({"_id": sid}) for sid in sectionIdList]
bounds = tauq.get_bounds(sectionJsonList)
gmap = pygmaps.maps(old_div((bounds[0].lat + bounds[1].lat),2), old_div((bounds[0].lon + bounds[1].lon),2), 10)
drawSections(sectionJsonList, ALL, gmap, "random")
gmap.draw(outPath)
def display_trip(user, date, option):
user_id = UUID(user)
user_home = detect_home(user_id)
gmap = pygmaps.maps(user_home[1], user_home[0], 14)
start, end = Date(date)
for section in get_section_db().find({"$and":[{'user_id':user_id},{"section_start_datetime": {"$gte": start, "$lt": end}}]}):
drawSection(section, option, gmap)
gmap.draw('gmap_display/' + str(start)[:10] + '_' + user + '.html')
def get_user_sections(user_id):
sections = list(edb.get_section_db().find(
{'$and': [{
'user_id': user_id
}, {
'type': 'move'
}]}))
return sections
def __init__(self):
self.featureLabels = ["distance", "duration", "first filter mode", "sectionId", "avg speed",
"speed EV", "speed variance", "max speed", "max accel", "isCommute",
"heading change rate", "stop rate", "velocity change rate",
"start lat", "start lng", "stop lat", "stop lng",
"start hour", "end hour", "close to bus stop", "close to train stop",
"close to airport"]
self.Sections = edb.get_section_db()
def Berkeley_pop_route(start,end):
Sections = get_section_db()
list_of_point=[]
# print(berkeley_area())
for section in Sections.find({"$and":[{'In_UCB':True },{'type':'move'},{"section_start_datetime": {"$gte": start, "$lt": end}}]}):
for pnt in section['track_points']:
list_of_point.append(pnt['track_location']['coordinates'])
return {"latlng": list_of_point}
def createDummySection(startTime, endTime, startLoc, endLoc, predictedMode = None, confirmedMode = None):
section = {
'source': 'Shankari',
'section_start_datetime': startTime,
'section_end_datetime': endTime,
'section_start_time': startTime.isoformat(),
'section_end_time': endTime.isoformat(),
'section_start_point': {'type': 'Point', 'coordinates': startLoc},
'section_end_point': {'type': 'Point', 'coordinates': endLoc},
}
if predictedMode != None:
section['predicted_mode'] = predictedMode
if confirmedMode != None:
section['confirmed_mode'] = confirmedMode
get_section_db().insert(section)
return section
def testRetrieveTripsToImproveWithClusters(self):
sectionList = list(get_section_db().find())
get_routeCluster_db().insert({"user": self.testUUID,
"clusters":
{"cluster1": [s["_id"] for s in sectionList[0:10]],
"cluster2": [s["_id"] for s in sectionList[10:20]]}})
trip_list = list(self.pipeline.get_trips_to_improve(self.testUUID))
self.assertEquals(len(trip_list), 2)
def get_user_morning_commute_sections(user):
# say should be from 1 to 5
# get a list of all the sections for Mon, or ...
Sections = edb.get_section_db()
list_of_commute = []
for date in range(1, 6):
list_of_commute.extend(get_daily_morning_commute_sections(user, date))
return list_of_commute
def get_user_morning_commute_sections(user):
# say should be from 1 to 5
# get a list of all the sections for Mon, or ...
Sections = edb.get_section_db()
list_of_commute = []
for date in range(1, 6):
list_of_commute.extend(get_daily_morning_commute_sections(user, date))
return list_of_commute
def testConfirmationModeQueryAutoNoManual(self):
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
clientSetQuery = Client(user.getFirstStudy()).clientSpecificSetters(
user.uuid, dummySection, dummyPredModeMap)
# Apply the change
get_database.get_section_db().update({'_id': dummySection['_id']},
clientSetQuery)
retrievedSection = get_database.get_section_db().find_one(
{'_id': dummySection['_id']})
self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
retrieveByQuery = get_database.get_section_db().find(
common.getConfirmationModeQuery(1))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 1)
def display_home():
mymap_home = pygmaps.maps(37.8556475757, -122.248774009, 14)
for user in get_section_db().distinct('user_id'):
print(user)
user_home = detect_home(user)
print(user_home)
if user_home != 'N/A':
mymap_home.addpoint(user_home[1], user_home[0], "#FF0000")
mymap_home.draw('mymap_home.html')
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("emission/tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
get_section_db().remove({"user_id": self.testUUID})
result = self.loadTestJSON("emission/tests/data/missing_trip")
collect.processResult(self.testUUID, result)
print(get_section_db().find().count())
self.pipeline = RecommendationPipeline()
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("emission/tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
get_section_db().remove({"user_id": self.testUUID})
result = self.loadTestJSON("emission/tests/data/missing_trip")
collect.processResult(self.testUUID, result)
print get_section_db().find().count()
self.pipeline = RecommendationPipeline()
def get_berkeley_sections():
Sections = get_section_db()
list_of_sections = []
for section in Sections.find():
if section['section_start_point']!=None and section['section_end_point']!=None and \
Inside_polygon(section['section_start_point'],berkeley_area()) and \
Inside_polygon(section['section_end_point'],berkeley_area()):
list_of_sections.append(section)
return list_of_sections
def display_home():
mymap_home = pygmaps.maps(37.8556475757, -122.248774009,14)
for user in get_section_db().distinct('user_id'):
print(user)
user_home=detect_home(user)
print(user_home)
if user_home!='N/A':
mymap_home.addpoint(user_home[1], user_home[0], "#FF0000")
mymap_home.draw('mymap_home.html')
def get_berkeley_sections():
Sections = get_section_db()
list_of_sections=[]
for section in Sections.find():
if section['section_start_point']!=None and section['section_end_point']!=None and \
Inside_polygon(section['section_start_point'],berkeley_area()) and \
Inside_polygon(section['section_end_point'],berkeley_area()):
list_of_sections.append(section)
return list_of_sections
def testBlankToday(self):
result = self.loadTestJSON("emission/tests/data/test2_blank_today")
collect.processResult(self.testUUID, [result[0]])
collect.processResult(self.testUUID, [result[1]])
SectionColl = get_section_db()
self.assertTrue(
SectionColl.find({
'user_id': self.testUUID
}).count() > 0)
def testYesterdayForJustSignedUp(self):
result = self.loadTestJSON(
"emission/tests/data/yesterday_for_justsignedup")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
self.assertEquals(
SectionColl.find({
"user_id": self.testUUID
}).count(), 0)
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
#self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
# "*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
self.testUsers = []
#for userEmail in self.testUserEmails:
# User.register(userEmail)
# self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = edb.get_mode_db()
self.ModesColl.remove()
edb.get_trip_db().remove()
edb.get_section_db().remove()
edb.get_alternatives_db().remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("emission/tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
# register each of the users and add sample trips to each user
result = self.loadTestJSON("emission/tests/data/missing_trip")
collect.processResult(self.testUUID, result)
for trip in edb.get_trip_db().find():
trip['trip_start_datetime'] = pydt.datetime.now() + pydt.timedelta(
hours=-5)
trip['trip_end_datetime'] = pydt.datetime.now()
edb.get_trip_db().update({"_id": trip["_id"]}, trip)
for section in edb.get_section_db().find():
section['section_start_datetime'] = pydt.datetime.now(
) + pydt.timedelta(hours=-5)
section['section_end_datetime'] = pydt.datetime.now()
edb.get_section_db().update({"_id": section["_id"]}, section)
self.pipeline = AlternativeTripsPipeline()
def get_user_clusters(user, method, nClusters, is_ground_truth=False):
if is_ground_truth:
routes_user = user_route_data2(user)
else:
routes_user = user_route_data(user, edb.get_section_db())
if nClusters == -1:
nClusters = int(math.ceil(old_div(len(routes_user), 8)) + 1)
clusters_user = emkm.kmedoids(routes_user, nClusters, user, method=method)
#update_user_routeClusters(user,clusters_user[2],method=method)
return clusters_user
