def createDummySection(startTime, endTime, startLoc, endLoc, predictedMode=None, confirmedMode=None): section = { 'source': 'Shankari', 'section_start_datetime': startTime, 'section_end_datetime': endTime, 'section_start_time': startTime.isoformat(), 'section_end_time': endTime.isoformat(), 'section_start_point': { 'type': 'Point', 'coordinates': startLoc }, 'section_end_point': { 'type': 'Point', 'coordinates': endLoc }, } if predictedMode != None: section['predicted_mode'] = predictedMode if confirmedMode != None: section['confirmed_mode'] = confirmedMode get_section_db().insert(section) return section
def testClientSpecificSettersWithOverride(self): fakeEmail = "*****@*****.**" client = Client("testclient") client.update(createKey = False) common.makeValid(client) (resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail) studyList = Client.getPendingClientRegs(fakeEmail) self.assertEqual(studyList, ["testclient"]) user = User.register("*****@*****.**") self.assertEqual(user.getFirstStudy(), 'testclient') dummyPredModeMap = {'walking': 1.0} dummySection = common.createDummySection(startTime = datetime.now() - timedelta(seconds = 60 * 60), endTime = datetime.now(), startLoc = [-122, 34], endLoc = [-122, 35], predictedMode = dummyPredModeMap) clientSetQuery = client.clientSpecificSetters(user.uuid, dummySection['_id'], dummyPredModeMap) self.assertEqual(clientSetQuery, {'$set': {'test_auto_confirmed': {'mode': 1, 'prob': 1.0}}}) # Apply the change get_section_db().update({'_id': dummySection['_id']}, clientSetQuery) retrievedSection = get_section_db().find_one({'_id': dummySection['_id']}) self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
def testConfirmationModeQueryNeither(self): (user, dummySection, dummyPredModeMap) = self.setupClientTest() retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(1)) self.assertEqual(retrieveByQuery.count(), 0) retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(4)) self.assertEqual(retrieveByQuery.count(), 0)
def get_trip_before(section_id): """ Return the trip just before the one that this section belongs to. """ section = rt.Section.section_from_json(get_section_db().find_one( {'_id': section_id})) logging.debug("Found section %s" % section) firstSection = rt.Section.section_from_json(get_section_db().find_one({ "trip_id": section.trip_id, "section_id": 0 })) logging.debug("First section %s" % firstSection) # First, try to find the seection assuming that data collection was continuous prevPlace = rt.Section.section_from_json(get_section_db().find_one( {"section_end_datetime": firstSection.start_time})) logging.debug("prevPlace %s" % prevPlace) # This should be the "place" trip if prevPlace is not None: logging.debug("prevPlace.section_type = %s" % prevPlace.section_type) if prevPlace.section_type != "place": return None else: prevTrip = get_section_db().find_one( {"section_end_datetime": prevPlace.start_time}) return prevTrip else: assert (False) return allSections
def testConfirmationModeQueryCorrectedManualAndAuto(self): (user, dummySection, dummyPredModeMap) = self.setupClientTest() clientSetQuery = Client(user.getFirstStudy()).clientSpecificSetters(user.uuid, dummySection, dummyPredModeMap) # Apply the change get_database.get_section_db().update({'_id': dummySection['_id']}, clientSetQuery) retrievedSection = get_database.get_section_db().find_one({'_id': dummySection['_id']}) self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1) get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': {'confirmed_mode': 4}}) get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': {'corrected_mode': 9}}) retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(1)) for entry in retrieveByQuery: print entry self.assertEqual(retrieveByQuery.count(), 0) retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(4)) for entry in retrieveByQuery: print entry self.assertEqual(retrieveByQuery.count(), 0) retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(9)) for entry in retrieveByQuery: print entry self.assertEqual(retrieveByQuery.count(), 1)
def get_all_sections(section_id): """ Return all sections in the trip that the specified section is a part of For example, if this is the section to go to the train station, return all sections for the same trip. The input is the _id field of the section """ section = rt.Section.section_from_json(get_section_db().find_one({'_id': section_id})) allSections = get_section_db().find({"trip_id": section.trip_id}) return list(allSections)
def testConfirmationModeQueryNeither(self): (user, dummySection, dummyPredModeMap) = self.setupClientTest() retrieveByQuery = get_database.get_section_db().find( common.getConfirmationModeQuery(1)) self.assertEqual(retrieveByQuery.count(), 0) retrieveByQuery = get_database.get_section_db().find( common.getConfirmationModeQuery(4)) self.assertEqual(retrieveByQuery.count(), 0)
def testConfirmationModeQueryManualNotAuto(self): (user, dummySection, dummyPredModeMap) = self.setupClientTest() get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': {'confirmed_mode': 4}}) retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(1)) self.assertEqual(retrieveByQuery.count(), 0) retrieveByQuery = get_database.get_section_db().find(common.getConfirmationModeQuery(4)) self.assertEqual(retrieveByQuery.count(), 1)
def get_all_sections(section_id): """ Return all sections in the trip that the specified section is a part of For example, if this is the section to go to the train station, return all sections for the same trip. The input is the _id field of the section """ section = rt.Section.section_from_json(get_section_db().find_one( {'_id': section_id})) allSections = get_section_db().find({"trip_id": section.trip_id}) return list(allSections)
def tearDown(self): get_section_db().remove({"_id": "foo_1"}) get_section_db().remove({"_id": "foo_2"}) get_section_db().remove({"_id": "foo_3"}) self.assertEqual(get_section_db().find({'_id': 'foo_1'}).count(), 0) self.assertEqual(get_section_db().find({'_id': 'foo_2'}).count(), 0) self.assertEqual(get_section_db().find({'_id': 'foo_3'}).count(), 0)
def tearDown(self): get_section_db().delete_one({"_id": "foo_1"}) get_section_db().delete_one({"_id": "foo_2"}) get_section_db().delete_one({"_id": "foo_3"}) self.assertEqual(get_section_db().count_documents({'_id': 'foo_1'}), 0) self.assertEqual(get_section_db().count_documents({'_id': 'foo_2'}), 0) self.assertEqual(get_section_db().count_documents({'_id': 'foo_3'}), 0)
def setUp(self): self.clearRelatedDb() edb.get_trip_db().remove() edb.get_section_db().remove() edb.get_trip_new_db().remove() edb.get_section_new_db().remove() etc.setupRealExample(self, "emission/tests/data/real_examples/shankari_2015-aug-21") eaicf.filter_accuracy(self.testUUID) estfm.move_all_filters_to_data() eaist.segment_current_trips(self.testUUID) eaiss.segment_current_sections(self.testUUID)
def testConfirmationModeQueryManualNotAuto(self): (user, dummySection, dummyPredModeMap) = self.setupClientTest() get_database.get_section_db().update({'_id': dummySection['_id']}, {'$set': { 'confirmed_mode': 4 }}) retrieveByQuery = get_database.get_section_db().find( common.getConfirmationModeQuery(1)) self.assertEqual(retrieveByQuery.count(), 0) retrieveByQuery = get_database.get_section_db().find( common.getConfirmationModeQuery(4)) self.assertEqual(retrieveByQuery.count(), 1)
def update_profiles(dummy_users=False): if dummy_users: user_list = ['1'] else: user_list = get_section_db().distinct('user_id') for user in user_list: generate_route_clusters(user)
def testRunBackgroundTasksForDay(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.SectionsColl = get_section_db() emission.tests.common.updateSections(self) self.assertNotEqual(len(self.uuid_list), 0) # Can access the zeroth element because we know that then length is greater than zero # (see above) test_uuid = self.uuid_list[0] test_user = User.fromUUID(test_uuid) self.assertNotIn('carbon_footprint', test_user.getProfile().keys()) data.runBackgroundTasks(test_user.uuid) self.assertIn('carbon_footprint', test_user.getProfile().keys())
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = edb.url self.ModesColl = get_mode_db() self.SectionsColl = get_section_db() # Let's make sure that the users are registered so that they have profiles user_objects = [] for userEmail in self.testUsers: user_objects.append(User.register(userEmail)) # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries for testUser in user_objects: etc.purgeSectionData(self.SectionsColl, testUser.uuid) if self.ModesColl.estimated_document_count() > 0: self.ModesColl.delete_many({}) self.assertEqual(self.ModesColl.estimated_document_count(), 0) self.assertEqual(self.SectionsColl.estimated_document_count(), 0) MongoClient(edb.url).drop_database("Backup_database") etc.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") etc.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testModeInferSeedFile") self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.dayago section['section_end_datetime'] = self.dayago + timedelta(hours=1) if (section['confirmed_mode'] == 5): # We only cluster bus and train trips # And our test data only has bus trips section['section_start_point'] = { u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728] } section['section_end_point'] = { u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595] } # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) # Replace the user email with the UUID section['user_id'] = User.fromEmail(section['user_id']).uuid edb.save(self.SectionsColl, section) self.pipeline = pipeline.ModeInferencePipelineMovesFormat() self.testLoadTrainingData()
def setUp(self): self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() # self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) self.SectionsColl = get_section_db() # self.SectionsColl.remove() self.assertEquals(self.SectionsColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0/1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks = 1) emission.tests.common.updateSections(self)
def __init__(self): self.featureLabels = [ "distance", "duration", "first filter mode", "sectionId", "avg speed", "speed EV", "speed variance", "max speed", "max accel", "isCommute", "heading change rate", "stop rate", "velocity change rate", "start lat", "start lng", "stop lat", "stop lng", "start hour", "end hour", "close to bus stop", "close to train stop", "close to airport", ] self.Sections = edb.get_section_db()
def get_section(section_id): section_json = edb.get_section_db().find_one({"id": section_id}) if section_json is None: logging.warning("Did not find match for section %s, returning None" % section_id) return None return ad.AttrDict(section_json)
def label_filtered_section(section): minimum_travel_time=120 minimum_travel_distance=200 Modes=get_mode_db() Sections=get_section_db() is_retained = False # logging.debug("Appending %s" % json.dumps(section)) if section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])>=2: if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \ max_Distance(section['track_points']) >= minimum_travel_distance: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] ='not a trip' elif section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])<2: if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] ='not a trip' elif (section['section_start_time']==''or section['section_end_time']=='') and len(section['track_points'])>=2: if max_Distance(section['track_points']) >= minimum_travel_distance: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] ='not a trip' else: section['type'] ='not complete information' section['retained'] = is_retained
def save_section_to_db(section): print "saving section to db" db = edb.get_section_db() db.insert({"user_id" : section.user_id, "trip_id" : section.trip_id, "distance" : section.distance, "type" : section.section_type, "section_start_datetime" : section.start_time.datetime, "section_end_datetime" : section.end_time.datetime, "section_start_point" : {"coordinates" : section.section_start_location.coordinate_list()}, "section_end_point" : {"coordinates" : section.section_end_location.coordinate_list()}, "mode" : section.mode, "confirmed_mode" : section.confirmed_mode})
def testBlankToday(self): result = self.loadTestJSON("emission/tests/data/test2_blank_today") collect.processResult(self.testUUID, [result[0]]) collect.processResult(self.testUUID, [result[1]]) SectionColl = get_section_db() self.assertTrue(SectionColl.find({'user_id': self.testUUID}).count() > 0)
def save_section_to_db(section): print("saving section to db") db = edb.get_section_db() db.insert({"user_id" : section.user_id, "trip_id" : section.trip_id, "distance" : section.distance, "type" : section.section_type, "section_start_datetime" : section.start_time.datetime, "section_end_datetime" : section.end_time.datetime, "section_start_point" : {"coordinates" : section.section_start_location.coordinate_list()}, "section_end_point" : {"coordinates" : section.section_end_location.coordinate_list()}, "mode" : section.mode, "confirmed_mode" : section.confirmed_mode})
def getRoute(section_id): route=[] Sections=edb.get_section_db() section=Sections.find_one({'_id':section_id}) for point in section['track_points']: route.append(point['track_location']['coordinates']) return route
def testPlaceLoad(self): result = self.loadTestJSON("emission/tests/data/test20140410") collect.processResult(self.testUUID, result) # Check that the trips are loaded correctly TripColl = get_trip_db() firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140409T191531-0700'}]}) logging.debug("selected trip = %s" % firstStoredTrip) # For some reason, the place is always "unknown", at least for this set of test trips. # Maybe it is related to the fact that they haven't been tagged in FourSquare self.assertEqual(firstStoredTrip['type'], 'place') self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700') self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700") self.assertIn('place_location', firstStoredTrip['place']) self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point', 'coordinates': [-122.08632, 37.391]}) # Now, check that we have the sections as well. The previous trip did not # have any sections. This one does tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140410T071320-0700'}]}) self.assertNotEqual(tripWithSections, None) self.assertEqual(tripWithSections['sections'], [0]) SectionColl = get_section_db() sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID, 'trip_id': '20140410T071320-0700', 'section_id': 0}]}) self.assertNotEqual(sectionForTrip, None)
def _init_sections(cls, user_id, trip_id, num_sections): sections = [] db = edb.get_section_db() json_object = db.find({'user_id': user_id, 'trip_id' : trip_id}, limit = num_sections) for section_json in json_object: sections.append(Section.section_from_json(section_json)) return sections
def getRoute(section_id): route = [] Sections = edb.get_section_db() section = Sections.find_one({'_id': section_id}) for point in section['track_points']: route.append(point['track_location']['coordinates']) return route
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() # self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) self.SectionsColl = get_section_db() # self.SectionsColl.remove() self.assertEquals(self.SectionsColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0 / 1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) emission.tests.common.updateSections(self)
def get_mode_share_by_count(list_idx): Sections=get_section_db() ## takes a list of idx's AllModeList = getAllModes() MODE = {} MODE2= {} for mode in AllModeList: MODE[mode['mode_id']]=0 for _id in list_idx: section=Sections.find_one({'_id': _id}) mode_id = section['confirmed_mode'] try: MODE[mode_id] += 1 except KeyError: MODE[mode_id] = 1 # print(sum(MODE.values())) if sum(MODE.values())==0: for mode in AllModeList: MODE2[mode['mode_id']]=0 # print(MODE2) else: for mode in AllModeList: MODE2[mode['mode_id']]=MODE[mode['mode_id']]/sum(MODE.values()) return MODE2
def get_mode_share_by_count(list_idx): Sections = get_section_db() BackupSections = safmt.AbstractCollection(edb.pm_address, "Backup_database", "Stage_Sections", None) ## takes a list of idx's AllModeList = getAllModes() MODE = {} MODE2 = {} for mode in AllModeList: MODE[mode['mode_id']] = 0 for _id in list_idx: section = Sections.find_one({'_id': _id}) if section is None: section = BackupSections.find_one({'id': _id}) mode_id = section['confirmed_mode'] try: MODE[mode_id] += 1 except KeyError: MODE[mode_id] = 1 # print(sum(MODE.values())) if sum(MODE.values()) == 0: for mode in AllModeList: MODE2[mode['mode_id']] = 0 # print(MODE2) else: for mode in AllModeList: MODE2[mode['mode_id']] = MODE[mode['mode_id']] / sum(MODE.values()) return MODE2
def get_clusters_info(uid): c_db = get_routeCluster_db() s_db = get_section_db() clusterJson = c_db.find_one({"clusters":{"$exists":True}, "user": uid}) if clusterJson is None: return [] c_info = [] clusterSectionLists= clusterJson["clusters"].values() logging.debug( "Number of section lists for user %s is %s" % (uid, len(clusterSectionLists))) for sectionList in clusterSectionLists: first = True logging.debug( "Number of sections in sectionList for user %s is %s" % (uid, len(sectionList))) if (len(sectionList) == 0): # There's no point in returning this cluster, let's move on continue distributionArrays = [[] for _ in range(5)] for section in sectionList: section_json = s_db.find_one({"_id":section}) if first: representative_trip = section_json first = False appendIfPresent(distributionArrays[0], section_json, "section_start_datetime") appendIfPresent(distributionArrays[1], section_json, "section_end_datetime") appendIfPresent(distributionArrays[2], section_json, "section_start_point") appendIfPresent(distributionArrays[3], section_json, "section_end_point") appendIfPresent(distributionArrays[4], section_json, "confirmed_mode") c_info.append((distributionArrays, representative_trip)) return c_info
def detect_daily_work_office(user_id,day): # say should be from 1 to 5 Sections=edb.get_section_db() office_candidate=[] home=eamh.detect_home(user_id) if home == 'N/A': return 'N/A' # print(list_first_pnt) for section in Sections.find({"$and":[{"user_id": user_id},{ "section_start_point": { "$ne": None }}]}): section_start_pnt=section['section_start_point'] section_end_pnt=section['section_end_point'] if ec.Is_date(section['section_start_time'],day)==True: # parameter that the distance away from home: away_home=400 if not ec.Is_place(section_start_pnt,home,away_home) and not ec.Is_place(section_end_pnt,home,away_home): office_candidate.append(section['track_points'][0]) office_candidate.append(section['track_points'][-1]) elif ec.Is_place(section_start_pnt,home,away_home) and not ec.Is_place(section_end_pnt,home,away_home): office_candidate.append(section['track_points'][-1]) elif not ec.Is_place(section_start_pnt,home,away_home) and ec.Is_place(section_end_pnt,home,away_home): office_candidate.append(section['track_points'][0]) if len(office_candidate)>0: office_candidate = sorted(office_candidate, key=lambda k: parser.parse(k['time'])) # print(office_candidate) weighted_office_candidate=ec.get_static_pnts(office_candidate) office_location=ec.most_common(weighted_office_candidate,200) # print(len(office_candidate)) # print(len(weighted_office_candidate)) # print(ec.calculate_appearance_rate(office_candidate,office_location)) # print(ec.calculate_appearance_rate(weighted_office_candidate,office_location)) return office_location else: return 'N/A'
def get_mode_share_by_count(list_idx): Sections = get_section_db() ## takes a list of idx's AllModeList = getAllModes() MODE = {} MODE2 = {} for mode in AllModeList: MODE[mode['mode_id']] = 0 for _id in list_idx: section = Sections.find_one({'_id': _id}) mode_id = section['confirmed_mode'] try: MODE[mode_id] += 1 except KeyError: MODE[mode_id] = 1 # print(sum(MODE.values())) if sum(MODE.values()) == 0: for mode in AllModeList: MODE2[mode['mode_id']] = 0 # print(MODE2) else: for mode in AllModeList: MODE2[mode['mode_id']] = MODE[mode['mode_id']] / sum(MODE.values()) return MODE2
def getResult(user_uuid): # This is in here, as opposed to the top level as recommended by the PEP # because then we don't have to worry about loading bottle in the unit tests from bottle import template original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}}) if original_trip is None: return template("clients/recommendation/no_recommendation.html") del original_trip['trip_start_datetime'] del original_trip['trip_end_datetime'] del original_trip['user_id'] del original_trip['pipelineFlags'] del original_trip['recommended_alternative']['user_id'] recommended_trip = original_trip['recommended_alternative'] original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']})) for section in original_sections: del section['user_id'] del section['section_start_datetime'] del section['section_end_datetime'] if 'retained' in section: del section['retained'] del section['manual'] del section['commute'] logging.debug("original sections = %s" % original_sections) logging.debug("recommended trip = %s" % recommended_trip) renderedTemplate = template("clients/recommendation/result_template.html", originalSections = json.dumps(original_sections), recommendedTrip = json.dumps(recommended_trip)) return renderedTemplate
def setUp(self): import emission.tests.common from copy import copy self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) self.setupUserAndClient() emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") self.SectionsColl = get_section_db() self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0/1609 self.airCarbon = 217.0/1609 self.driveCarbon = 278.0/1609 self.busOptimalCarbon = 92.0/1609 self.allDriveExpect = (self.busExpect * self.driveCarbon + self.walkExpect * self.driveCarbon)/1000 self.myFootprintExpect = float(self.busExpect * self.busCarbon)/1000 self.sb375GoalExpect = 40.142892/7 self.mineMinusOptimalExpect = 0 self.allDriveMinusMineExpect = float(self.allDriveExpect - self.myFootprintExpect)/self.allDriveExpect self.sb375DailyGoalMinusMineExpect = float(self.sb375GoalExpect - self.myFootprintExpect)/self.sb375GoalExpect self.now = datetime.now() self.twodaysago = self.now - timedelta(days=2) self.weekago = self.now - timedelta(weeks = 1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.twodaysago section['section_end_datetime'] = self.twodaysago + timedelta(hours = 1) section['predicted_mode'] = {'walking': 1.0} if section['user_id'] == '*****@*****.**': logging.debug("Setting user_id for section %s, %s = %s" % (section['trip_id'], section['section_id'], self.user.uuid)) section['user_id'] = self.user.uuid if section['confirmed_mode'] == 5: airSection = copy(section) airSection['confirmed_mode'] = 9 airSection['_id'] = section['_id'] + "_air" self.SectionsColl.insert(airSection) airSection['confirmed_mode'] = '' airSection['_id'] = section['_id'] + "_unconf" self.SectionsColl.insert(airSection) # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) self.SectionsColl.save(section)
def testRetrieveTripsToImproveWithClusters(self): sectionList = list(get_section_db().find()) get_routeCluster_db().insert({"user": self.testUUID, "clusters": {"cluster1": [s["_id"] for s in sectionList[0:10]], "cluster2": [s["_id"] for s in sectionList[10:20]]}}) trip_list = list(self.pipeline.get_trips_to_improve(self.testUUID)) self.assertEquals(len(trip_list), 2)
def drawSectionsFromList(sectionIdList, outPath): sectionJsonList = [get_section_db().find_one({"_id": sid}) for sid in sectionIdList] bounds = tauq.get_bounds(sectionJsonList) gmap = pygmaps.maps(old_div((bounds[0].lat + bounds[1].lat),2), old_div((bounds[0].lon + bounds[1].lon),2), 10) drawSections(sectionJsonList, ALL, gmap, "random") gmap.draw(outPath)
def display_trip(user, date, option): user_id = UUID(user) user_home = detect_home(user_id) gmap = pygmaps.maps(user_home[1], user_home[0], 14) start, end = Date(date) for section in get_section_db().find({"$and":[{'user_id':user_id},{"section_start_datetime": {"$gte": start, "$lt": end}}]}): drawSection(section, option, gmap) gmap.draw('gmap_display/' + str(start)[:10] + '_' + user + '.html')
def get_user_sections(user_id): sections = list(edb.get_section_db().find( {'$and': [{ 'user_id': user_id }, { 'type': 'move' }]})) return sections
def __init__(self): self.featureLabels = ["distance", "duration", "first filter mode", "sectionId", "avg speed", "speed EV", "speed variance", "max speed", "max accel", "isCommute", "heading change rate", "stop rate", "velocity change rate", "start lat", "start lng", "stop lat", "stop lng", "start hour", "end hour", "close to bus stop", "close to train stop", "close to airport"] self.Sections = edb.get_section_db()
def Berkeley_pop_route(start,end): Sections = get_section_db() list_of_point=[] # print(berkeley_area()) for section in Sections.find({"$and":[{'In_UCB':True },{'type':'move'},{"section_start_datetime": {"$gte": start, "$lt": end}}]}): for pnt in section['track_points']: list_of_point.append(pnt['track_location']['coordinates']) return {"latlng": list_of_point}
def createDummySection(startTime, endTime, startLoc, endLoc, predictedMode = None, confirmedMode = None): section = { 'source': 'Shankari', 'section_start_datetime': startTime, 'section_end_datetime': endTime, 'section_start_time': startTime.isoformat(), 'section_end_time': endTime.isoformat(), 'section_start_point': {'type': 'Point', 'coordinates': startLoc}, 'section_end_point': {'type': 'Point', 'coordinates': endLoc}, } if predictedMode != None: section['predicted_mode'] = predictedMode if confirmedMode != None: section['confirmed_mode'] = confirmedMode get_section_db().insert(section) return section
def get_user_morning_commute_sections(user): # say should be from 1 to 5 # get a list of all the sections for Mon, or ... Sections = edb.get_section_db() list_of_commute = [] for date in range(1, 6): list_of_commute.extend(get_daily_morning_commute_sections(user, date)) return list_of_commute
def testConfirmationModeQueryAutoNoManual(self): (user, dummySection, dummyPredModeMap) = self.setupClientTest() clientSetQuery = Client(user.getFirstStudy()).clientSpecificSetters( user.uuid, dummySection, dummyPredModeMap) # Apply the change get_database.get_section_db().update({'_id': dummySection['_id']}, clientSetQuery) retrievedSection = get_database.get_section_db().find_one( {'_id': dummySection['_id']}) self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1) retrieveByQuery = get_database.get_section_db().find( common.getConfirmationModeQuery(1)) for entry in retrieveByQuery: print entry self.assertEqual(retrieveByQuery.count(), 1)
def display_home(): mymap_home = pygmaps.maps(37.8556475757, -122.248774009, 14) for user in get_section_db().distinct('user_id'): print(user) user_home = detect_home(user) print(user_home) if user_home != 'N/A': mymap_home.addpoint(user_home[1], user_home[0], "#FF0000") mymap_home.draw('mymap_home.html')
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) get_section_db().remove({"user_id": self.testUUID}) result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) print(get_section_db().find().count()) self.pipeline = RecommendationPipeline()
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) get_section_db().remove({"user_id": self.testUUID}) result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) print get_section_db().find().count() self.pipeline = RecommendationPipeline()
def get_berkeley_sections(): Sections = get_section_db() list_of_sections = [] for section in Sections.find(): if section['section_start_point']!=None and section['section_end_point']!=None and \ Inside_polygon(section['section_start_point'],berkeley_area()) and \ Inside_polygon(section['section_end_point'],berkeley_area()): list_of_sections.append(section) return list_of_sections
def display_home(): mymap_home = pygmaps.maps(37.8556475757, -122.248774009,14) for user in get_section_db().distinct('user_id'): print(user) user_home=detect_home(user) print(user_home) if user_home!='N/A': mymap_home.addpoint(user_home[1], user_home[0], "#FF0000") mymap_home.draw('mymap_home.html')
def get_berkeley_sections(): Sections = get_section_db() list_of_sections=[] for section in Sections.find(): if section['section_start_point']!=None and section['section_end_point']!=None and \ Inside_polygon(section['section_start_point'],berkeley_area()) and \ Inside_polygon(section['section_end_point'],berkeley_area()): list_of_sections.append(section) return list_of_sections
def testBlankToday(self): result = self.loadTestJSON("emission/tests/data/test2_blank_today") collect.processResult(self.testUUID, [result[0]]) collect.processResult(self.testUUID, [result[1]]) SectionColl = get_section_db() self.assertTrue( SectionColl.find({ 'user_id': self.testUUID }).count() > 0)
def testYesterdayForJustSignedUp(self): result = self.loadTestJSON( "emission/tests/data/yesterday_for_justsignedup") collect.processResult(self.testUUID, result) SectionColl = get_section_db() self.assertEquals( SectionColl.find({ "user_id": self.testUUID }).count(), 0)
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" #self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**", # "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' self.testUsers = [] #for userEmail in self.testUserEmails: # User.register(userEmail) # self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = edb.get_mode_db() self.ModesColl.remove() edb.get_trip_db().remove() edb.get_section_db().remove() edb.get_alternatives_db().remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) # register each of the users and add sample trips to each user result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) for trip in edb.get_trip_db().find(): trip['trip_start_datetime'] = pydt.datetime.now() + pydt.timedelta( hours=-5) trip['trip_end_datetime'] = pydt.datetime.now() edb.get_trip_db().update({"_id": trip["_id"]}, trip) for section in edb.get_section_db().find(): section['section_start_datetime'] = pydt.datetime.now( ) + pydt.timedelta(hours=-5) section['section_end_datetime'] = pydt.datetime.now() edb.get_section_db().update({"_id": section["_id"]}, section) self.pipeline = AlternativeTripsPipeline()
def get_user_clusters(user, method, nClusters, is_ground_truth=False): if is_ground_truth: routes_user = user_route_data2(user) else: routes_user = user_route_data(user, edb.get_section_db()) if nClusters == -1: nClusters = int(math.ceil(old_div(len(routes_user), 8)) + 1) clusters_user = emkm.kmedoids(routes_user, nClusters, user, method=method) #update_user_routeClusters(user,clusters_user[2],method=method) return clusters_user