def label_filtered_section(section): minimum_travel_time=120 minimum_travel_distance=200 Modes=get_mode_db() Sections=get_section_db() is_retained = False # logging.debug("Appending %s" % json.dumps(section)) if section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])>=2: if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \ max_Distance(section['track_points']) >= minimum_travel_distance: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] ='not a trip' elif section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])<2: if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] ='not a trip' elif (section['section_start_time']==''or section['section_end_time']=='') and len(section['track_points'])>=2: if max_Distance(section['track_points']) >= minimum_travel_distance: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] ='not a trip' else: section['type'] ='not complete information' section['retained'] = is_retained
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() # self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) self.SectionsColl = get_section_db() # self.SectionsColl.remove() self.assertEquals(self.SectionsColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0 / 1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) emission.tests.common.updateSections(self)
def setUp(self): self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() # self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) self.SectionsColl = get_section_db() # self.SectionsColl.remove() self.assertEquals(self.SectionsColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0/1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks = 1) emission.tests.common.updateSections(self)
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" #self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**", # "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' self.testUsers = [] #for userEmail in self.testUserEmails: # User.register(userEmail) # self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) #TODO: add many trip filter functions to play with self.trip_filters = None result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) self.pipeline = UtilityModelPipeline()
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = edb.url self.ModesColl = get_mode_db() self.SectionsColl = get_section_db() # Let's make sure that the users are registered so that they have profiles user_objects = [] for userEmail in self.testUsers: user_objects.append(User.register(userEmail)) # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries for testUser in user_objects: etc.purgeSectionData(self.SectionsColl, testUser.uuid) if self.ModesColl.estimated_document_count() > 0: self.ModesColl.delete_many({}) self.assertEqual(self.ModesColl.estimated_document_count(), 0) self.assertEqual(self.SectionsColl.estimated_document_count(), 0) MongoClient(edb.url).drop_database("Backup_database") etc.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") etc.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testModeInferSeedFile") self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.dayago section['section_end_datetime'] = self.dayago + timedelta(hours=1) if (section['confirmed_mode'] == 5): # We only cluster bus and train trips # And our test data only has bus trips section['section_start_point'] = { u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728] } section['section_end_point'] = { u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595] } # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) # Replace the user email with the UUID section['user_id'] = User.fromEmail(section['user_id']).uuid edb.save(self.SectionsColl, section) self.pipeline = pipeline.ModeInferencePipelineMovesFormat() self.testLoadTrainingData()
def setUp(self): import emission.tests.common from copy import copy self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) self.setupUserAndClient() emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") self.SectionsColl = get_section_db() self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0/1609 self.airCarbon = 217.0/1609 self.driveCarbon = 278.0/1609 self.busOptimalCarbon = 92.0/1609 self.allDriveExpect = (self.busExpect * self.driveCarbon + self.walkExpect * self.driveCarbon)/1000 self.myFootprintExpect = float(self.busExpect * self.busCarbon)/1000 self.sb375GoalExpect = 40.142892/7 self.mineMinusOptimalExpect = 0 self.allDriveMinusMineExpect = float(self.allDriveExpect - self.myFootprintExpect)/self.allDriveExpect self.sb375DailyGoalMinusMineExpect = float(self.sb375GoalExpect - self.myFootprintExpect)/self.sb375GoalExpect self.now = datetime.now() self.twodaysago = self.now - timedelta(days=2) self.weekago = self.now - timedelta(weeks = 1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.twodaysago section['section_end_datetime'] = self.twodaysago + timedelta(hours = 1) section['predicted_mode'] = {'walking': 1.0} if section['user_id'] == '*****@*****.**': logging.debug("Setting user_id for section %s, %s = %s" % (section['trip_id'], section['section_id'], self.user.uuid)) section['user_id'] = self.user.uuid if section['confirmed_mode'] == 5: airSection = copy(section) airSection['confirmed_mode'] = 9 airSection['_id'] = section['_id'] + "_air" self.SectionsColl.insert(airSection) airSection['confirmed_mode'] = '' airSection['_id'] = section['_id'] + "_unconf" self.SectionsColl.insert(airSection) # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) self.SectionsColl.save(section)
def getModeOptions(): Modes = get_mode_db() return_dict = {} modes = [] for mode in Modes.find(): modes.append(mode["mode_name"]) return_dict["modelist"] = modes return return_dict
def getModeOptions(): Modes = get_mode_db() return_dict = {} modes = [] for mode in Modes.find(): modes.append(mode['mode_name']) return_dict['modelist'] = modes return return_dict
def getCommutePopRoute(selMode): mode = get_mode_db().find_one({'mode_name': selMode}) fromTs = request.query.from_ts toTs = request.query.to_ts logging.debug("Filtering values for range %s -> %s" % (fromTs, toTs)) retVal = visualize.Commute_pop_route(mode['mode_id'], datetime.fromtimestamp(float(fromTs)/1000), datetime.fromtimestamp(float(toTs)/1000)) # retVal = common.generateRandomResult(['00-04', '04-08', '08-10']) # logging.debug("In getCalPopRoute, retVal is %s" % retVal) return retVal
def setSectionClassification(uuid, userClassifications): number_class_sec = len(userClassifications) user_uuid = uuid Sections = get_section_db() Modes = get_mode_db() logging.debug("userClassifications = %s" % userClassifications) logging.debug("number_class_sec = %s" % number_class_sec) if number_class_sec != 0: logging.debug("number_class_sec = %s" % number_class_sec) for sectionindex in range(number_class_sec): if userClassifications[sectionindex]["userMode"] == "not a trip": logging.debug("usermode = %s" % userClassifications[sectionindex]["userMode"]) Sections.update( { "$and": [ {"source": "Shankari"}, {"user_id": user_uuid}, {"trip_id": userClassifications[sectionindex]["trip_id"]}, {"section_id": int(userClassifications[sectionindex]["section_id"])}, ] }, {"$set": {"type": userClassifications[sectionindex]["userMode"]}}, ) logging.debug("update done") else: Sections.update( { "$and": [ {"source": "Shankari"}, {"user_id": user_uuid}, {"trip_id": userClassifications[sectionindex]["trip_id"]}, {"section_id": int(userClassifications[sectionindex]["section_id"])}, ] }, { "$set": { "confirmed_mode": int( "".join( map( str, [ mode["mode_id"] for mode in Modes.find( {"mode_name": userClassifications[sectionindex]["userMode"]} ) ], ) ) ) if Modes.find({"mode_name": userClassifications[sectionindex]["userMode"]}).count() != 0 else userClassifications[sectionindex]["userMode"] } }, ) logging.debug("update done")
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) self.SectionsColl = get_section_db() self.assertEquals(self.SectionsColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable( self.serverName, "Stage_Sections", "emission/tests/data/testModeInferFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.dayago section['section_end_datetime'] = self.dayago + timedelta(hours=1) if (section['confirmed_mode'] == 5): # We only cluster bus and train trips # And our test data only has bus trips section['section_start_point'] = { u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728] } section['section_end_point'] = { u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595] } # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) # Replace the user email with the UUID section['user_id'] = User.fromEmail(section['user_id']).uuid self.SectionsColl.save(section) self.pipeline = pipeline.ModeInferencePipeline() self.testLoadTrainingData()
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row)
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row)
def loadTrainingDataStep(self, sectionQuery, sectionDb=None): logging.debug("START TRAINING DATA STEP") if (sectionDb == None): sectionDb = self.Sections begin = time.time() logging.debug("Section data set size = %s" % sectionDb.find({ 'type': 'move' }).count()) duration = time.time() - begin logging.debug("Getting dataset size took %s" % (duration)) logging.debug("Querying confirmedSections %s" % (datetime.now())) begin = time.time() confirmedSections = sectionDb.find(sectionQuery).sort('_id', 1) duration = time.time() - begin logging.debug("Querying confirmedSection took %s" % (duration)) logging.debug("Querying stage modes %s" % (datetime.now())) begin = time.time() modeList = [] for mode in edb.get_mode_db().find(): modeList.append(mode) logging.debug(mode) duration = time.time() - begin logging.debug("Querying stage modes took %s" % (duration)) logging.debug("Section query with ground truth %s" % (datetime.now())) begin = time.time() logging.debug("Training set total size = %s" % sectionDb.find( ModeInferencePipelineMovesFormat.getSectionQueryWithGroundTruth( {'$ne': ''})).count()) for mode in modeList: logging.debug( "%s: %s" % (mode['mode_name'], sectionDb.find( ModeInferencePipelineMovesFormat. getSectionQueryWithGroundTruth(mode['mode_id'])))) duration = time.time() - begin logging.debug("Getting section query with ground truth took %s" % (duration)) duration = time.time() - begin return confirmedSections
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", ] self.serverName = "localhost" # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) self.SectionsColl = get_section_db() self.assertEquals(self.SectionsColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testModeInferFile") # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) for section in self.SectionsColl.find(): section["section_start_datetime"] = self.dayago section["section_end_datetime"] = self.dayago + timedelta(hours=1) if section["confirmed_mode"] == 5: # We only cluster bus and train trips # And our test data only has bus trips section["section_start_point"] = {u"type": u"Point", u"coordinates": [-122.270039042, 37.8800285728]} section["section_end_point"] = {u"type": u"Point", u"coordinates": [-122.2690412952, 37.8739578595]} # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) # Replace the user email with the UUID section["user_id"] = User.fromEmail(section["user_id"]).uuid self.SectionsColl.save(section) self.pipeline = pipeline.ModeInferencePipeline() self.testLoadTrainingData()
def loadTrainingDataStep(self, sectionQuery, sectionDb=None): logging.debug("START TRAINING DATA STEP") if sectionDb == None: sectionDb = self.Sections begin = time.time() logging.debug("Section data set size = %s" % sectionDb.find({"type": "move"}).count()) duration = time.time() - begin logging.debug("Getting dataset size took %s" % (duration)) logging.debug("Querying confirmedSections %s" % (datetime.now())) begin = time.time() confirmedSections = sectionDb.find(sectionQuery) duration = time.time() - begin logging.debug("Querying confirmedSection took %s" % (duration)) logging.debug("Querying stage modes %s" % (datetime.now())) begin = time.time() modeList = [] for mode in edb.get_mode_db().find(): modeList.append(mode) logging.debug(mode) duration = time.time() - begin logging.debug("Querying stage modes took %s" % (duration)) logging.debug("Section query with ground truth %s" % (datetime.now())) begin = time.time() logging.debug( "Training set total size = %s" % sectionDb.find(ModeInferencePipeline.getSectionQueryWithGroundTruth({"$ne": ""})).count() ) for mode in modeList: logging.debug( "%s: %s" % ( mode["mode_name"], sectionDb.find(ModeInferencePipeline.getSectionQueryWithGroundTruth(mode["mode_id"])), ) ) duration = time.time() - begin logging.debug("Getting section query with ground truth took %s" % (duration)) duration = time.time() - begin return (modeList, confirmedSections)
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) get_section_db().remove({"user_id": self.testUUID}) result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) print(get_section_db().find().count()) self.pipeline = RecommendationPipeline()
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = get_mode_db() self.ModesColl.remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) get_section_db().remove({"user_id": self.testUUID}) result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) print get_section_db().find().count() self.pipeline = RecommendationPipeline()
def setUp(self): import emission.tests.common from copy import copy self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") self.SectionsColl = get_section_db() self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0/1609 self.airCarbon = 217.0/1609 self.driveCarbon = 278.0/1609 self.busOptimalCarbon = 92.0/1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks = 1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.dayago section['section_end_datetime'] = self.dayago + timedelta(hours = 1) # Note that we currently only search by the date/time of the section, not the points. # If we ever change that, this test will start failing and will need to be fixed as well track_pt_array = [] for i in range(10): track_pt_array.append({'time': '20140829T170451-0700', 'track_location': {'coordinates': [-122.114642519, 37.4021455446], 'type': 'Point'}}) track_pt_array.append({'time': '20140829T170620-0700', 'track_location': {'coordinates': [-122.1099155383, 37.399523614], 'type': 'Point'}}) section['track_points'] = track_pt_array self.SectionsColl.save(section)
def setUp(self): from copy import copy self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries etc.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) etc.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") etc.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") self.SectionsColl = get_section_db() self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0 / 1609 self.airCarbon = 217.0 / 1609 self.driveCarbon = 278.0 / 1609 self.busOptimalCarbon = 92.0 / 1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.dayago section['section_end_datetime'] = self.dayago + timedelta(hours=1) if section['confirmed_mode'] == 5: airSection = copy(section) airSection['confirmed_mode'] = 9 airSection['_id'] = section['_id'] + "_air" self.SectionsColl.insert(airSection) # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) self.SectionsColl.save(section)
def clientSpecificSetters(uuid, sectionId, predictedModeMap): from emission.core import common from emission.core.get_database import get_mode_db maxMode = None maxProb = 0 for mode, prob in predictedModeMap.iteritems(): print "Considering mode %s and prob %s" % (mode, prob) if prob > maxProb: maxProb = prob maxMode = mode print "maxMode = %s, maxProb = %s" % (mode, prob) return {"$set": {"test_auto_confirmed": { "mode": common.convertModeNameToIndex(get_mode_db(), maxMode), "prob": maxProb, } } }
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" #self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**", # "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' self.testUsers = [] #for userEmail in self.testUserEmails: # User.register(userEmail) # self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = edb.get_mode_db() self.ModesColl.remove() edb.get_trip_db().remove() edb.get_section_db().remove() edb.get_alternatives_db().remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) # register each of the users and add sample trips to each user result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) for trip in edb.get_trip_db().find(): trip['trip_start_datetime'] = pydt.datetime.now() + pydt.timedelta( hours=-5) trip['trip_end_datetime'] = pydt.datetime.now() edb.get_trip_db().update({"_id": trip["_id"]}, trip) for section in edb.get_section_db().find(): section['section_start_datetime'] = pydt.datetime.now( ) + pydt.timedelta(hours=-5) section['section_end_datetime'] = pydt.datetime.now() edb.get_section_db().update({"_id": section["_id"]}, section) self.pipeline = AlternativeTripsPipeline()
def setUp(self): from copy import copy self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries etc.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) etc.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") etc.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") self.SectionsColl = get_section_db() self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0/1609 self.airCarbon = 217.0/1609 self.driveCarbon = 278.0/1609 self.busOptimalCarbon = 92.0/1609 self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks = 1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.dayago section['section_end_datetime'] = self.dayago + timedelta(hours = 1) if section['confirmed_mode'] == 5: airSection = copy(section) airSection['confirmed_mode'] = 9 airSection['_id'] = section['_id'] + "_air" self.SectionsColl.insert(airSection) # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) self.SectionsColl.save(section)
def setUp(self): self.testUUID = "myuuidisverylongandcomplicated" #self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**", # "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' self.testUsers = [] #for userEmail in self.testUserEmails: # User.register(userEmail) # self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries self.ModesColl = edb.get_mode_db() self.ModesColl.remove() edb.get_trip_db().remove() edb.get_section_db().remove() edb.get_alternatives_db().remove() self.assertEquals(self.ModesColl.find().count(), 0) dataJSON = json.load(open("emission/tests/data/modes.json")) for row in dataJSON: self.ModesColl.insert(row) # register each of the users and add sample trips to each user result = self.loadTestJSON("emission/tests/data/missing_trip") collect.processResult(self.testUUID, result) for trip in edb.get_trip_db().find(): trip['trip_start_datetime'] = pydt.datetime.now() + pydt.timedelta(hours=-5) trip['trip_end_datetime'] = pydt.datetime.now() edb.get_trip_db().update({"_id": trip["_id"]}, trip) for section in edb.get_section_db().find(): section['section_start_datetime'] = pydt.datetime.now() + pydt.timedelta(hours=-5) section['section_end_datetime'] = pydt.datetime.now() edb.get_section_db().update({"_id": section["_id"]}, section) self.pipeline = AlternativeTripsPipeline()
def label_filtered_section(section): minimum_travel_time = 120 minimum_travel_distance = 200 Modes = get_mode_db() Sections = get_section_db() is_retained = False # logging.debug("Appending %s" % json.dumps(section)) if section['section_start_time'] != '' and section[ 'section_end_time'] != '' and len(section['track_points']) >= 2: if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \ max_Distance(section['track_points']) >= minimum_travel_distance: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] = 'not a trip' elif section['section_start_time'] != '' and section[ 'section_end_time'] != '' and len(section['track_points']) < 2: if travel_time(section['section_start_time'], section['section_end_time']) >= minimum_travel_time: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] = 'not a trip' elif (section['section_start_time'] == '' or section['section_end_time'] == '') and len(section['track_points']) >= 2: if max_Distance(section['track_points']) >= minimum_travel_distance: section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \ if type(section['mode'])!=type('aa') else section['mode'] is_retained = True else: section['type'] = 'not a trip' else: section['type'] = 'not complete information' section['retained'] = is_retained
# Standard imports from __future__ import division import math import logging import numpy as np import utm from sklearn.cluster import DBSCAN # Our imports from emission.core.get_database import get_section_db, get_mode_db, get_routeCluster_db, get_transit_db from emission.core.common import calDistance, Include_place_2 from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import getRoute, fullMatchDistance, matchTransitRoutes, matchTransitStops Sections = get_section_db() Modes = get_mode_db() # The speed is in m/s def calSpeed(trackpoint1, trackpoint2): from dateutil import parser distanceDelta = calDistance(trackpoint1['track_location']['coordinates'], trackpoint2['track_location']['coordinates']) timeDelta = parser.parse(trackpoint2['time']) - parser.parse( trackpoint1['time']) # logging.debug("while calculating speed form %s -> %s, distanceDelta = %s, timeDelta = %s" % # (trackpoint1, trackpoint2, distanceDelta, timeDelta)) if timeDelta.total_seconds() != 0: return distanceDelta / timeDelta.total_seconds() else: return None
# Standard imports import math import logging import numpy as np import utm from sklearn.cluster import DBSCAN # Our imports from emission.core.get_database import get_section_db, get_mode_db, get_routeCluster_db,get_transit_db from emission.core.common import calDistance, Include_place_2 from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import getRoute,fullMatchDistance,matchTransitRoutes,matchTransitStops Sections = get_section_db() Modes = get_mode_db() # The speed is in m/s def calSpeed(trackpoint1, trackpoint2): from dateutil import parser distanceDelta = calDistance(trackpoint1['track_location']['coordinates'], trackpoint2['track_location']['coordinates']) timeDelta = parser.parse(trackpoint2['time']) - parser.parse(trackpoint1['time']) # logging.debug("while calculating speed form %s -> %s, distanceDelta = %s, timeDelta = %s" % # (trackpoint1, trackpoint2, distanceDelta, timeDelta)) if timeDelta.total_seconds() != 0: return distanceDelta / timeDelta.total_seconds() else: return None # This formula is from: # http://www.movable-type.co.uk/scripts/latlong.html
def processTripArray(user_uuid, trip_array): # First, we open a connection to the database Stage_Trips = get_trip_db() Stage_Sections = get_section_db() Modes = get_mode_db() number_of_trips = len(trip_array) logging.info("number of trips = %s" % number_of_trips) for trip in range(number_of_trips): seg_note = trip_array[trip] trip_id = seg_note["startTime"] _id_trip = str(user_uuid) + '_' + seg_note["startTime"] #logging.debug(json.dumps(seg_note)) if "activities" in seg_note: number_of_sections = len(seg_note["activities"]) logging.debug("number of sections = %s" % number_of_sections) for sectionindex in range(number_of_sections): seg_act_note = seg_note["activities"][sectionindex] # if the section is missing some data that we access later, then we skip it if Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }, { "section_id": sectionindex }] }).count() == 0: try: _id_section = str(user_uuid) + '_' + seg_act_note[ "startTime"] + '_' + str(sectionindex) _mode = convertModeNameToIndex( Modes, seg_act_note["activity"]) isManual = seg_act_note[ "manual"] if "manual" in seg_act_note else False sections_todo = { 'source': 'Shankari', '_id': _id_section, 'user_id': user_uuid, 'trip_id': trip_id, 'type': seg_note["type"], 'section_id': sectionindex, 'mode': _mode, # SHANKARI: what does seg_act_note["manual"] mean? 'confirmed_mode': _mode if isManual else '', # 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})]))) # if "group" in seg_act_note else '', } fillSectionWithMovesData(seg_act_note, sections_todo) label_filtered_section(sections_todo) # Now that we have created this section, let's insert it into the database try: logging.info( "About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " % (trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"])) Stage_Sections.insert(sections_todo) except DuplicateKeyError: logging.warning( "DuplicateKeyError, skipping insert %s" % sections_todo) logging.warning( "Existing section is %s" % Stage_Sections.find_one({"_id": _id_section})) except KeyError, e: logging.warning( "Missing key %s, skipping section insert %s" % (e, seg_act_note)) insertedSectionCount = Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }, { "section_id": sectionindex }] }).count() if insertedSectionCount == 0: logging.error( "Insert appears to have FAILED. No entry for %s, %s, %s found" % (user_uuid, trip_id, sectionindex)) else: logging.debug( "Found existing matching entry for %s, %s, %s, skipping entry" % (user_uuid, trip_id, sectionindex)) # Insert a trip if it doesn't already exist # SHANKARI: What if we get other sections for a trip later? When do we update the trip? # Do we even need to keep this linkage, with the concomittant # management cost if we can just find all sections by trip_id # instead? How expensive is the query? if Stage_Trips.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }] }).count() == 0: trips_todo = { 'source': 'Shankari', '_id': _id_trip, 'user_id': user_uuid, 'trip_id': trip_id, 'sections': [ sections['section_id'] for sections in Stage_Sections.find({ "$and": [{ "user_id": user_uuid }, { "trip_id": trip_id }] }) ] } fillTripWithMovesData(seg_note, trips_todo) logging.info("About to insert trip with trip_id = %s " % (trip_id)) Stage_Trips.insert(trips_todo) else: logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def setSectionClassification(uuid, userClassifications): number_class_sec = len(userClassifications) user_uuid = uuid Sections = get_section_db() Modes = get_mode_db() logging.debug("userClassifications = %s" % userClassifications) logging.debug("number_class_sec = %s" % number_class_sec) if number_class_sec != 0: logging.debug("number_class_sec = %s" % number_class_sec) for sectionindex in range(number_class_sec): if userClassifications[sectionindex]['userMode'] == 'not a trip': logging.debug("usermode = %s" % userClassifications[sectionindex]['userMode']) Sections.update( { "$and": [{ 'source': 'Shankari' }, { 'user_id': user_uuid }, { 'trip_id': userClassifications[sectionindex]['trip_id'] }, { 'section_id': int(userClassifications[sectionindex] ['section_id']) }] }, { "$set": { 'type': userClassifications[sectionindex]['userMode'] } }) logging.debug("update done") else: Sections.update( { "$and": [{ 'source': 'Shankari' }, { 'user_id': user_uuid }, { 'trip_id': userClassifications[sectionindex]['trip_id'] }, { 'section_id': int(userClassifications[sectionindex] ['section_id']) }] }, { "$set": { 'confirmed_mode': int(''.join( map(str, [ mode['mode_id'] for mode in Modes.find({ 'mode_name': userClassifications[sectionindex] ['userMode'] }) ]))) if Modes.find({ 'mode_name': userClassifications[sectionindex]['userMode'] }).count() != 0 else userClassifications[sectionindex]['userMode'] } }) logging.debug("update done")
def processTripArray(user_uuid, trip_array): # First, we open a connection to the database Stage_Trips=get_trip_db() Stage_Sections=get_section_db() Modes=get_mode_db() number_of_trips=len(trip_array) logging.info("number of trips = %s" % number_of_trips) for trip in range(number_of_trips): seg_note=trip_array[trip] trip_id=seg_note["startTime"] _id_trip=str(user_uuid)+'_'+seg_note["startTime"] #logging.debug(json.dumps(seg_note)) if "activities" in seg_note: number_of_sections=len(seg_note["activities"]) logging.debug("number of sections = %s" % number_of_sections) for sectionindex in range(number_of_sections): seg_act_note=seg_note["activities"][sectionindex] # if the section is missing some data that we access later, then we skip it if Stage_Sections.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id},{"section_id": sectionindex}]}).count()==0: try: _id_section = str(user_uuid)+'_'+seg_act_note["startTime"]+'_'+str(sectionindex) _mode = convertModeNameToIndex(Modes, seg_act_note["activity"]) isManual = seg_act_note["manual"] if "manual" in seg_act_note else False sections_todo={'source':'Shankari', '_id':_id_section, 'user_id': user_uuid, 'trip_id':trip_id, 'type':seg_note["type"], 'section_id':sectionindex, 'mode' : _mode, # SHANKARI: what does seg_act_note["manual"] mean? 'confirmed_mode' :_mode if isManual else '', # 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})]))) # if "group" in seg_act_note else '', } fillSectionWithMovesData(seg_act_note, sections_todo) label_filtered_section(sections_todo) # Now that we have created this section, let's insert it into the database try: logging.info("About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " % (trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"])) Stage_Sections.insert(sections_todo) except DuplicateKeyError: logging.warning("DuplicateKeyError, skipping insert %s" % sections_todo) logging.warning("Existing section is %s" % Stage_Sections.find_one({"_id": _id_section})) except KeyError, e: logging.warning("Missing key %s, skipping section insert %s" % (e, seg_act_note)) insertedSectionCount = Stage_Sections.find({"$and" : [{"user_id": user_uuid}, {"trip_id": trip_id}, {"section_id": sectionindex}]}).count() if insertedSectionCount == 0: logging.error("Insert appears to have FAILED. No entry for %s, %s, %s found" % (user_uuid, trip_id, sectionindex)) else: logging.debug("Found existing matching entry for %s, %s, %s, skipping entry" % (user_uuid, trip_id, sectionindex)) # Insert a trip if it doesn't already exist # SHANKARI: What if we get other sections for a trip later? When do we update the trip? # Do we even need to keep this linkage, with the concomittant # management cost if we can just find all sections by trip_id # instead? How expensive is the query? if Stage_Trips.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id}]}).count()==0: trips_todo={ 'source':'Shankari', '_id':_id_trip, 'user_id': user_uuid, 'trip_id':trip_id, 'sections':[sections['section_id'] for sections in Stage_Sections.find({"$and":[{"user_id":user_uuid}, {"trip_id":trip_id}]})]} fillTripWithMovesData(seg_note, trips_todo) logging.info("About to insert trip with trip_id = %s " % (trip_id)) Stage_Trips.insert(trips_todo) else: logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def getAllModes(): Modes = get_mode_db() modes = [] for mode in Modes.find(): modes.append(mode) return modes
def setUp(self): import emission.tests.common from copy import copy self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(get_db()) self.ModesColl = get_mode_db() self.assertEquals(self.ModesColl.find().count(), 0) self.setupUserAndClient() emission.tests.common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json") emission.tests.common.loadTable(self.serverName, "Stage_Sections", "emission/tests/data/testCarbonFile") self.SectionsColl = get_section_db() self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = 267.0 / 1609 self.airCarbon = 217.0 / 1609 self.driveCarbon = 278.0 / 1609 self.busOptimalCarbon = 92.0 / 1609 self.allDriveExpect = (self.busExpect * self.driveCarbon + self.walkExpect * self.driveCarbon) / 1000 self.myFootprintExpect = float(self.busExpect * self.busCarbon) / 1000 self.sb375GoalExpect = 40.142892 / 7 self.mineMinusOptimalExpect = 0 self.allDriveMinusMineExpect = float( self.allDriveExpect - self.myFootprintExpect) / self.allDriveExpect self.sb375DailyGoalMinusMineExpect = float( self.sb375GoalExpect - self.myFootprintExpect) / self.sb375GoalExpect self.now = datetime.now() self.twodaysago = self.now - timedelta(days=2) self.weekago = self.now - timedelta(weeks=1) for section in self.SectionsColl.find(): section['section_start_datetime'] = self.twodaysago section['section_end_datetime'] = self.twodaysago + timedelta( hours=1) section['predicted_mode'] = {'walking': 1.0} if section['user_id'] == '*****@*****.**': logging.debug("Setting user_id for section %s, %s = %s" % (section['trip_id'], section['section_id'], self.user.uuid)) section['user_id'] = self.user.uuid if section['confirmed_mode'] == 5: airSection = copy(section) airSection['confirmed_mode'] = 9 airSection['_id'] = section['_id'] + "_air" self.SectionsColl.insert(airSection) airSection['confirmed_mode'] = '' airSection['_id'] = section['_id'] + "_unconf" self.SectionsColl.insert(airSection) # print("Section start = %s, section end = %s" % # (section['section_start_datetime'], section['section_end_datetime'])) self.SectionsColl.save(section)