def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
tests.common.updateSections(self)
def getAllModes():
Modes = get_mode_db()
# Modes = MongoClient().Test_database.Test_Modes
modes = []
for mode in Modes.find():
modes.append(mode)
return modes
def setUp(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
tests.common.updateSections(self)
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
#self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
# "*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
self.testUsers = []
#for userEmail in self.testUserEmails:
# User.register(userEmail)
# self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
#TODO: add many trip filter functions to play with
self.trip_filters = None
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
self.pipeline = UtilityModelPipeline()
def setSectionClassification(uuid, userClassifications):
number_class_sec=len(userClassifications)
user_uuid=uuid
Sections=get_section_db()
Modes=get_mode_db()
logging.debug("userClassifications = %s" % userClassifications)
logging.debug("number_class_sec = %s" % number_class_sec)
if number_class_sec!=0:
logging.debug("number_class_sec = %s" % number_class_sec)
for sectionindex in range(number_class_sec):
if userClassifications[sectionindex]['userMode']=='not a trip':
logging.debug("usermode = %s" % userClassifications[sectionindex]['userMode'])
Sections.update({"$and":[{'source':'Shankari'},
{'user_id': user_uuid},
{'trip_id': userClassifications[sectionindex]['trip_id']},
{'section_id': int(userClassifications[sectionindex]['section_id'])}]},
{"$set":{'type':userClassifications[sectionindex]['userMode']}})
logging.debug("update done" )
else:
Sections.update({"$and":[{'source':'Shankari'},
{'user_id': user_uuid},
{'trip_id': userClassifications[sectionindex]['trip_id']},
{'section_id': int(userClassifications[sectionindex]['section_id'])}]},
{"$set":{'confirmed_mode':int(''.join(map(str, [mode['mode_id']
for mode in Modes.find({'mode_name':userClassifications[sectionindex]['userMode']})])))
if Modes.find({'mode_name':userClassifications[sectionindex]['userMode']}).count()!=0
else userClassifications[sectionindex]['userMode']}})
logging.debug("update done" )
def label_filtered_section(section):
minimum_travel_time=120
minimum_travel_distance=200
Modes=get_mode_db()
Sections=get_section_db()
is_retained = False
# logging.debug("Appending %s" % json.dumps(section))
if section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])>=2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \
max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
elif section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])<2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
elif (section['section_start_time']==''or section['section_end_time']=='') and len(section['track_points'])>=2:
if max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
else:
section['type'] ='not complete information'
section['retained'] = is_retained
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
#self.testUserEmails = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
# "*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
self.testUsers = []
#for userEmail in self.testUserEmails:
# User.register(userEmail)
# self.testUsers += [User.fromEmail(section['user_id'])] # can access uuid with .uuid
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
#TODO: add many trip filter functions to play with
self.trip_filters = None
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
self.pipeline = UtilityModelPipeline()
def setUp(self):
import tests.common
from copy import copy
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
self.setupUserAndClient()
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.airCarbon = 217.0/1609
self.driveCarbon = 278.0/1609
self.busOptimalCarbon = 92.0/1609
self.allDriveExpect = (self.busExpect * self.driveCarbon + self.walkExpect * self.driveCarbon)/1000
self.myFootprintExpect = float(self.busExpect * self.busCarbon)/1000
self.sb375GoalExpect = 40.142892/7
self.mineMinusOptimalExpect = 0
self.allDriveMinusMineExpect = float(self.allDriveExpect - self.myFootprintExpect)/self.allDriveExpect
self.sb375DailyGoalMinusMineExpect = float(self.sb375GoalExpect - self.myFootprintExpect)/self.sb375GoalExpect
self.now = datetime.now()
self.twodaysago = self.now - timedelta(days=2)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.twodaysago
section['section_end_datetime'] = self.twodaysago + timedelta(hours = 1)
section['predicted_mode'] = {'walking': 1.0}
if section['user_id'] == '*****@*****.**':
logging.debug("Setting user_id for section %s, %s = %s" %
(section['trip_id'], section['section_id'], self.user.uuid))
section['user_id'] = self.user.uuid
if section['confirmed_mode'] == 5:
airSection = copy(section)
airSection['confirmed_mode'] = 9
airSection['_id'] = section['_id'] + "_air"
self.SectionsColl.insert(airSection)
airSection['confirmed_mode'] = ''
airSection['_id'] = section['_id'] + "_unconf"
self.SectionsColl.insert(airSection)
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
self.SectionsColl.save(section)
def getModeOptions():
Modes = get_mode_db()
return_dict = {}
modes = []
for mode in Modes.find():
modes.append(mode['mode_name'])
return_dict['modelist'] = modes
return return_dict
def getModeOptions():
Modes=get_mode_db()
return_dict = {}
modes=[]
for mode in Modes.find():
modes.append(mode['mode_name'])
return_dict['modelist'] = modes
return return_dict
def setUp(self):
import tests.common
from copy import copy
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.airCarbon = 217.0 / 1609
self.driveCarbon = 278.0 / 1609
self.busOptimalCarbon = 92.0 / 1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours=1)
# Note that we currently only search by the date/time of the section, not the points.
# If we ever change that, this test will start failing and will need to be fixed as well
track_pt_array = []
for i in range(10):
track_pt_array.append({
'time': '20140829T170451-0700',
'track_location': {
'coordinates': [-122.114642519, 37.4021455446],
'type': 'Point'
}
})
track_pt_array.append({
'time': '20140829T170620-0700',
'track_location': {
'coordinates': [-122.1099155383, 37.399523614],
'type': 'Point'
}
})
section['track_points'] = track_pt_array
self.SectionsColl.save(section)
def getCommutePopRoute(selMode):
mode = get_mode_db().find_one({'mode_name': selMode})
fromTs = request.query.from_ts
toTs = request.query.to_ts
logging.debug("Filtering values for range %s -> %s" % (fromTs, toTs))
retVal = visualize.Commute_pop_route(mode['mode_id'],
datetime.fromtimestamp(float(fromTs)/1000), datetime.fromtimestamp(float(toTs)/1000))
# retVal = common.generateRandomResult(['00-04', '04-08', '08-10'])
# logging.debug("In getCalPopRoute, retVal is %s" % retVal)
return retVal
def getCommutePopRoute(selMode):
mode = get_mode_db().find_one({'mode_name': selMode})
fromTs = request.query.from_ts
toTs = request.query.to_ts
logging.debug("Filtering values for range %s -> %s" % (fromTs, toTs))
retVal = visualize.Commute_pop_route(mode['mode_id'],
datetime.fromtimestamp(float(fromTs)/1000), datetime.fromtimestamp(float(toTs)/1000))
# retVal = common.generateRandomResult(['00-04', '04-08', '08-10'])
# logging.debug("In getCalPopRoute, retVal is %s" % retVal)
return retVal
def setUp(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testModeInferFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours=1)
if (section['confirmed_mode'] == 5):
# We only cluster bus and train trips
# And our test data only has bus trips
section['section_start_point'] = {
u'type': u'Point',
u'coordinates': [-122.270039042, 37.8800285728]
}
section['section_end_point'] = {
u'type': u'Point',
u'coordinates': [-122.2690412952, 37.8739578595]
}
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
# Replace the user email with the UUID
section['user_id'] = User.fromEmail(section['user_id']).uuid
self.SectionsColl.save(section)
self.pipeline = pipeline.ModeInferencePipeline()
self.testLoadTrainingData()
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
def clientSpecificSetters(uuid, sectionId, predictedModeMap):
from main import common
from get_database import get_mode_db
maxMode = None
maxProb = 0
for mode, prob in predictedModeMap.iteritems():
if prob > maxProb:
maxProb = prob
maxMode = mode
return {"$set":
{"test_auto_confirmed": {
"mode": common.convertModeNameToIndex(get_mode_db(), maxMode),
"prob": maxProb,
}
}
}
def setUp(self):
import tests.common
from copy import copy
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.airCarbon = 217.0 / 1609
self.driveCarbon = 278.0 / 1609
self.busOptimalCarbon = 92.0 / 1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours=1)
if section['confirmed_mode'] == 5:
airSection = copy(section)
airSection['confirmed_mode'] = 9
airSection['_id'] = section['_id'] + "_air"
self.SectionsColl.insert(airSection)
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
self.SectionsColl.save(section)
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
get_section_db().remove({"user_id": self.testUUID})
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
print get_section_db().find().count()
self.pipeline = RecommendationPipeline()
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
get_section_db().remove({"user_id": self.testUUID})
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
print get_section_db().find().count()
self.pipeline = RecommendationPipeline()
def clientSpecificSetters(uuid, sectionId, predictedModeMap):
from main import common
from get_database import get_mode_db
maxMode = None
maxProb = 0
for mode, prob in predictedModeMap.iteritems():
if prob > maxProb:
maxProb = prob
maxMode = mode
return {
"$set": {
"test_auto_confirmed": {
"mode": common.convertModeNameToIndex(get_mode_db(), maxMode),
"prob": maxProb,
}
}
}
def setUp(self):
import tests.common
from copy import copy
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.airCarbon = 217.0/1609
self.driveCarbon = 278.0/1609
self.busOptimalCarbon = 92.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
# Note that we currently only search by the date/time of the section, not the points.
# If we ever change that, this test will start failing and will need to be fixed as well
track_pt_array = []
for i in range(10):
track_pt_array.append({'time': '20140829T170451-0700',
'track_location': {'coordinates': [-122.114642519, 37.4021455446],
'type': 'Point'}})
track_pt_array.append({'time': '20140829T170620-0700',
'track_location': {'coordinates': [-122.1099155383, 37.399523614],
'type': 'Point'}})
section['track_points'] = track_pt_array
self.SectionsColl.save(section)
def clientSpecificSetters(uuid, sectionId, predictedModeMap):
from main import common
from get_database import get_mode_db
maxMode = None
maxProb = 0
for mode, prob in predictedModeMap.iteritems():
logging.debug("Considering mode %s and prob %s" % (mode, prob))
if prob > maxProb:
maxProb = prob
maxMode = mode
logging.debug("maxMode = %s, maxProb = %s" % (maxMode, maxProb))
return {"$set":
{"auto_confirmed": {
"mode": common.convertModeNameToIndex(get_mode_db(), maxMode),
"prob": maxProb,
}
}
}
def clientSpecificSetters(uuid, sectionId, predictedModeMap):
from main import common
from get_database import get_mode_db
maxMode = None
maxProb = 0
for mode, prob in predictedModeMap.iteritems():
logging.debug("Considering mode %s and prob %s" % (mode, prob))
if prob > maxProb:
maxProb = prob
maxMode = mode
logging.debug("maxMode = %s, maxProb = %s" % (maxMode, maxProb))
return {
"$set": {
"auto_confirmed": {
"mode": common.convertModeNameToIndex(get_mode_db(), maxMode),
"prob": maxProb,
}
}
}
def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
section['predicted_mode'] = [0, 0.4, 0.6, 0]
section['confirmed_mode'] = ''
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
# Replace the user email with the UUID
section['user_id'] = User.fromEmail(section['user_id']).uuid
self.SectionsColl.save(section)
def setUp(self):
import tests.common
from copy import copy
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.airCarbon = 217.0/1609
self.driveCarbon = 278.0/1609
self.busOptimalCarbon = 92.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
if section['confirmed_mode'] == 5:
airSection = copy(section)
airSection['confirmed_mode'] = 9
airSection['_id'] = section['_id'] + "_air"
self.SectionsColl.insert(airSection)
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
self.SectionsColl.save(section)
def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testModeInferFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours = 1)
if (section['confirmed_mode'] == 5):
# We only cluster bus and train trips
# And our test data only has bus trips
section['section_start_point'] = {u'type': u'Point', u'coordinates': [-122.270039042, 37.8800285728]}
section['section_end_point'] = {u'type': u'Point', u'coordinates': [-122.2690412952, 37.8739578595]}
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
# Replace the user email with the UUID
section['user_id'] = User.fromEmail(section['user_id']).uuid
self.SectionsColl.save(section)
self.pr = precompute_results.PrecomputeResults()
def label_filtered_section(section):
minimum_travel_time = 120
minimum_travel_distance = 200
Modes = get_mode_db()
Sections = get_section_db()
is_retained = False
# logging.debug("Appending %s" % json.dumps(section))
if section['section_start_time'] != '' and section[
'section_end_time'] != '' and len(section['track_points']) >= 2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \
max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] = 'not a trip'
elif section['section_start_time'] != '' and section[
'section_end_time'] != '' and len(section['track_points']) < 2:
if travel_time(section['section_start_time'],
section['section_end_time']) >= minimum_travel_time:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] = 'not a trip'
elif (section['section_start_time'] == '' or section['section_end_time']
== '') and len(section['track_points']) >= 2:
if max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] = 'not a trip'
else:
section['type'] = 'not complete information'
section['retained'] = is_retained
def setSectionClassification(uuid, userClassifications):
number_class_sec = len(userClassifications)
user_uuid = uuid
Sections = get_section_db()
Modes = get_mode_db()
logging.debug("userClassifications = %s" % userClassifications)
logging.debug("number_class_sec = %s" % number_class_sec)
if number_class_sec != 0:
logging.debug("number_class_sec = %s" % number_class_sec)
for sectionindex in range(number_class_sec):
if userClassifications[sectionindex]['userMode'] == 'not a trip':
logging.debug("usermode = %s" %
userClassifications[sectionindex]['userMode'])
Sections.update(
{
"$and": [{
'source': 'Shankari'
}, {
'user_id': user_uuid
}, {
'trip_id':
userClassifications[sectionindex]['trip_id']
}, {
'section_id':
int(userClassifications[sectionindex]
['section_id'])
}]
}, {
"$set": {
'type':
userClassifications[sectionindex]['userMode']
}
})
logging.debug("update done")
else:
Sections.update(
{
"$and": [{
'source': 'Shankari'
}, {
'user_id': user_uuid
}, {
'trip_id':
userClassifications[sectionindex]['trip_id']
}, {
'section_id':
int(userClassifications[sectionindex]
['section_id'])
}]
}, {
"$set": {
'confirmed_mode':
int(''.join(
map(str, [
mode['mode_id'] for mode in Modes.find({
'mode_name':
userClassifications[sectionindex]
['userMode']
})
])))
if Modes.find({
'mode_name':
userClassifications[sectionindex]['userMode']
}).count() != 0 else
userClassifications[sectionindex]['userMode']
}
})
logging.debug("update done")
__author__ = 'Yin'
from pymongo import MongoClient
from get_database import get_mode_db, get_section_db, get_trip_db
Modes = get_mode_db()
for items in Modes.find():
Modes.remove()
modes_todo = {'mode_id': 1, 'mode_name': 'walking'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 2, 'mode_name': 'running'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 3, 'mode_name': 'cycling'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 4, 'mode_name': 'transport'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 5, 'mode_name': 'bus'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 6, 'mode_name': 'train'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 7, 'mode_name': 'car'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 8, 'mode_name': 'mixed'}
Modes.insert(modes_todo)
modes_todo = {'mode_id': 9, 'mode_name': 'air'}
Modes.insert(modes_todo)
def processTripArray(user_uuid, trip_array):
# First, we open a connection to the database
Stage_Trips = get_trip_db()
Stage_Sections = get_section_db()
Modes = get_mode_db()
number_of_trips = len(trip_array)
logging.info("number of trips = %s" % number_of_trips)
for trip in range(number_of_trips):
seg_note = trip_array[trip]
trip_id = seg_note["startTime"]
_id_trip = str(user_uuid) + '_' + seg_note["startTime"]
#logging.debug(json.dumps(seg_note))
if "activities" in seg_note:
number_of_sections = len(seg_note["activities"])
logging.debug("number of sections = %s" % number_of_sections)
for sectionindex in range(number_of_sections):
seg_act_note = seg_note["activities"][sectionindex]
# if the section is missing some data that we access later, then we skip it
if Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}, {
"section_id": sectionindex
}]
}).count() == 0:
try:
_id_section = str(user_uuid) + '_' + seg_act_note[
"startTime"] + '_' + str(sectionindex)
_mode = convertModeNameToIndex(
Modes, seg_act_note["activity"])
isManual = seg_act_note[
"manual"] if "manual" in seg_act_note else False
sections_todo = {
'source': 'Shankari',
'_id': _id_section,
'user_id': user_uuid,
'trip_id': trip_id,
'type': seg_note["type"],
'section_id': sectionindex,
'mode': _mode,
# SHANKARI: what does seg_act_note["manual"] mean?
'confirmed_mode': _mode if isManual else '',
# 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})])))
# if "group" in seg_act_note else '',
}
fillSectionWithMovesData(seg_act_note, sections_todo)
# Now that we have created this section, let's insert it into the database
try:
logging.info(
"About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s "
% (trip_id, sectionindex,
sections_todo['section_start_time'],
seg_note["type"], seg_act_note["activity"]))
Stage_Sections.insert(sections_todo)
except DuplicateKeyError:
logging.warning(
"DuplicateKeyError, skipping insert %s" %
sections_todo)
logging.warning(
"Existing section is %s" %
Stage_Sections.find_one({"_id": _id_section}))
except KeyError, e:
logging.warning(
"Missing key %s, skipping section insert %s" %
(e, seg_act_note))
insertedSectionCount = Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}, {
"section_id": sectionindex
}]
}).count()
if insertedSectionCount == 0:
logging.error(
"Insert appears to have FAILED. No entry for %s, %s, %s found"
% (user_uuid, trip_id, sectionindex))
else:
logging.debug(
"Found existing matching entry for %s, %s, %s, skipping entry"
% (user_uuid, trip_id, sectionindex))
# Insert a trip if it doesn't already exist
# SHANKARI: What if we get other sections for a trip later? When do we update the trip?
# Do we even need to keep this linkage, with the concomittant
# management cost if we can just find all sections by trip_id
# instead? How expensive is the query?
if Stage_Trips.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}]
}).count() == 0:
trips_todo = {
'source':
'Shankari',
'_id':
_id_trip,
'user_id':
user_uuid,
'trip_id':
trip_id,
'sections': [
sections['section_id']
for sections in Stage_Sections.find({
"$and": [{
"user_id": user_uuid
}, {
"trip_id": trip_id
}]
})
]
}
fillTripWithMovesData(seg_note, trips_todo)
logging.info("About to insert trip with trip_id = %s " % (trip_id))
Stage_Trips.insert(trips_todo)
else:
logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def processTripArray(user_uuid, trip_array):
# First, we open a connection to the database
Stage_Trips=get_trip_db()
Stage_Sections=get_section_db()
Modes=get_mode_db()
number_of_trips=len(trip_array)
logging.info("number of trips = %s" % number_of_trips)
for trip in range(number_of_trips):
seg_note=trip_array[trip]
trip_id=seg_note["startTime"]
_id_trip=str(user_uuid)+'_'+seg_note["startTime"]
#logging.debug(json.dumps(seg_note))
if "activities" in seg_note:
number_of_sections=len(seg_note["activities"])
logging.debug("number of sections = %s" % number_of_sections)
for sectionindex in range(number_of_sections):
seg_act_note=seg_note["activities"][sectionindex]
# if the section is missing some data that we access later, then we skip it
if Stage_Sections.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id},{"section_id": sectionindex}]}).count()==0:
try:
_id_section = str(user_uuid)+'_'+seg_act_note["startTime"]+'_'+str(sectionindex)
_mode = convertModeNameToIndex(Modes, seg_act_note["activity"])
isManual = seg_act_note["manual"] if "manual" in seg_act_note else False
sections_todo={'source':'Shankari',
'_id':_id_section,
'user_id': user_uuid,
'trip_id':trip_id,
'type':seg_note["type"],
'section_id':sectionindex,
'mode' : _mode,
# SHANKARI: what does seg_act_note["manual"] mean?
'confirmed_mode' :_mode if isManual else '',
# 'group':int(''.join(map(str, [group['group_id'] for group in Groups.find({'group_name':seg_act_note["group"]})])))
# if "group" in seg_act_note else '',
}
fillSectionWithMovesData(seg_act_note, sections_todo)
label_filtered_section(sections_todo)
# Now that we have created this section, let's insert it into the database
try:
logging.info("About to insert section with trip_id = %s,p section_id = %s, section_start_time = %s, type = %s and mode = %s " %
(trip_id, sectionindex, sections_todo['section_start_time'], seg_note["type"], seg_act_note["activity"]))
Stage_Sections.insert(sections_todo)
except DuplicateKeyError:
logging.warning("DuplicateKeyError, skipping insert %s" % sections_todo)
logging.warning("Existing section is %s" % Stage_Sections.find_one({"_id": _id_section}))
except KeyError, e:
logging.warning("Missing key %s, skipping section insert %s" % (e, seg_act_note))
insertedSectionCount = Stage_Sections.find({"$and" : [{"user_id": user_uuid},
{"trip_id": trip_id},
{"section_id": sectionindex}]}).count()
if insertedSectionCount == 0:
logging.error("Insert appears to have FAILED. No entry for %s, %s, %s found" %
(user_uuid, trip_id, sectionindex))
else:
logging.debug("Found existing matching entry for %s, %s, %s, skipping entry" %
(user_uuid, trip_id, sectionindex))
# Insert a trip if it doesn't already exist
# SHANKARI: What if we get other sections for a trip later? When do we update the trip?
# Do we even need to keep this linkage, with the concomittant
# management cost if we can just find all sections by trip_id
# instead? How expensive is the query?
if Stage_Trips.find({"$and":[ {"user_id":user_uuid},{"trip_id": trip_id}]}).count()==0:
trips_todo={ 'source':'Shankari',
'_id':_id_trip,
'user_id': user_uuid,
'trip_id':trip_id,
'sections':[sections['section_id'] for sections in Stage_Sections.find({"$and":[{"user_id":user_uuid}, {"trip_id":trip_id}]})]}
fillTripWithMovesData(seg_note, trips_todo)
logging.info("About to insert trip with trip_id = %s " % (trip_id))
Stage_Trips.insert(trips_todo)
else:
logging.debug("Found existing trip with trip_id = %s " % (trip_id))
def setUp(self):
import tests.common
from copy import copy
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
self.setupUserAndClient()
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.airCarbon = 217.0 / 1609
self.driveCarbon = 278.0 / 1609
self.busOptimalCarbon = 92.0 / 1609
self.allDriveExpect = (self.busExpect * self.driveCarbon +
self.walkExpect * self.driveCarbon) / 1000
self.myFootprintExpect = (self.busExpect * self.busCarbon) / 1000
self.sb375GoalExpect = 40.142892 / 7
self.mineMinusOptimalExpect = 0
self.allDriveMinusMineExpect = (
self.allDriveExpect - self.myFootprintExpect) / self.allDriveExpect
self.sb375DailyGoalMinusMineExpect = (
self.sb375GoalExpect -
self.myFootprintExpect) / self.sb375GoalExpect
self.now = datetime.now()
self.twodaysago = self.now - timedelta(days=2)
self.weekago = self.now - timedelta(weeks=1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.twodaysago
section['section_end_datetime'] = self.twodaysago + timedelta(
hours=1)
section['predicted_mode'] = {'walking': 1.0}
if section['user_id'] == '*****@*****.**':
logging.debug("Setting user_id for section %s, %s = %s" %
(section['trip_id'], section['section_id'],
self.user.uuid))
section['user_id'] = self.user.uuid
if section['confirmed_mode'] == 5:
airSection = copy(section)
airSection['confirmed_mode'] = 9
airSection['_id'] = section['_id'] + "_air"
self.SectionsColl.insert(airSection)
airSection['confirmed_mode'] = ''
airSection['_id'] = section['_id'] + "_unconf"
self.SectionsColl.insert(airSection)
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
self.SectionsColl.save(section)
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import *
__author__ = 'Yin'
from pymongo import MongoClient
from get_database import get_mode_db, get_section_db, get_trip_db
Modes=get_mode_db()
for items in Modes.find():
Modes.remove()
modes_todo={ 'mode_id': 1,'mode_name':'walking'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 2,'mode_name':'running'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 3,'mode_name':'cycling'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 4,'mode_name':'transport'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 5,'mode_name':'bus'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 6,'mode_name':'train'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 7,'mode_name':'car'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 8,'mode_name':'mixed'}
Modes.insert(modes_todo)
modes_todo={ 'mode_id': 9,'mode_name':'air'}
def filter_unclassifiedSections(UnclassifiedSections):
minimum_travel_time = 120
minimum_travel_distance = 200
Modes = get_mode_db()
Sections = get_section_db()
filtered_Sections = []
for section in UnclassifiedSections:
# logging.debug("Appending %s" % json.dumps(section))
if section['section_start_time'] != '' and section[
'section_end_time'] != '' and len(
section['track_points']) >= 2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \
max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
filtered_Sections.append(section)
else:
Sections.update(
{
"$and": [{
'source': 'Shankari'
}, {
'user_id': section['user_id']
}, {
'trip_id': section['trip_id']
}, {
'section_id': section['section_id']
}]
}, {"$set": {
'type': 'not a trip'
}})
elif section['section_start_time'] != '' and section[
'section_end_time'] != '' and len(section['track_points']) < 2:
if travel_time(section['section_start_time'],
section['section_end_time']) >= minimum_travel_time:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
filtered_Sections.append(section)
else:
Sections.update(
{
"$and": [{
'source': 'Shankari'
}, {
'user_id': section['user_id']
}, {
'trip_id': section['trip_id']
}, {
'section_id': section['section_id']
}]
}, {"$set": {
'type': 'not a trip'
}})
elif (section['section_start_time'] == ''
or section['section_end_time']
== '') and len(section['track_points']) >= 2:
if max_Distance(
section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
filtered_Sections.append(section)
else:
Sections.update(
{
"$and": [{
'source': 'Shankari'
}, {
'user_id': section['user_id']
}, {
'trip_id': section['trip_id']
}, {
'section_id': section['section_id']
}]
}, {"$set": {
'type': 'not a trip'
}})
else:
Sections.update(
{
"$and": [{
'source': 'Shankari'
}, {
'user_id': section['user_id']
}, {
'trip_id': section['trip_id']
}, {
'section_id': section['section_id']
}]
}, {"$set": {
'type': 'not complete information'
}})
return filtered_Sections
