def testConfirmationModeQueryCorrectedManualAndAuto(self):
from dao.client import Client
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
clientSetQuery = Client(user.getFirstStudy()).clientSpecificSetters(user.uuid, dummySection, dummyPredModeMap)
# Apply the change
get_section_db().update({'_id': dummySection['_id']}, clientSetQuery)
retrievedSection = get_section_db().find_one({'_id': dummySection['_id']})
self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
get_section_db().update({'_id': dummySection['_id']}, {'$set': {'confirmed_mode': 4}})
get_section_db().update({'_id': dummySection['_id']}, {'$set': {'corrected_mode': 9}})
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(1))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(4))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(9))
for entry in retrieveByQuery:
print entry
self.assertEqual(retrieveByQuery.count(), 1)
def testClientSpecificSettersWithOverride(self):
from dao.user import User
fakeEmail = "*****@*****.**"
client = Client("testclient")
client.update(createKey = False)
common.makeValid(client)
(resultPre, resultReg) = client.preRegister("this_is_the_super_secret_id", fakeEmail)
studyList = Client.getPendingClientRegs(fakeEmail)
self.assertEqual(studyList, ["testclient"])
user = User.register("*****@*****.**")
self.assertEqual(user.getFirstStudy(), 'testclient')
dummyPredModeMap = {'walking': 1.0}
dummySection = common.createDummySection(startTime = datetime.now() - timedelta(seconds = 60 * 60),
endTime = datetime.now(),
startLoc = [-122, 34],
endLoc = [-122, 35],
predictedMode = dummyPredModeMap)
clientSetQuery = client.clientSpecificSetters(user.uuid, dummySection['_id'], dummyPredModeMap)
self.assertEqual(clientSetQuery, {'$set': {'test_auto_confirmed': {'mode': 1, 'prob': 1.0}}})
# Apply the change
get_section_db().update({'_id': dummySection['_id']}, clientSetQuery)
retrievedSection = get_section_db().find_one({'_id': dummySection['_id']})
self.assertEqual(retrievedSection['test_auto_confirmed']['mode'], 1)
def correctTrip(tripId, sectionId, correctedMode):
get_section_db().update({
'trip_id': tripId,
'section_id': sectionId
}, {'$set': {
'corrected_mode': correctedMode
}},
upsert=False)
def testConfirmationModeQueryNeither(self):
from dao.client import Client
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(1))
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(4))
self.assertEqual(retrieveByQuery.count(), 0)
def testConfirmationModeQueryManualNotAuto(self):
from dao.client import Client
(user, dummySection, dummyPredModeMap) = self.setupClientTest()
get_section_db().update({'_id': dummySection['_id']}, {'$set': {'confirmed_mode': 4}})
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(1))
self.assertEqual(retrieveByQuery.count(), 0)
retrieveByQuery = get_section_db().find(common.getConfirmationModeQuery(4))
self.assertEqual(retrieveByQuery.count(), 1)
def label_filtered_section(section):
minimum_travel_time=120
minimum_travel_distance=200
Modes=get_mode_db()
Sections=get_section_db()
is_retained = False
# logging.debug("Appending %s" % json.dumps(section))
if section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])>=2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time and \
max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
elif section['section_start_time']!=''and section['section_end_time']!=''and len(section['track_points'])<2:
if travel_time(section['section_start_time'],section['section_end_time']) >= minimum_travel_time:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
elif (section['section_start_time']==''or section['section_end_time']=='') and len(section['track_points'])>=2:
if max_Distance(section['track_points']) >= minimum_travel_distance:
section['mode']=''.join(mode['mode_name'] for mode in Modes.find({"mode_id":section['mode']})) \
if type(section['mode'])!=type('aa') else section['mode']
is_retained = True
else:
section['type'] ='not a trip'
else:
section['type'] ='not complete information'
section['retained'] = is_retained
def testRunBackgroundTasksForDay(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.SectionsColl = get_section_db()
tests.common.updateSections(self)
self.assertNotEqual(len(self.uuid_list), 0)
# Can access the zeroth element because we know that then length is greater than zero
# (see above)
test_uuid = self.uuid_list[0]
test_user = User.fromUUID(test_uuid)
self.assertNotIn('carbon_footprint', test_user.getProfile().keys())
default.runBackgroundTasks(test_user.uuid)
self.assertIn('carbon_footprint', test_user.getProfile().keys())
def get_daily_morning_commute_sections(user_id,day):
# say should be from 1 to 5
# get a list of all the sections for Mon, or ...
Sections=get_section_db()
list_of_commute=[]
candidate_sections=[]
home=detect_home(user_id)
work=detect_daily_work_office(user_id,day)
if work == 'N/A':
return []
else:
# print(list_first_pnt)
for section in Sections.find({"$and":[{"user_id": user_id},{ "section_start_point": { "$ne": None }},\
{'commute':{ "$exists": False }}]}):
if Is_date(section['section_start_time'],day):
candidate_sections.append(section)
if len(candidate_sections)>0:
candidate_sections = sorted(candidate_sections, key=lambda k: parser.parse(k['section_start_time']))
max_sec=0
for i in range(len(candidate_sections)):
if i>=max_sec:
if Is_place(candidate_sections[i]['section_start_point'],home,200):
for j in range(i,len(candidate_sections)):
if Is_place(candidate_sections[j]['section_end_point'],work,200) and \
travel_time(candidate_sections[i]['section_start_time'],\
candidate_sections[j]['section_end_time'])<=24*60*60:
sections_todo=[]
sections_todo.extend(candidate_sections[i:j+1])
list_of_commute.append(sections_todo)
max_sec=j+1
break
return list_of_commute
def plot_each_route_cluster_for_user(user_id,method='lcs'):
i=0
Sections = get_section_db()
user_route_clusters = get_routeCluster_db().find_one({'$and':[{'user':user_id},{'method':method}]})
# plot each cluster as a file.
for idx in user_route_clusters['clusters'].keys():
print idx
gmap = pygmaps.maps(37.8717, -122.2728, 14)
# gmap = pygmaps.maps(getRoute(idx)[0][0], getRoute(idx)[0][1], 14)
section=Sections.find_one({'_id': idx})
r = lambda: random.randint(0,255)
color = '#%02X%02X%02X' % (r(),r(),r())
drawSection(section, 'path', gmap,color)
print len(user_route_clusters['clusters'][idx])
first = True
for idi in user_route_clusters['clusters'][idx]:
# print(Sections.find({'_id': idi}).count())
section=Sections.find_one({'_id': idi})
if first:
print section
first = False
color = '#%02X%02X%02X' % (r(),r(),r())
drawSection(section, 'path', gmap,color)
gmap.draw(str(user_id) + '_'+ method+ '_'+str(i) + '.html')
i+=1
break
def update_profiles(dummy_users=False):
if dummy_users:
user_list = ['1']
else:
user_list = get_section_db().distinct('user_id')
for user in user_list:
generate_user_home_work(user)
def testBlankToday(self):
result = self.loadTestJSON("tests/data/test2_blank_today")
collect.processResult(self.testUUID, [result[0]])
collect.processResult(self.testUUID, [result[1]])
SectionColl = get_section_db()
self.assertTrue(SectionColl.find({'user_id': self.testUUID}).count() > 0)
def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
tests.common.updateSections(self)
def setUp(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
# self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
self.SectionsColl = get_section_db()
# self.SectionsColl.remove()
self.assertEquals(self.SectionsColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
tests.common.updateSections(self)
def regenerateData():
dec_days = pd.date_range(START_DATE, END_DATE)
uuidDF = pd.read_csv(UUID_GROUP_CSV)
for date in dec_days:
print "Generating scores for day %s" % date
for uuidStr in uuidDF.uuid:
uuid = UUID(uuidStr)
print "Generating scores for uuid %s " % uuid
choice.runBackgroundTasksForDay(uuid, date)
# Get the number of trips for a day
dateQuery = {
"section_start_datetime": {
"$gt": date,
"$lt": date + pydt.timedelta(days=1)
}
}
tripsForDay = get_section_db().find({
"$and": [{
"user_id": uuid
}, dateQuery]
}).count()
stats.storeServerEntry(uuid, stats.STAT_TRIP_MGR_TRIPS_FOR_DAY,
pytime.time(), tripsForDay)
fixClientTimestamps(date)
def setSectionClassification(uuid, userClassifications):
number_class_sec=len(userClassifications)
user_uuid=uuid
Sections=get_section_db()
Modes=get_mode_db()
logging.debug("userClassifications = %s" % userClassifications)
logging.debug("number_class_sec = %s" % number_class_sec)
if number_class_sec!=0:
logging.debug("number_class_sec = %s" % number_class_sec)
for sectionindex in range(number_class_sec):
if userClassifications[sectionindex]['userMode']=='not a trip':
logging.debug("usermode = %s" % userClassifications[sectionindex]['userMode'])
Sections.update({"$and":[{'source':'Shankari'},
{'user_id': user_uuid},
{'trip_id': userClassifications[sectionindex]['trip_id']},
{'section_id': int(userClassifications[sectionindex]['section_id'])}]},
{"$set":{'type':userClassifications[sectionindex]['userMode']}})
logging.debug("update done" )
else:
Sections.update({"$and":[{'source':'Shankari'},
{'user_id': user_uuid},
{'trip_id': userClassifications[sectionindex]['trip_id']},
{'section_id': int(userClassifications[sectionindex]['section_id'])}]},
{"$set":{'confirmed_mode':int(''.join(map(str, [mode['mode_id']
for mode in Modes.find({'mode_name':userClassifications[sectionindex]['userMode']})])))
if Modes.find({'mode_name':userClassifications[sectionindex]['userMode']}).count()!=0
else userClassifications[sectionindex]['userMode']}})
logging.debug("update done" )
def getRoute(section_id):
route = []
Sections = get_section_db()
section = Sections.find_one({'_id': section_id})
for point in section['track_points']:
route.append(point['track_location']['coordinates'])
return route
def get_clusters_info(uid):
c_db = get_routeCluster_db()
s_db = get_section_db()
clusterJson = c_db.find_one({"clusters":{"$exists":True}, "user": uid})
if clusterJson is None:
return []
c_info = []
clusterSectionLists= clusterJson["clusters"].values()
logging.debug( "Number of section lists for user %s is %s" % (uid, len(clusterSectionLists)))
for sectionList in clusterSectionLists:
first = True
logging.debug( "Number of sections in sectionList for user %s is %s" % (uid, len(sectionList)))
if (len(sectionList) == 0):
# There's no point in returning this cluster, let's move on
continue
distributionArrays = [[] for _ in range(5)]
for section in sectionList:
section_json = s_db.find_one({"_id":section})
if first:
representative_trip = section_json
first = False
appendIfPresent(distributionArrays[0], section_json, "section_start_datetime")
appendIfPresent(distributionArrays[1], section_json, "section_end_datetime")
appendIfPresent(distributionArrays[2], section_json, "section_start_point")
appendIfPresent(distributionArrays[3], section_json, "section_end_point")
appendIfPresent(distributionArrays[4], section_json, "confirmed_mode")
c_info.append((distributionArrays, representative_trip))
return c_info
def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
original_trip = get_trip_db().find_one({'user_id': user_uuid, 'recommended_alternative': {'$exists': True}})
if original_trip is None:
return template("clients/recommendation/no_recommendation.html")
del original_trip['trip_start_datetime']
del original_trip['trip_end_datetime']
del original_trip['user_id']
del original_trip['pipelineFlags']
del original_trip['recommended_alternative']['user_id']
recommended_trip = original_trip['recommended_alternative']
original_sections = list(get_section_db().find({'trip_id': original_trip['trip_id']}))
for section in original_sections:
del section['user_id']
del section['section_start_datetime']
del section['section_end_datetime']
if 'retained' in section:
del section['retained']
del section['manual']
del section['commute']
renderedTemplate = template("clients/recommendation/result_template.html",
originalSections = ast.literal_eval(json.dumps(original_sections)),
recommendedTrip = ast.literal_eval(json.dumps(recommended_trip)))
return renderedTemplate
def testPlaceLoad(self):
result = self.loadTestJSON("tests/data/test20140410")
collect.processResult(self.testUUID, result)
# Check that the trips are loaded correctly
TripColl = get_trip_db()
firstStoredTrip = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140409T191531-0700'}]})
logging.debug("selected trip = %s" % firstStoredTrip)
# For some reason, the place is always "unknown", at least for this set of test trips.
# Maybe it is related to the fact that they haven't been tagged in FourSquare
self.assertEqual(firstStoredTrip['type'], 'place')
self.assertEqual(firstStoredTrip['trip_start_time'], '20140409T191531-0700')
self.assertEqual(firstStoredTrip['trip_end_time'], "20140410T065227-0700")
self.assertIn('place_location', firstStoredTrip['place'])
self.assertEqual(firstStoredTrip['place']['place_location'], {'type': 'Point',
'coordinates': [-122.08632, 37.391]})
# Now, check that we have the sections as well. The previous trip did not
# have any sections. This one does
tripWithSections = TripColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700'}]})
self.assertNotEqual(tripWithSections, None)
self.assertEqual(tripWithSections['sections'], [0])
SectionColl = get_section_db()
sectionForTrip = SectionColl.find_one({'$and': [{'user_id': self.testUUID,
'trip_id': '20140410T071320-0700',
'section_id': 0}]})
self.assertNotEqual(sectionForTrip, None)
def setUp(self):
import tests.common
from copy import copy
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
self.setupUserAndClient()
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections", "tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.airCarbon = 217.0/1609
self.driveCarbon = 278.0/1609
self.busOptimalCarbon = 92.0/1609
self.allDriveExpect = (self.busExpect * self.driveCarbon + self.walkExpect * self.driveCarbon)/1000
self.myFootprintExpect = float(self.busExpect * self.busCarbon)/1000
self.sb375GoalExpect = 40.142892/7
self.mineMinusOptimalExpect = 0
self.allDriveMinusMineExpect = float(self.allDriveExpect - self.myFootprintExpect)/self.allDriveExpect
self.sb375DailyGoalMinusMineExpect = float(self.sb375GoalExpect - self.myFootprintExpect)/self.sb375GoalExpect
self.now = datetime.now()
self.twodaysago = self.now - timedelta(days=2)
self.weekago = self.now - timedelta(weeks = 1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.twodaysago
section['section_end_datetime'] = self.twodaysago + timedelta(hours = 1)
section['predicted_mode'] = {'walking': 1.0}
if section['user_id'] == '*****@*****.**':
logging.debug("Setting user_id for section %s, %s = %s" %
(section['trip_id'], section['section_id'], self.user.uuid))
section['user_id'] = self.user.uuid
if section['confirmed_mode'] == 5:
airSection = copy(section)
airSection['confirmed_mode'] = 9
airSection['_id'] = section['_id'] + "_air"
self.SectionsColl.insert(airSection)
airSection['confirmed_mode'] = ''
airSection['_id'] = section['_id'] + "_unconf"
self.SectionsColl.insert(airSection)
# print("Section start = %s, section end = %s" %
# (section['section_start_datetime'], section['section_end_datetime']))
self.SectionsColl.save(section)
def createDummySection(startTime, endTime, startLoc, endLoc, predictedMode = None, confirmedMode = None):
from get_database import get_section_db
section = {'section_start_datetime': startTime,
'section_end_datetime': endTime,
'section_start_time': startTime.isoformat(),
'section_end_time': endTime.isoformat(),
'section_start_point': {'type': 'Point', 'coordinates': startLoc},
'section_end_point': {'type': 'Point', 'coordinates': endLoc},
}
if predictedMode != None:
section['predicted_mode'] = predictedMode
if confirmedMode != None:
section['confirmed_mode'] = confirmedMode
get_section_db().insert(section)
return section
def testRetrieveTripsToImproveWithClusters(self):
sectionList = list(get_section_db().find())
get_routeCluster_db().insert({"user": self.testUUID,
"clusters":
{"cluster1": [s["_id"] for s in sectionList[0:10]],
"cluster2": [s["_id"] for s in sectionList[10:20]]}})
trip_list = list(self.pipeline.get_trips_to_improve(self.testUUID))
self.assertEquals(len(trip_list), 2)
def Berkeley_pop_route(start,end):
Sections = get_section_db()
list_of_point=[]
# print(berkeley_area())
for section in Sections.find({"$and":[{'In_UCB':True },{'type':'move'},{"section_start_datetime": {"$gte": start, "$lt": end}}]}):
for pnt in section['track_points']:
list_of_point.append(pnt['track_location']['coordinates'])
return {"latlng": list_of_point}
def Berkeley_pop_route(start,end):
Sections = get_section_db()
list_of_point=[]
# print(berkeley_area())
for section in Sections.find({"$and":[{'In_UCB':True },{'type':'move'},{"section_start_datetime": {"$gte": start, "$lt": end}}]}):
for pnt in section['track_points']:
list_of_point.append(pnt['track_location']['coordinates'])
return {"latlng": list_of_point}
def loadData(serverName, fileName):
Sections = get_section_db()
dataJSON = json.load(open(fileName))
for section in dataJSON:
try:
Sections.insert(section)
except DuplicateKeyError:
print "Duplicate key found while loading, skipping..."
def display_trip(user, date, option):
user_id = UUID(user)
user_home = detect_home(user_id)
gmap = pygmaps.maps(user_home[1], user_home[0], 14)
start, end = Date(date)
for section in get_section_db().find({"$and":[{'user_id':user_id},{"section_start_datetime": {"$gte": start, "$lt": end}}]}):
drawSection(section, option, gmap)
gmap.draw('gmap_display/' + str(start)[:10] + '_' + user + '.html')
def plot_common_routes_for_user(user_id, method='lcs'):
Sections = get_section_db()
idxs = get_common_routes_for_user(user_id, method)
gmap = pygmaps.maps(37.8717, -122.2728, 14)
for idx in idxs:
section = Sections.find_one({'_id': idx})
drawSection(section, 'path', gmap)
gmap.draw(str(user_id) + '_' + method + '.html')
def loadData(serverName, fileName):
Sections=get_section_db()
dataJSON = json.load(open(fileName))
for section in dataJSON:
try:
Sections.insert(section)
except DuplicateKeyError:
print "Duplicate key found while loading, skipping..."
def updateExistence():
Sections = get_section_db()
for tempSec in Sections:
if 'retained' not in list(tempSec.keys()) or tempSec['retained'] is None:
label_filtered_section(tempSec)
else:
#don't update
pass
def __init__(self, user_uuid, queryList=[]):
# Query section_db with queryList and
# instantiate the cursor object
# returned as an instance variable
clientSpecificQuery = getClientSpecificQueryFilter(user_uuid)
completeQuery = clientSpecificQuery + queryList
Sections = get_section_db()
self.cur = Sections.find({"and": completeQuery})
def get_user_evening_commute_sections(user):
# say should be from 1 to 5
# get a list of all the sections for Mon, or ...
Sections=get_section_db()
list_of_commute=[]
for date in range(1,6):
list_of_commute.extend(get_daily_evening_commute_sections(user,date))
return list_of_commute
def get_user_sections(user_id):
sections = list(get_section_db().find(
{'$and': [{
'user_id': user_id
}, {
'type': 'move'
}]}))
return sections
def updateExistence():
Sections = get_section_db()
for tempSec in Sections:
if 'retained' not in list(
tempSec.keys()) or tempSec['retained'] is None:
label_filtered_section(tempSec)
else:
#don't update
pass
def testYesterdayForJustSignedUp(self):
result = self.loadTestJSON("tests/data/yesterday_for_justsignedup")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
self.assertEquals(
SectionColl.find({
"user_id": self.testUUID
}).count(), 0)
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
get_section_db().remove({"user_id": self.testUUID})
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
print get_section_db().find().count()
self.pipeline = RecommendationPipeline()
def get_berkeley_sections():
Sections = get_section_db()
list_of_sections=[]
for section in Sections.find():
if section['section_start_point']!=None and section['section_end_point']!=None and \
Inside_polygon(section['section_start_point'],berkeley_area()) and \
Inside_polygon(section['section_end_point'],berkeley_area()):
list_of_sections.append(section)
return list_of_sections
def setUp(self):
self.testUUID = "myuuidisverylongandcomplicated"
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
self.ModesColl = get_mode_db()
self.ModesColl.remove()
self.assertEquals(self.ModesColl.find().count(), 0)
dataJSON = json.load(open("tests/data/modes.json"))
for row in dataJSON:
self.ModesColl.insert(row)
get_section_db().remove({"user_id": self.testUUID})
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
print get_section_db().find().count()
self.pipeline = RecommendationPipeline()
def createDummySection(startTime, endTime, startLoc, endLoc, predictedMode = None, confirmedMode = None):
from get_database import get_section_db
section = {'section_start_datetime': startTime,
'section_end_datetime': endTime,
'section_start_time': startTime.isoformat(),
'section_end_time': endTime.isoformat(),
'section_start_point': {'type': 'Point', 'coordinates': startLoc},
'section_end_point': {'type': 'Point', 'coordinates': endLoc},
}
if predictedMode != None:
section['predicted_mode'] = predictedMode
if confirmedMode != None:
section['confirmed_mode'] = confirmedMode
get_section_db().insert(section)
return section
def display_home():
mymap_home = pygmaps.maps(37.8556475757, -122.248774009,14)
for user in get_section_db().distinct('user_id'):
print(user)
user_home=detect_home(user)
print(user_home)
if user_home!='N/A':
mymap_home.addpoint(user_home[1], user_home[0], "#FF0000")
mymap_home.draw('mymap_home.html')
def setUp(self):
import tests.common
from copy import copy
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.ModesColl = get_mode_db()
self.assertEquals(self.ModesColl.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Modes",
"tests/data/modes.json")
load_database_json.loadTable(self.serverName, "Stage_Sections",
"tests/data/testCarbonFile")
self.SectionsColl = get_section_db()
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0 / 1609
self.airCarbon = 217.0 / 1609
self.driveCarbon = 278.0 / 1609
self.busOptimalCarbon = 92.0 / 1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
for section in self.SectionsColl.find():
section['section_start_datetime'] = self.dayago
section['section_end_datetime'] = self.dayago + timedelta(hours=1)
# Note that we currently only search by the date/time of the section, not the points.
# If we ever change that, this test will start failing and will need to be fixed as well
track_pt_array = []
for i in range(10):
track_pt_array.append({
'time': '20140829T170451-0700',
'track_location': {
'coordinates': [-122.114642519, 37.4021455446],
'type': 'Point'
}
})
track_pt_array.append({
'time': '20140829T170620-0700',
'track_location': {
'coordinates': [-122.1099155383, 37.399523614],
'type': 'Point'
}
})
section['track_points'] = track_pt_array
self.SectionsColl.save(section)
def getResult(user_uuid):
# This is in here, as opposed to the top level as recommended by the PEP
# because then we don't have to worry about loading bottle in the unit tests
from bottle import template
logging.debug("carshare.getResult called for user %s!" % user_uuid)
tripCount = get_section_db().find({"$and": [{'user_id': user_uuid}, {'type': 'move'}, {'confirmed_mode': {'$ne': ''}}]}).count()
renderedTemplate = template("clients/carshare/result_template.html",
count = tripCount)
return renderedTemplate
def confirmCorrection(tripId, sectionId, userId, correctedMode):
entry = get_section_db().find_one(
{'$and': [{
'trip_id': tripId,
'section_id': sectionId
}]})
print("Checking against entry %s" % entry)
assert (entry['user_id'] == userId)
assert (entry['corrected_mode'] == correctedMode)
assert (entry['confirmed_mode'] != correctedMode)
def testBlankToday(self):
result = self.loadTestJSON("tests/data/test2_blank_today")
collect.processResult(self.testUUID, [result[0]])
collect.processResult(self.testUUID, [result[1]])
SectionColl = get_section_db()
self.assertTrue(
SectionColl.find({
'user_id': self.testUUID
}).count() > 0)
def plot_common_routes_for_user(user_id,method='lcs'):
Sections = get_section_db()
idxs = get_common_routes_for_user(user_id,method)
gmap = pygmaps.maps(37.8717, -122.2728, 14)
for idx in idxs:
r = lambda: random.randint(0,255)
color = '#%02X%02X%02X' % (r(),r(),r())
section = Sections.find_one({'_id': idx})
print section
drawSection(section,'path',gmap,color)
gmap.draw(str(user_id) + '_'+ method + '.html')
def get_user_clusters(user, method, nClusters, is_ground_truth=False):
if is_ground_truth:
routes_user = user_route_data2(user)
else:
routes_user = user_route_data(user,get_section_db())
if nClusters == -1:
nClusters = int(math.ceil(len(routes_user)/8) + 1)
clusters_user = kmedoids(routes_user,nClusters,user,method=method)
#update_user_routeClusters(user,clusters_user[2],method=method)
return clusters_user
def testMissingSections(self):
result = self.loadTestJSON("tests/data/missing_trip")
collect.processResult(self.testUUID, result)
SectionColl = get_section_db()
storedSections = SectionColl.find({'user_id': self.testUUID})
self.assertEquals(storedSections.count(), 21)
tripToWorkSections = SectionColl.find({'$and' : [{'user_id': self.testUUID,
'trip_id': '20140407T085210-0700'}]})
self.assertEquals(tripToWorkSections.count(), 5)
def cluster_to_kml(user, cluster, cluster_id):
"""
Creates a single, or possibly multiple KML files a given cluster.
A KML file is limited by MyMaps to having only 10 layers, so only
10 sections will be in a given KML file.
Responsibilty of caller to check existence and formatting of cluster
"""
Sections = get_section_db()
for i,chunk in enumerate(chunks(cluster,10)):
sections = map(lambda section_id: Sections.find_one({'_id':section_id}), chunk)
sections_to_kml("%s_cluster_data_kml/CLUSTER_%s_%i" % (user, str(cluster_id), i), sections)
def testPartialLoadUpdate(self):
resultAll = self.loadTestJSON("tests/data/missing_trip")
resultSubset = self.loadTestJSON("tests/data/missing_trip_subset")
collect.processResult(self.testUUID, resultSubset)
SectionColl = get_section_db()
tripToWorkSections = SectionColl.find({'$and' : [{'user_id': self.testUUID,
'trip_id': '20140407T085210-0700'}]})
self.assertEquals(tripToWorkSections.count(), 3)
collect.processResult(self.testUUID, resultAll)
tripToWorkSections = SectionColl.find({'$and' : [{'user_id': self.testUUID,
'trip_id': '20140407T085210-0700'}]})
self.assertEquals(tripToWorkSections.count(), 5)
def test(self):
workDetection = pygmaps.maps(37.8656475757, -122.258774009,14)
for user in get_section_db().distinct('user_id'):
user_work=work_place.detect_work_office(user)
print user
if user_work == 'N/A' or '':
print '--- USER WORK ADDRESS UNAVAILABLE ---'
else:
print(Geocoder.reverse_geocode(user_work[0], user_work[1])[0])
workDetection.addpoint(user_work[0], user_work[1], "#FF0000")
print '...'
if not os.path.exists('gmap_display'):
os.makedirs('gmap_display')
workDetection.draw('gmap_display/workDetection.html')
def test(self):
homeDetection = pygmaps.maps(37.8656475757, -122.258774009,14)
for user in get_section_db().distinct('user_id'):
user_home=home.detect_home(user)
print user
if user_home == 'N/A' or '':
print '--- USER HOME ADDRESS UNAVAILABLE ---'
else:
print(Geocoder.reverse_geocode(user_home[0], user_home[1])[0])
homeDetection.addpoint(user_home[0], user_home[1], "#FF0000")
print '...'
if not os.path.exists('gmap_display'):
os.makedirs('gmap_display')
homeDetection.draw('gmap_display/homeDetection.html')
def Commute_pop_route(modeId,start,end):
Sections = get_section_db()
list_of_point=[]
# print(berkeley_area())
commuteQuery = {"$or": [{'commute': 'to'}, {'commute': 'from'}]}
modeQuery = {"$or": [{'mode': modeId}, getConfirmationModeQuery(modeId)]}
dateTimeQuery = {"section_start_datetime": {"$gte": start, "$lt": end}}
for section in Sections.find({"$and":[modeQuery,dateTimeQuery,{'type':'move'}]}):
if len(section['track_points']) > 5:
# skip routes that have less than 3 points
for pnt in section['track_points'][5:-5]:
list_of_point.append(pnt['track_location']['coordinates'])
return {"latlng": list_of_point}
def get_user_disMat(user, method, is_ground_truth=False):
## update route clusters:
print "Generating route clusters for %s" % user
if is_ground_truth:
cluster_section_ids = get_ground_truth_sections(user)
routes_user = user_route_data2(cluster_section_ids)
user_disMat = update_user_routeDistanceMatrix(str(user) + '_ground_truth',routes_user,step1=100000,step2=100000,method=method)
else:
routes_user = user_route_data(user,get_section_db())
#print(routes_user)
user_disMat = update_user_routeDistanceMatrix(user,routes_user,step1=100000,step2=100000,method=method)
print(type(user_disMat))
return user_disMat
def regenerateData():
dec_days = pd.date_range(START_DATE, END_DATE)
uuidDF = pd.read_csv(UUID_GROUP_CSV)
for date in dec_days:
print "Generating scores for day %s" % date
for uuidStr in uuidDF.uuid:
uuid = UUID(uuidStr)
print "Generating scores for uuid %s " % uuid
choice.runBackgroundTasksForDay(uuid, date)
# Get the number of trips for a day
dateQuery = {"section_start_datetime": {"$gt": date, "$lt": date + pydt.timedelta(days=1)}}
tripsForDay = get_section_db().find({"$and": [{"user_id" : uuid}, dateQuery]}).count()
stats.storeServerEntry(uuid, stats.STAT_TRIP_MGR_TRIPS_FOR_DAY, pytime.time(), tripsForDay)
fixClientTimestamps(date)
