def setUp(self):
from get_database import get_profile_db, get_uuid_db, get_client_db, get_pending_signup_db
# Make sure we start with a clean slate every time
get_client_db().remove()
get_profile_db().remove()
get_pending_signup_db().remove()
get_uuid_db().remove()
def setUp(self): from get_database import get_profile_db, get_uuid_db, get_client_db, get_pending_signup_db # Make sure we start with a clean slate every time get_client_db().remove() get_profile_db().remove() get_pending_signup_db().remove() get_uuid_db().remove()
def setMpgArray(self, mpg_array):
logging.debug("Setting MPG array for user %s to : %s" %
(self.uuid, mpg_array))
get_profile_db().update({'user_id': self.uuid},
{'$set': {
'mpg_array': mpg_array
}})
def setUp(self):
# Make sure we start with a clean slate every time
self.serverName = 'localhost'
common.dropAllCollections(get_db())
logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" %
(get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count()))
load_database_json.loadTable(self.serverName, "Stage_Modes", "tests/data/modes.json")
def get_Alluser_work_end_time():
list_of_time=[]
Profiles=get_profile_db()
for user in Profiles.distinct("user_id"):
for day in range(1,6):
list_of_time.extend(get_work_end_time(user,day))
return list_of_time
def detect_daily_work_office_from_db(user_id, day):
Profiles = get_profile_db()
user_pro = Profiles.find_one(
{"$and": [{
'source': 'Shankari'
}, {
'user_id': user_id
}]})
return user_pro['work' + str(day)]
def detect_home_from_db_2(user_id):
Profiles = get_profile_db()
user_pro = Profiles.find_one(
{'$and': [{
'source': 'Shankari'
}, {
'user_id': user_id
}]})
return user_pro['home']
def createProfile(uuid, ts, studyList):
initProfileObj = {'user_id': uuid,
'source':'Shankari',
'update_ts': ts,
'mpg_array': [defaultMpg]}
writeResultProfile = get_profile_db().update(
{'user_id': uuid},
{'$set': initProfileObj,
'$addToSet': {'study_list': {'$each': studyList}}},
upsert=True)
return writeResultProfile
def unsetStudy(self, study):
# Here's what we want to do:
# - if there is no profile entry, ignore
# - if there is a profile entry and the study_list does not contain this study, ignore
# - if there is a profile entry and the study_list contains the entry, remove it
# The following mongodb statement is supposed to handle all of those cases correctly :)
writeResult = get_profile_db().update({'user_id': self.uuid},
{'$pullAll': {'study_list': [study]}})
if 'err' in writeResult and writeResult['err'] is not None:
logging.error("In setStudy, err = %s" % writeResult['err'])
raise Exception()
def detect_home_from_db(user_id):
Profiles = get_profile_db()
user_pro = Profiles.find_one(
{"$and": [{
'source': 'Shankari'
}, {
'user_id': user_id
}]})
if Profiles.find({'user_id': user_id}).count() == 0:
return 'N/A'
return user_pro['home']
def get_userZipcode(user, valid_zip):
location = detect_home_from_db(user)
current_db = get_profile_db()
user_pro = current_db.find_one({"$and":[{'source':'Shankari'},{'user_id':user}]})
if valid_zip and user_pro.get('zip'):
return user_pro['zip']
elif location!='N/A':
# Convert from our internal GeoJSON specific (lng, lat) to the (lat, lng)
# format required by geocoder
return _geocodeZipcode(user_pro, location)
else:
return 'N/A'
def getZipcode():
Profiles = get_profile_db()
zips = Profiles.distinct("zip")
# print(users)
userZipcodes = []
ZipDictCount = {}
ZipDictUser = defaultdict(list)
for zip in zips:
if zip != 'N/A':
ZipDictCount[zip] = Profiles.find({'zip': zip}).count()
# print(set(userZipcodes))
# print(list(set(userZipcodes)))
return ZipDictCount
def getZipcode():
Profiles=get_profile_db()
zips = Profiles.distinct("zip")
# print(users)
userZipcodes = []
ZipDictCount={}
ZipDictUser = defaultdict(list)
for zip in zips:
if zip!='N/A':
ZipDictCount[zip] = Profiles.find({'zip':zip}).count()
# print(set(userZipcodes))
# print(list(set(userZipcodes)))
return ZipDictCount
def unsetStudy(self, study):
# Here's what we want to do:
# - if there is no profile entry, ignore
# - if there is a profile entry and the study_list does not contain this study, ignore
# - if there is a profile entry and the study_list contains the entry, remove it
# The following mongodb statement is supposed to handle all of those cases correctly :)
writeResult = get_profile_db().update(
{'user_id': self.uuid}, {'$pullAll': {
'study_list': [study]
}})
if 'err' in writeResult and writeResult['err'] is not None:
logging.error("In setStudy, err = %s" % writeResult['err'])
raise Exception()
def createProfile(uuid, ts, studyList):
initProfileObj = {
'user_id': uuid,
'source': 'Shankari',
'update_ts': ts
}
writeResultProfile = get_profile_db().update({'user_id': uuid}, {
'$set': initProfileObj,
'$addToSet': {
'study_list': {
'$each': studyList
}
}
},
upsert=True)
return writeResultProfile
def carbon_by_zip(start,end):
Profiles=get_profile_db()
carbon_list=[]
for zip in Profiles.distinct('zip'):
# print(zip)
if zip!='N/A':
tempdict={}
tempdict['weight']=Profiles.find({'zip':zip}).count()
# print(Profiles.find({'zip':zip}).count())
tempdict['carbon']=0
for profile in Profiles.find({'zip':zip}):
tempdict['loc']=profile['zip_centroid']
user=profile['user_id']
user_carbon=getModeCarbonFootprint(user,carbonFootprintForMode,start,end)
tempdict['carbon']+=sum(list(user_carbon.values()))
tempdict['carbon']=tempdict['carbon']/tempdict['weight']
carbon_list.append(tempdict)
return {"weightedLoc": carbon_list}
def carbon_by_zip(start,end):
Profiles=get_profile_db()
carbon_list=[]
for zip in Profiles.distinct('zip'):
# print(zip)
if zip!='N/A':
tempdict={}
tempdict['weight']=Profiles.find({'zip':zip}).count()
# print(Profiles.find({'zip':zip}).count())
tempdict['carbon']=0
for profile in Profiles.find({'zip':zip}):
tempdict['loc']=profile['zip_centroid']
user=profile['user_id']
user_carbon=getModeCarbonFootprint(user,carbonFootprintForMode,start,end)
tempdict['carbon']+=sum(list(user_carbon.values()))
tempdict['carbon']=tempdict['carbon']/tempdict['weight']
carbon_list.append(tempdict)
return {"weightedLoc": carbon_list}
def get_mode_share_by_Zipcode(zip,flag,start,end):
Profiles=get_profile_db()
user_list=[]
Modeshare={}
logging.debug("Called get_mode_share_by_Zipcode(%s)" % zip)
for profile in Profiles.find({'zip':zip}):
user_list.append(profile['user_id'])
if len(user_list)!=0:
# print(user_list)
for user in user_list:
user_share=get_user_mode_share_by_distance(user,flag,start,end)
# print(user_share)
for mode in user_share.keys():
if mode not in Modeshare:
Modeshare[mode]=user_share[mode]
else:
Modeshare[mode]+=user_share[mode]
return Modeshare
else:
return 'N/A'
def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
tests.common.dropAllCollections(get_db())
self.Profiles = get_profile_db()
self.assertEquals(self.Profiles.find().count(), 0)
load_database_json.loadTable(self.serverName, "Stage_Profiles", "tests/data/profiles.json")
self.assertEquals(self.Profiles.find().count(), 1)
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = 267.0/1609
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
def changeUpdateTs(self, timedelta):
newTs = self.getUpdateTS() + timedelta
get_profile_db().update({'user_id': self.uuid},
{'$set': {
'update_ts': newTs
}})
def setMpgArray(self, mpg_array):
logging.debug("Setting MPG array for user %s to : %s" % (self.uuid, mpg_array))
get_profile_db().update({'user_id': self.uuid}, {'$set': {'mpg_array': mpg_array}})
def unregister(userEmail):
user = User.fromEmail(userEmail)
uuid = user.uuid
get_uuid_db().remove({'user_email': userEmail})
get_profile_db().remove({'user_id': uuid})
return uuid
def getDistinctUserCount():
Profiles=get_profile_db()
distinctUserCount = len(Profiles.distinct("user_id"))
logging.debug("Found %s distinct users " % distinctUserCount)
return distinctUserCount
def getProfile(self):
# is user_id a uuid?
return get_profile_db().find_one({'user_id': self.user_id})
# carbon footprint from the currentScore field to the carbon_footprint field,
# and delete the currentScore and previousScore fields
from get_database import get_uuid_db, get_profile_db
from dao.user import User
from clients.default import default
import logging
logging.basicConfig(level=logging.DEBUG)
for user_uuid_dict in get_uuid_db().find({}, {'uuid': 1, '_id': 0}):
currUUID = user_uuid_dict['uuid']
logging.info("Fixing results for %s" % currUUID)
currUser = User.fromUUID(currUUID)
if currUser.getFirstStudy() is None:
currFootprint = currUser.getProfile().get("currentScore", None)
default.setCarbonFootprint(currUser, currFootprint)
get_profile_db().update({'user_id': currUUID}, {'$unset': {'previousScore': "",
'currentScore': ""}})
logging.debug("After change, currentScore = %s, currFootprint = %s" % (
currUser.getProfile().get("currentScore"),
default.getCarbonFootprint(currUser)))
# Informal testing from the command line since this is a one-time script
# Can be pulled out into a unit test if reworked
# Test setup steps from the REPL:
# In [52]: userTest = User.register("*****@*****.**")
# In [53]: userTest1 = User.register("*****@*****.**")
# In [54]: gamified.setScores(userTest, None, [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}])
# In [55]: gamified.setScores(userTest1, None, [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}])
def getProfile(self):
# is user_id a uuid?
return get_profile_db().find_one({'user_id': self.user_id})
def setClientSpecificProfileFields(self, setQuery):
logging.debug("Changing profile for user %s to %s" %
(self.uuid, setQuery))
get_profile_db().update({'user_id': self.uuid}, {'$set': setQuery})
def unregister(userEmail):
user = User.fromEmail(userEmail)
uuid = user.uuid
get_uuid_db().remove({'user_email': userEmail})
get_profile_db().remove({'user_id': uuid})
return uuid
def getUsersForClient(clientName):
# Find all users for this client
client_uuids = []
for user in get_profile_db().find(getClientQuery(clientName)):
client_uuids.append(user)
def detect_daily_work_office_from_db_2(user_id, day):
Profiles = get_profile_db()
user_pro = Profiles.find_one({'$and': [{'source': 'Shankari'}, {'user_id': user_id}]})
return user_pro['work' + str(day)]
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import *
from pymongo import MongoClient
from get_database import get_uuid_db, get_moves_db, get_profile_db
from datetime import datetime
for entry in get_moves_db().find():
print("%s -> %s" % (entry['user'], entry['uuid']))
userEmail = entry['user']
userUUID = entry['uuid']
userUUIDEntry = \
{
'user_email': userEmail,
'uuid': userUUID,
'update_ts': datetime.now()
}
get_uuid_db().update({'user_email': userEmail}, userUUIDEntry, upsert=True)
profileUpdateObj = {'user_id': userUUID, 'study_list': [], 'update_ts': datetime.now()}
get_profile_db().update({'user_id': userUUID}, {'$set': profileUpdateObj}, upsert=True)
get_moves_db().update({'uuid': userUUID}, {'$set': {'our_uuid': userUUID}})
get_moves_db().update({'uuid': userUUID}, {'$unset': {'uuid': "", 'user': ""}})
from dao.user import User
from clients.default import default
import logging
logging.basicConfig(level=logging.DEBUG)
for user_uuid_dict in get_uuid_db().find({}, {'uuid': 1, '_id': 0}):
currUUID = user_uuid_dict['uuid']
logging.info("Fixing results for %s" % currUUID)
currUser = User.fromUUID(currUUID)
if currUser.getFirstStudy() is None:
currFootprint = currUser.getProfile().get("currentScore", None)
default.setCarbonFootprint(currUser, currFootprint)
get_profile_db().update(
{'user_id': currUUID},
{'$unset': {
'previousScore': "",
'currentScore': ""
}})
logging.debug("After change, currentScore = %s, currFootprint = %s" %
(currUser.getProfile().get("currentScore"),
default.getCarbonFootprint(currUser)))
# Informal testing from the command line since this is a one-time script
# Can be pulled out into a unit test if reworked
# Test setup steps from the REPL:
# In [52]: userTest = User.register("*****@*****.**")
# In [53]: userTest1 = User.register("*****@*****.**")
# In [54]: gamified.setScores(userTest, None, [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}])
# In [55]: gamified.setScores(userTest1, None, [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}])
def setClientSpecificProfileFields(self, setQuery):
logging.debug("Changing profile for user %s to %s" % (self.uuid, setQuery))
get_profile_db().update({'user_id': self.uuid}, {'$set': setQuery})
def detect_home_from_db_2(user_id):
Profiles = get_profile_db()
user_pro = Profiles.find_one({'$and': [{'source': 'Shankari'}, {'user_id': user_id}]})
return user_pro['home']
def getDistinctUserCount():
Profiles = get_profile_db()
distinctUserCount = len(Profiles.distinct("user_id"))
logging.debug("Found %s distinct users " % distinctUserCount)
return distinctUserCount
def countForStudy(study):
return get_profile_db().find(getClientQuery(study)).count()
def detect_home_from_db(user_id):
Profiles=get_profile_db()
user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]})
if user_pro is None or 'home' not in user_pro:
return 'N/A'
return user_pro['home']
__author__ = 'Yin'
import logging
from home import detect_home, detect_home_from_db
from zipcode import get_userZipcode
from work_place import detect_work_office, detect_daily_work_office
from get_database import get_section_db, get_profile_db
from pygeocoder import Geocoder
from common import calDistance
from route_matching import update_user_routeDistanceMatrix, update_user_routeClusters
from K_medoid_2 import kmedoids, user_route_data
import math
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.DEBUG)
Profiles = get_profile_db()
TOLERANCE = 200 #How much movement we allow before updating zip codes again. Should be pretty large.. this is conservative
def update_profiles(dummy_users=False):
if dummy_users:
user_list = ['1']
else:
user_list = get_section_db().distinct('user_id')
for user in user_list:
generate_user_home_work(user)
generate_route_clusters(user, 20)
def generate_user_home_work(user):
user_home = detect_home(user)
# print user_home
def changeUpdateTs(self, timedelta):
newTs = self.getUpdateTS() + timedelta
get_profile_db().update({'user_id': self.uuid}, {'$set': {'update_ts': newTs}})
