def setMpgArray(self, mpg_array):
logging.debug("Setting MPG array for user %s to : %s" %
(self.uuid, mpg_array))
get_profile_db().update({'user_id': self.uuid},
{'$set': {
'mpg_array': mpg_array
}})
def invalidate_entries(ret_tokens_list):
for token_entry in ret_tokens_list:
edb.get_profile_db().update({"device_token": token_entry["token"]}, {
"$set": {
"device_token_valid": token_entry["valid"],
"device_token_invalidated": token_entry["invalidated"]
}
})
def setUp(self):
# Make sure we start with a clean slate every time
self.serverName = 'localhost'
common.dropAllCollections(get_db())
logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" %
(get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count()))
common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def setUp(self):
# Make sure we start with a clean slate every time
self.serverName = 'localhost'
common.dropAllCollections(get_db())
logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" %
(get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count()))
common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def setUp(self):
self.testUsers = [
"*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"
]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
emission.tests.common.dropAllCollections(edb._get_current_db())
self.Profiles = get_profile_db()
self.assertEquals(self.Profiles.find().count(), 0)
emission.tests.common.loadTable(self.serverName, "Stage_Profiles",
"emission/tests/data/profiles.json")
self.assertEquals(self.Profiles.find().count(), 1)
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = old_div(267.0, 1609)
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks=1)
def get_Alluser_work_end_time():
list_of_time=[]
Profiles=edb.get_profile_db()
for user in Profiles.distinct("user_id"):
for day in range(1,6):
list_of_time.extend(get_work_end_time(user,day))
return list_of_time
def get_Alluser_work_end_time():
list_of_time = []
Profiles = edb.get_profile_db()
for user in Profiles.distinct("user_id"):
for day in range(1, 6):
list_of_time.extend(get_work_end_time(user, day))
return list_of_time
def get_matching_tokens(query):
logging.debug("Getting tokens matching query %s" % query)
ret_cursor = edb.get_profile_db().find(query, {
"_id": False,
"device_token": True
})
mapped_list = map(lambda e: e.get("device_token"), ret_cursor)
ret_list = [item for item in mapped_list if item is not None]
return ret_list
def get_matching_user_ids(query):
logging.debug("Getting tokens matching query %s" % query)
ret_cursor = edb.get_profile_db().find(query, {
"_id": False,
"user_id": True
})
mapped_list = [e.get("user_id") for e in ret_cursor]
ret_list = [item for item in mapped_list if item is not None]
return ret_list
def detect_daily_work_office_from_db(user_id, day):
Profiles = edb.get_profile_db()
user_pro = Profiles.find_one(
{"$and": [{
'source': 'Shankari'
}, {
'user_id': user_id
}]})
return user_pro['work' + str(day)]
def createProfile(uuid, ts):
initProfileObj = {
'user_id': uuid,
'source': 'Shankari',
'update_ts': ts,
'mpg_array': [defaultMpg]
}
writeResultProfile = get_profile_db().update_one(
{'user_id': uuid}, {'$set': initProfileObj}, upsert=True)
return writeResultProfile
def createProfile(uuid, ts):
initProfileObj = {'user_id': uuid,
'source':'Shankari',
'update_ts': ts,
'mpg_array': [defaultMpg]}
writeResultProfile = get_profile_db().update_one(
{'user_id': uuid},
{'$set': initProfileObj},
upsert=True)
return writeResultProfile
def createProfile(uuid, ts, studyList):
initProfileObj = {'user_id': uuid,
'source':'Shankari',
'update_ts': ts,
'mpg_array': [defaultMpg]}
writeResultProfile = get_profile_db().update(
{'user_id': uuid},
{'$set': initProfileObj,
'$addToSet': {'study_list': {'$each': studyList}}},
upsert=True)
return writeResultProfile
def unsetStudy(self, study):
# Here's what we want to do:
# - if there is no profile entry, ignore
# - if there is a profile entry and the study_list does not contain this study, ignore
# - if there is a profile entry and the study_list contains the entry, remove it
# The following mongodb statement is supposed to handle all of those cases correctly :)
writeResult = get_profile_db().update({'user_id': self.uuid},
{'$pullAll': {'study_list': [study]}})
if 'err' in writeResult and writeResult['err'] is not None:
logging.error("In setStudy, err = %s" % writeResult['err'])
raise Exception()
def createProfile(uuid, ts, studyList):
initProfileObj = {'user_id': uuid,
'source':'Shankari',
'update_ts': ts,
'mpg_array': [defaultMpg]}
writeResultProfile = get_profile_db().update(
{'user_id': uuid},
{'$set': initProfileObj,
'$addToSet': {'study_list': {'$each': studyList}}},
upsert=True)
return writeResultProfile
def _find_platform_users(platform):
# Since all new clients register a profile with the server, we don't have
# to run a 'distinct' query over the entire contents of the timeseries.
# Instead, we can simply query from the profile users, which is
# significantly faster
# Use the commented out line instead for better performance.
# Soon, we can move to the more performant option, because there will be
# no users that don't have a profile
# return edb.get_timeseries_db().find({'metadata.platform': platform}).distinct(
# 'user_id')
return edb.get_profile_db().find({"curr_platform": platform}).distinct("user_id")
def _find_platform_users(platform):
# Since all new clients register a profile with the server, we don't have
# to run a 'distinct' query over the entire contents of the timeseries.
# Instead, we can simply query from the profile users, which is
# significantly faster
# Use the commented out line instead for better performance.
# Soon, we can move to the more performant option, because there will be
# no users that don't have a profile
# return edb.get_timeseries_db().find({'metadata.platform': platform}).distinct(
# 'user_id')
return edb.get_profile_db().find({"curr_platform": platform}).distinct("user_id")
def unsetStudy(self, study):
# Here's what we want to do:
# - if there is no profile entry, ignore
# - if there is a profile entry and the study_list does not contain this study, ignore
# - if there is a profile entry and the study_list contains the entry, remove it
# The following mongodb statement is supposed to handle all of those cases correctly :)
writeResult = get_profile_db().update({'user_id': self.uuid},
{'$pullAll': {'study_list': [study]}})
if 'err' in writeResult and writeResult['err'] is not None:
logging.error("In setStudy, err = %s" % writeResult['err'])
raise Exception()
def get_userZipcode(user, valid_zip):
location = detect_home_from_db(user)
current_db = get_profile_db()
user_pro = current_db.find_one({"$and":[{'source':'Shankari'},{'user_id':user}]})
if valid_zip and user_pro.get('zip'):
return user_pro['zip']
elif location!='N/A':
# Convert from our internal GeoJSON specific (lng, lat) to the (lat, lng)
# format required by geocoder
return _geocodeZipcode(user_pro, location)
else:
return 'N/A'
def setUp(self):
# Make sure we start with a clean slate every time
self.serverName = 'localhost'
common.dropAllCollections(edb._get_current_db())
import shutil
self.config_path = "conf/clients/testclient.settings.json"
shutil.copyfile("%s.sample" % self.config_path,
self.config_path)
logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" %
(get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count()))
common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def getZipcode():
Profiles=get_profile_db()
zips = Profiles.distinct("zip")
# print(users)
userZipcodes = []
ZipDictCount={}
ZipDictUser = defaultdict(list)
for zip in zips:
if zip!='N/A':
ZipDictCount[zip] = Profiles.find({'zip':zip}).count()
# print(set(userZipcodes))
# print(list(set(userZipcodes)))
return ZipDictCount
def setUp(self):
# Make sure we start with a clean slate every time
self.serverName = 'localhost'
common.dropAllCollections(edb._get_current_db())
import shutil
self.config_path = "conf/clients/testclient.settings.json"
shutil.copyfile("%s.sample" % self.config_path,
self.config_path)
logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" %
(get_client_db().estimated_document_count(), get_profile_db().estimated_document_count(), get_uuid_db().estimated_document_count()))
common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def get_participant_uuids(program):
"""
Get the list of participant UUIDs for the specified program.
Note that the "program" parameter is currently a NOP but will be enabled
once we have other programs start
"""
participant_uuid_obj = list(edb.get_profile_db().find(
{"install_group": "participant"}, {
"user_id": 1,
"_id": 0
}))
participant_uuid_str = [u["user_id"] for u in participant_uuid_obj]
disp.display(participant_uuid_str)
return participant_uuid_str
def carbon_by_zip(start,end):
Profiles=get_profile_db()
carbon_list=[]
for zip in Profiles.distinct('zip'):
# print(zip)
if zip!='N/A':
tempdict={}
tempdict['weight']=Profiles.find({'zip':zip}).count()
# print(Profiles.find({'zip':zip}).count())
tempdict['carbon']=0
for profile in Profiles.find({'zip':zip}):
tempdict['loc']=profile['zip_centroid']
user=profile['user_id']
user_carbon=getModeCarbonFootprint(user,carbonFootprintForMode,start,end)
tempdict['carbon']+=sum(list(user_carbon.values()))
tempdict['carbon']=tempdict['carbon']/tempdict['weight']
carbon_list.append(tempdict)
return {"weightedLoc": carbon_list}
def get_matching_tokens(query):
logging.debug("Getting tokens matching query %s" % query)
ret_map = {"ios": [], "android": []}
ret_cursor = edb.get_profile_db().find(query, {
"_id": False,
"device_token": True,
"curr_platform": True
})
for i, entry in enumerate(ret_cursor):
curr_platform = entry["curr_platform"]
device_token = entry["device_token"]
if curr_platform is None or device_token is None:
logging.warning("ignoring entry %s due to None values" % entry)
else:
logging.debug("adding token %s to list for platform %s" %
(device_token, curr_platform))
ret_map[curr_platform].append(device_token)
return ret_map
def get_mode_share_by_Zipcode(zip,flag,start,end):
Profiles=get_profile_db()
user_list=[]
Modeshare={}
logging.debug("Called get_mode_share_by_Zipcode(%s)" % zip)
for profile in Profiles.find({'zip':zip}):
user_list.append(profile['user_id'])
if len(user_list)!=0:
# print(user_list)
for user in user_list:
user_share=get_user_mode_share_by_distance(user,flag,start,end)
# print(user_share)
for mode in user_share.keys():
if mode not in Modeshare:
Modeshare[mode]=user_share[mode]
else:
Modeshare[mode]+=user_share[mode]
return Modeshare
else:
return 'N/A'
def carbon_by_zip(start, end):
Profiles = get_profile_db()
carbon_list = []
for zip in Profiles.distinct('zip'):
# print(zip)
if zip != 'N/A':
tempdict = {}
tempdict['weight'] = Profiles.find({'zip': zip}).count()
# print(Profiles.find({'zip':zip}).count())
tempdict['carbon'] = 0
for profile in Profiles.find({'zip': zip}):
tempdict['loc'] = profile['zip_centroid']
user = profile['user_id']
user_carbon = getModeCarbonFootprint(user,
carbonFootprintForMode,
start, end)
tempdict['carbon'] += sum(list(user_carbon.values()))
tempdict['carbon'] = tempdict['carbon'] / tempdict['weight']
carbon_list.append(tempdict)
return {"weightedLoc": carbon_list}
def setUp(self):
self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**",
"*****@*****.**", "*****@*****.**"]
self.serverName = 'localhost'
# Sometimes, we may have entries left behind in the database if one of the tests failed
# or threw an exception, so let us start by cleaning up all entries
emission.tests.common.dropAllCollections(edb._get_current_db())
self.Profiles = get_profile_db()
self.assertEquals(self.Profiles.find().count(), 0)
emission.tests.common.loadTable(self.serverName, "Stage_Profiles", "emission/tests/data/profiles.json")
self.assertEquals(self.Profiles.find().count(), 1)
# Let's make sure that the users are registered so that they have profiles
for userEmail in self.testUsers:
User.register(userEmail)
self.walkExpect = 1057.2524056424411
self.busExpect = 2162.668467546699
self.busCarbon = old_div(267.0,1609)
self.now = datetime.now()
self.dayago = self.now - timedelta(days=1)
self.weekago = self.now - timedelta(weeks = 1)
# extract_timeline_for_day_range_and_user.py script
# The channel is stored in the "client" field of the profile
import emission.core.wrapper.user as ecwu
import sys
import argparse
import logging
import json
import bson.json_util as bju
import emission.core.get_database as edb
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
parser = argparse.ArgumentParser(prog="get_users_for_channel")
parser.add_argument("channel", help="the channel that the users signed in to")
parser.add_argument("-o", "--outfile", help="the output filename (default: stdout)")
args = parser.parse_args()
matched_profiles_it = edb.get_profile_db().find({"client": args.channel})
matched_uuids_it = [p["user_id"] for p in matched_profiles_it]
matched_email2uuid_it = [edb.get_uuid_db().find_one({"uuid": u}) for u in matched_uuids_it]
logging.debug("Mapped %d entries for channel %s" % (len(matched_email2uuid_it), args.channel))
out_fd = sys.stdout if args.outfile is None else open(args.outfile, "w")
json.dump(matched_email2uuid_it, out_fd, default=bju.default)
def update(self, update_doc):
logging.debug("Updating user %s with fields %s" %
(self.uuid, update_doc))
get_profile_db().update({'user_id': self.uuid}, {'$set': update_doc})
def getProfile(self):
return get_profile_db().find_one({'user_id': self.uuid})
def unregister(userEmail):
user = User.fromEmail(userEmail)
uuid = user.uuid
get_uuid_db().remove({'user_email': userEmail})
get_profile_db().remove({'user_id': uuid})
return uuid
def changeUpdateTs(self, timedelta):
newTs = self.getUpdateTS() + timedelta
get_profile_db().update({'user_id': self.uuid},
{'$set': {
'update_ts': newTs
}})
edb.get_uuid_db().update({"uuid" : user.uuid},
{"$set": {"uuid" : new_uuid}})
logging.debug("Resetting alternatives...")
reset_collection(edb.get_alternatives_db(), user.uuid, new_uuid)
logging.debug("Resetting analysis...")
reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid)
logging.debug("Resetting client...")
reset_collection(edb.get_client_db(), user.uuid, new_uuid)
logging.debug("Resetting client_stats_backup...")
reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid)
logging.debug("Resetting server_stats_backup...")
reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid)
logging.debug("Resetting result_stats_backup...")
reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_common_place_db...")
reset_collection(edb.get_common_place_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_common_trip_db...")
reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_habitica_db...")
reset_collection(edb.get_habitica_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_pipeline_state_db...")
reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_profile_db...")
reset_collection(edb.get_profile_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_db...")
reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_error_db...")
reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_usercache_db...")
reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
def get_matching_user_ids(query):
logging.debug("Getting tokens matching query %s" % query)
ret_cursor = edb.get_profile_db().find(query, {"_id": False, "user_id": True})
mapped_list = [e.get("user_id") for e in ret_cursor]
ret_list = [item for item in mapped_list if item is not None]
return ret_list
def detect_daily_work_office_from_db(user_id,day):
Profiles=edb.get_profile_db()
user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]})
return user_pro['work'+str(day)]
def getDistinctUserCount():
Profiles=get_profile_db()
distinctUserCount = len(Profiles.distinct("user_id"))
logging.debug("Found %s distinct users " % distinctUserCount)
return distinctUserCount
def setMpgArray(self, mpg_array):
logging.debug("Setting MPG array for user %s to : %s" % (self.uuid, mpg_array))
get_profile_db().update_one({'user_id': self.uuid}, {'$set': {'mpg_array': mpg_array}})
def setClientSpecificProfileFields(self, setQuery):
logging.debug("Changing profile for user %s to %s" % (self.uuid, setQuery))
get_profile_db().update({'user_id': self.uuid}, {'$set': setQuery})
def getProfile(self):
# is user_id a uuid?
return edb.get_profile_db().find_one({'user_id': self.user_id})
import logging
from pygeocoder import Geocoder
import math
# Our imports
from emission.analysis.modelling.home import detect_home, detect_home_from_db
from zipcode import get_userZipcode
from emission.analysis.modelling.work_place import detect_work_office, detect_daily_work_office
from emission.core.get_database import get_trip_db, get_section_db,get_profile_db
from emission.core.common import calDistance
from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import update_user_routeDistanceMatrix, update_user_routeClusters
from emission.analysis.modelling.tour_model.K_medoid import kmedoids, user_route_data
import emission.analysis.modelling.tour_model.cluster_pipeline as cp
TOLERANCE = 200 #How much movement we allow before updating zip codes again. Should be pretty large.. this is conservative
Profiles=get_profile_db()
def update_profiles(dummy_users=False):
if dummy_users:
user_list = ['1']
else:
user_list = get_section_db().distinct('user_id')
for user in user_list:
generate_user_home_work(user)
generate_route_clusters(user)
def generate_user_home_work(user):
user_home=detect_home(user)
# print user_home
zip_is_valid = _check_zip_validity(user_home, user)
logging.debug('starting for %s' % user)
def invalidate_entries(ret_tokens_list):
for token_entry in ret_tokens_list:
edb.get_profile_db().update({"device_token": token_entry["token"]}, {"$set": {
"device_token_valid": token_entry["valid"],
"device_token_invalidated": token_entry["invalidated"]
}});
def detect_home_from_db(user_id):
Profiles=edb.get_profile_db()
user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]})
if user_pro is None or 'home' not in user_pro:
return 'N/A'
return user_pro['home']
def getProfile(self):
# is user_id a uuid?
return edb.get_profile_db().find_one({'user_id': self.user_id})
reset_collection(edb.get_analysis_timeseries_db(), user.uuid,
new_uuid)
logging.debug("Resetting client...")
reset_collection(edb.get_client_db(), user.uuid, new_uuid)
logging.debug("Resetting client_stats_backup...")
reset_collection(edb.get_client_stats_db_backup(), user.uuid,
new_uuid)
logging.debug("Resetting server_stats_backup...")
reset_collection(edb.get_server_stats_db_backup(), user.uuid,
new_uuid)
logging.debug("Resetting result_stats_backup...")
reset_collection(edb.get_result_stats_db_backup(), user.uuid,
new_uuid)
logging.debug("Resetting edb.get_common_place_db...")
reset_collection(edb.get_common_place_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_common_trip_db...")
reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_habitica_db...")
reset_collection(edb.get_habitica_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_pipeline_state_db...")
reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_profile_db...")
reset_collection(edb.get_profile_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_db...")
reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid)
logging.debug("Resetting edb.get_timeseries_error_db...")
reset_collection(edb.get_timeseries_error_db(), user.uuid,
new_uuid)
logging.debug("Resetting edb.get_usercache_db...")
reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
def countForStudy(study): return get_profile_db().find(getClientQuery(study)).count()
from pygeocoder import Geocoder
import math
# Our imports
from emission.analysis.modelling.home import detect_home, detect_home_from_db
from zipcode import get_userZipcode
from emission.analysis.modelling.work_place import detect_work_office, detect_daily_work_office
from emission.core.get_database import get_trip_db, get_section_db, get_profile_db
from emission.core.common import calDistance
from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import update_user_routeDistanceMatrix, update_user_routeClusters
from emission.analysis.modelling.tour_model.K_medoid import kmedoids, user_route_data
import emission.analysis.modelling.tour_model.cluster_pipeline as cp
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
level=logging.DEBUG)
Profiles = get_profile_db()
TOLERANCE = 200 #How much movement we allow before updating zip codes again. Should be pretty large.. this is conservative
def update_profiles(dummy_users=False):
if dummy_users:
user_list = ['1']
else:
user_list = get_section_db().distinct('user_id')
for user in user_list:
generate_user_home_work(user)
generate_route_clusters(user)
def generate_user_home_work(user):
user_home = detect_home(user)
def changeUpdateTs(self, timedelta):
newTs = self.getUpdateTS() + timedelta
get_profile_db().update({'user_id': self.uuid}, {'$set': {'update_ts': newTs}})
def unregister(userEmail):
user = User.fromEmail(userEmail)
uuid = user.uuid
get_uuid_db().delete_one({'user_email': userEmail})
get_profile_db().delete_one({'user_id': uuid})
return uuid
def getUsersForClient(clientName):
# Find all users for this client
client_uuids = []
for user in get_profile_db().find(getClientQuery(clientName)):
client_uuids.append(user)
def countForStudy(study): return get_profile_db().count_documents(getClientQuery(study))
def getUsersForClient(clientName):
# Find all users for this client
client_uuids = []
for user in get_profile_db().find(getClientQuery(clientName)):
client_uuids.append(user)
def getProfile(self):
return get_profile_db().find_one({'user_id': self.uuid})
def get_matching_tokens(query):
logging.debug("Getting tokens matching query %s" % query)
ret_cursor = edb.get_profile_db().find(query, {"_id": False, "device_token": True})
mapped_list = map(lambda e: e.get("device_token"), ret_cursor)
ret_list = [item for item in mapped_list if item is not None]
return ret_list
def update(self, update_doc):
logging.debug("Updating user %s with fields %s" % (self.uuid, update_doc))
get_profile_db().update_one({'user_id': self.uuid}, {'$set': update_doc})
def setClientSpecificProfileFields(self, setQuery):
logging.debug("Changing profile for user %s to %s" % (self.uuid, setQuery))
get_profile_db().update({'user_id': self.uuid}, {'$set': setQuery})
