def setMpgArray(self, mpg_array): logging.debug("Setting MPG array for user %s to : %s" % (self.uuid, mpg_array)) get_profile_db().update({'user_id': self.uuid}, {'$set': { 'mpg_array': mpg_array }})
def invalidate_entries(ret_tokens_list): for token_entry in ret_tokens_list: edb.get_profile_db().update({"device_token": token_entry["token"]}, { "$set": { "device_token_valid": token_entry["valid"], "device_token_invalidated": token_entry["invalidated"] } })
def setUp(self): # Make sure we start with a clean slate every time self.serverName = 'localhost' common.dropAllCollections(get_db()) logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" % (get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count())) common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def setUp(self): self.testUsers = [ "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**" ] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(edb._get_current_db()) self.Profiles = get_profile_db() self.assertEquals(self.Profiles.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Profiles", "emission/tests/data/profiles.json") self.assertEquals(self.Profiles.find().count(), 1) # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = old_div(267.0, 1609) self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks=1)
def get_Alluser_work_end_time(): list_of_time=[] Profiles=edb.get_profile_db() for user in Profiles.distinct("user_id"): for day in range(1,6): list_of_time.extend(get_work_end_time(user,day)) return list_of_time
def get_Alluser_work_end_time(): list_of_time = [] Profiles = edb.get_profile_db() for user in Profiles.distinct("user_id"): for day in range(1, 6): list_of_time.extend(get_work_end_time(user, day)) return list_of_time
def get_matching_tokens(query): logging.debug("Getting tokens matching query %s" % query) ret_cursor = edb.get_profile_db().find(query, { "_id": False, "device_token": True }) mapped_list = map(lambda e: e.get("device_token"), ret_cursor) ret_list = [item for item in mapped_list if item is not None] return ret_list
def get_matching_user_ids(query): logging.debug("Getting tokens matching query %s" % query) ret_cursor = edb.get_profile_db().find(query, { "_id": False, "user_id": True }) mapped_list = [e.get("user_id") for e in ret_cursor] ret_list = [item for item in mapped_list if item is not None] return ret_list
def detect_daily_work_office_from_db(user_id, day): Profiles = edb.get_profile_db() user_pro = Profiles.find_one( {"$and": [{ 'source': 'Shankari' }, { 'user_id': user_id }]}) return user_pro['work' + str(day)]
def createProfile(uuid, ts): initProfileObj = { 'user_id': uuid, 'source': 'Shankari', 'update_ts': ts, 'mpg_array': [defaultMpg] } writeResultProfile = get_profile_db().update_one( {'user_id': uuid}, {'$set': initProfileObj}, upsert=True) return writeResultProfile
def createProfile(uuid, ts): initProfileObj = {'user_id': uuid, 'source':'Shankari', 'update_ts': ts, 'mpg_array': [defaultMpg]} writeResultProfile = get_profile_db().update_one( {'user_id': uuid}, {'$set': initProfileObj}, upsert=True) return writeResultProfile
def createProfile(uuid, ts, studyList): initProfileObj = {'user_id': uuid, 'source':'Shankari', 'update_ts': ts, 'mpg_array': [defaultMpg]} writeResultProfile = get_profile_db().update( {'user_id': uuid}, {'$set': initProfileObj, '$addToSet': {'study_list': {'$each': studyList}}}, upsert=True) return writeResultProfile
def unsetStudy(self, study): # Here's what we want to do: # - if there is no profile entry, ignore # - if there is a profile entry and the study_list does not contain this study, ignore # - if there is a profile entry and the study_list contains the entry, remove it # The following mongodb statement is supposed to handle all of those cases correctly :) writeResult = get_profile_db().update({'user_id': self.uuid}, {'$pullAll': {'study_list': [study]}}) if 'err' in writeResult and writeResult['err'] is not None: logging.error("In setStudy, err = %s" % writeResult['err']) raise Exception()
def _find_platform_users(platform): # Since all new clients register a profile with the server, we don't have # to run a 'distinct' query over the entire contents of the timeseries. # Instead, we can simply query from the profile users, which is # significantly faster # Use the commented out line instead for better performance. # Soon, we can move to the more performant option, because there will be # no users that don't have a profile # return edb.get_timeseries_db().find({'metadata.platform': platform}).distinct( # 'user_id') return edb.get_profile_db().find({"curr_platform": platform}).distinct("user_id")
def get_userZipcode(user, valid_zip): location = detect_home_from_db(user) current_db = get_profile_db() user_pro = current_db.find_one({"$and":[{'source':'Shankari'},{'user_id':user}]}) if valid_zip and user_pro.get('zip'): return user_pro['zip'] elif location!='N/A': # Convert from our internal GeoJSON specific (lng, lat) to the (lat, lng) # format required by geocoder return _geocodeZipcode(user_pro, location) else: return 'N/A'
def setUp(self): # Make sure we start with a clean slate every time self.serverName = 'localhost' common.dropAllCollections(edb._get_current_db()) import shutil self.config_path = "conf/clients/testclient.settings.json" shutil.copyfile("%s.sample" % self.config_path, self.config_path) logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" % (get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count())) common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def getZipcode(): Profiles=get_profile_db() zips = Profiles.distinct("zip") # print(users) userZipcodes = [] ZipDictCount={} ZipDictUser = defaultdict(list) for zip in zips: if zip!='N/A': ZipDictCount[zip] = Profiles.find({'zip':zip}).count() # print(set(userZipcodes)) # print(list(set(userZipcodes))) return ZipDictCount
def setUp(self): # Make sure we start with a clean slate every time self.serverName = 'localhost' common.dropAllCollections(edb._get_current_db()) import shutil self.config_path = "conf/clients/testclient.settings.json" shutil.copyfile("%s.sample" % self.config_path, self.config_path) logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" % (get_client_db().estimated_document_count(), get_profile_db().estimated_document_count(), get_uuid_db().estimated_document_count())) common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def get_participant_uuids(program): """ Get the list of participant UUIDs for the specified program. Note that the "program" parameter is currently a NOP but will be enabled once we have other programs start """ participant_uuid_obj = list(edb.get_profile_db().find( {"install_group": "participant"}, { "user_id": 1, "_id": 0 })) participant_uuid_str = [u["user_id"] for u in participant_uuid_obj] disp.display(participant_uuid_str) return participant_uuid_str
def carbon_by_zip(start,end): Profiles=get_profile_db() carbon_list=[] for zip in Profiles.distinct('zip'): # print(zip) if zip!='N/A': tempdict={} tempdict['weight']=Profiles.find({'zip':zip}).count() # print(Profiles.find({'zip':zip}).count()) tempdict['carbon']=0 for profile in Profiles.find({'zip':zip}): tempdict['loc']=profile['zip_centroid'] user=profile['user_id'] user_carbon=getModeCarbonFootprint(user,carbonFootprintForMode,start,end) tempdict['carbon']+=sum(list(user_carbon.values())) tempdict['carbon']=tempdict['carbon']/tempdict['weight'] carbon_list.append(tempdict) return {"weightedLoc": carbon_list}
def get_matching_tokens(query): logging.debug("Getting tokens matching query %s" % query) ret_map = {"ios": [], "android": []} ret_cursor = edb.get_profile_db().find(query, { "_id": False, "device_token": True, "curr_platform": True }) for i, entry in enumerate(ret_cursor): curr_platform = entry["curr_platform"] device_token = entry["device_token"] if curr_platform is None or device_token is None: logging.warning("ignoring entry %s due to None values" % entry) else: logging.debug("adding token %s to list for platform %s" % (device_token, curr_platform)) ret_map[curr_platform].append(device_token) return ret_map
def get_mode_share_by_Zipcode(zip,flag,start,end): Profiles=get_profile_db() user_list=[] Modeshare={} logging.debug("Called get_mode_share_by_Zipcode(%s)" % zip) for profile in Profiles.find({'zip':zip}): user_list.append(profile['user_id']) if len(user_list)!=0: # print(user_list) for user in user_list: user_share=get_user_mode_share_by_distance(user,flag,start,end) # print(user_share) for mode in user_share.keys(): if mode not in Modeshare: Modeshare[mode]=user_share[mode] else: Modeshare[mode]+=user_share[mode] return Modeshare else: return 'N/A'
def carbon_by_zip(start, end): Profiles = get_profile_db() carbon_list = [] for zip in Profiles.distinct('zip'): # print(zip) if zip != 'N/A': tempdict = {} tempdict['weight'] = Profiles.find({'zip': zip}).count() # print(Profiles.find({'zip':zip}).count()) tempdict['carbon'] = 0 for profile in Profiles.find({'zip': zip}): tempdict['loc'] = profile['zip_centroid'] user = profile['user_id'] user_carbon = getModeCarbonFootprint(user, carbonFootprintForMode, start, end) tempdict['carbon'] += sum(list(user_carbon.values())) tempdict['carbon'] = tempdict['carbon'] / tempdict['weight'] carbon_list.append(tempdict) return {"weightedLoc": carbon_list}
def setUp(self): self.testUsers = ["*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**"] self.serverName = 'localhost' # Sometimes, we may have entries left behind in the database if one of the tests failed # or threw an exception, so let us start by cleaning up all entries emission.tests.common.dropAllCollections(edb._get_current_db()) self.Profiles = get_profile_db() self.assertEquals(self.Profiles.find().count(), 0) emission.tests.common.loadTable(self.serverName, "Stage_Profiles", "emission/tests/data/profiles.json") self.assertEquals(self.Profiles.find().count(), 1) # Let's make sure that the users are registered so that they have profiles for userEmail in self.testUsers: User.register(userEmail) self.walkExpect = 1057.2524056424411 self.busExpect = 2162.668467546699 self.busCarbon = old_div(267.0,1609) self.now = datetime.now() self.dayago = self.now - timedelta(days=1) self.weekago = self.now - timedelta(weeks = 1)
# extract_timeline_for_day_range_and_user.py script # The channel is stored in the "client" field of the profile import emission.core.wrapper.user as ecwu import sys import argparse import logging import json import bson.json_util as bju import emission.core.get_database as edb if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser(prog="get_users_for_channel") parser.add_argument("channel", help="the channel that the users signed in to") parser.add_argument("-o", "--outfile", help="the output filename (default: stdout)") args = parser.parse_args() matched_profiles_it = edb.get_profile_db().find({"client": args.channel}) matched_uuids_it = [p["user_id"] for p in matched_profiles_it] matched_email2uuid_it = [edb.get_uuid_db().find_one({"uuid": u}) for u in matched_uuids_it] logging.debug("Mapped %d entries for channel %s" % (len(matched_email2uuid_it), args.channel)) out_fd = sys.stdout if args.outfile is None else open(args.outfile, "w") json.dump(matched_email2uuid_it, out_fd, default=bju.default)
def update(self, update_doc): logging.debug("Updating user %s with fields %s" % (self.uuid, update_doc)) get_profile_db().update({'user_id': self.uuid}, {'$set': update_doc})
def getProfile(self): return get_profile_db().find_one({'user_id': self.uuid})
def unregister(userEmail): user = User.fromEmail(userEmail) uuid = user.uuid get_uuid_db().remove({'user_email': userEmail}) get_profile_db().remove({'user_id': uuid}) return uuid
def changeUpdateTs(self, timedelta): newTs = self.getUpdateTS() + timedelta get_profile_db().update({'user_id': self.uuid}, {'$set': { 'update_ts': newTs }})
edb.get_uuid_db().update({"uuid" : user.uuid}, {"$set": {"uuid" : new_uuid}}) logging.debug("Resetting alternatives...") reset_collection(edb.get_alternatives_db(), user.uuid, new_uuid) logging.debug("Resetting analysis...") reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting client...") reset_collection(edb.get_client_db(), user.uuid, new_uuid) logging.debug("Resetting client_stats_backup...") reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting server_stats_backup...") reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting result_stats_backup...") reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_place_db...") reset_collection(edb.get_common_place_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_trip_db...") reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_habitica_db...") reset_collection(edb.get_habitica_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_pipeline_state_db...") reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_profile_db...") reset_collection(edb.get_profile_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_db...") reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_error_db...") reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_usercache_db...") reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
def get_matching_user_ids(query): logging.debug("Getting tokens matching query %s" % query) ret_cursor = edb.get_profile_db().find(query, {"_id": False, "user_id": True}) mapped_list = [e.get("user_id") for e in ret_cursor] ret_list = [item for item in mapped_list if item is not None] return ret_list
def detect_daily_work_office_from_db(user_id,day): Profiles=edb.get_profile_db() user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]}) return user_pro['work'+str(day)]
def getDistinctUserCount(): Profiles=get_profile_db() distinctUserCount = len(Profiles.distinct("user_id")) logging.debug("Found %s distinct users " % distinctUserCount) return distinctUserCount
def setMpgArray(self, mpg_array): logging.debug("Setting MPG array for user %s to : %s" % (self.uuid, mpg_array)) get_profile_db().update_one({'user_id': self.uuid}, {'$set': {'mpg_array': mpg_array}})
def setClientSpecificProfileFields(self, setQuery): logging.debug("Changing profile for user %s to %s" % (self.uuid, setQuery)) get_profile_db().update({'user_id': self.uuid}, {'$set': setQuery})
def getProfile(self): # is user_id a uuid? return edb.get_profile_db().find_one({'user_id': self.user_id})
import logging from pygeocoder import Geocoder import math # Our imports from emission.analysis.modelling.home import detect_home, detect_home_from_db from zipcode import get_userZipcode from emission.analysis.modelling.work_place import detect_work_office, detect_daily_work_office from emission.core.get_database import get_trip_db, get_section_db,get_profile_db from emission.core.common import calDistance from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import update_user_routeDistanceMatrix, update_user_routeClusters from emission.analysis.modelling.tour_model.K_medoid import kmedoids, user_route_data import emission.analysis.modelling.tour_model.cluster_pipeline as cp TOLERANCE = 200 #How much movement we allow before updating zip codes again. Should be pretty large.. this is conservative Profiles=get_profile_db() def update_profiles(dummy_users=False): if dummy_users: user_list = ['1'] else: user_list = get_section_db().distinct('user_id') for user in user_list: generate_user_home_work(user) generate_route_clusters(user) def generate_user_home_work(user): user_home=detect_home(user) # print user_home zip_is_valid = _check_zip_validity(user_home, user) logging.debug('starting for %s' % user)
def invalidate_entries(ret_tokens_list): for token_entry in ret_tokens_list: edb.get_profile_db().update({"device_token": token_entry["token"]}, {"$set": { "device_token_valid": token_entry["valid"], "device_token_invalidated": token_entry["invalidated"] }});
def detect_home_from_db(user_id): Profiles=edb.get_profile_db() user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]}) if user_pro is None or 'home' not in user_pro: return 'N/A' return user_pro['home']
reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting client...") reset_collection(edb.get_client_db(), user.uuid, new_uuid) logging.debug("Resetting client_stats_backup...") reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting server_stats_backup...") reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting result_stats_backup...") reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_place_db...") reset_collection(edb.get_common_place_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_common_trip_db...") reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_habitica_db...") reset_collection(edb.get_habitica_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_pipeline_state_db...") reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_profile_db...") reset_collection(edb.get_profile_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_db...") reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_timeseries_error_db...") reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid) logging.debug("Resetting edb.get_usercache_db...") reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
def countForStudy(study): return get_profile_db().find(getClientQuery(study)).count()
from pygeocoder import Geocoder import math # Our imports from emission.analysis.modelling.home import detect_home, detect_home_from_db from zipcode import get_userZipcode from emission.analysis.modelling.work_place import detect_work_office, detect_daily_work_office from emission.core.get_database import get_trip_db, get_section_db, get_profile_db from emission.core.common import calDistance from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import update_user_routeDistanceMatrix, update_user_routeClusters from emission.analysis.modelling.tour_model.K_medoid import kmedoids, user_route_data import emission.analysis.modelling.tour_model.cluster_pipeline as cp logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.DEBUG) Profiles = get_profile_db() TOLERANCE = 200 #How much movement we allow before updating zip codes again. Should be pretty large.. this is conservative def update_profiles(dummy_users=False): if dummy_users: user_list = ['1'] else: user_list = get_section_db().distinct('user_id') for user in user_list: generate_user_home_work(user) generate_route_clusters(user) def generate_user_home_work(user): user_home = detect_home(user)
def changeUpdateTs(self, timedelta): newTs = self.getUpdateTS() + timedelta get_profile_db().update({'user_id': self.uuid}, {'$set': {'update_ts': newTs}})
def unregister(userEmail): user = User.fromEmail(userEmail) uuid = user.uuid get_uuid_db().delete_one({'user_email': userEmail}) get_profile_db().delete_one({'user_id': uuid}) return uuid
def getUsersForClient(clientName): # Find all users for this client client_uuids = [] for user in get_profile_db().find(getClientQuery(clientName)): client_uuids.append(user)
def countForStudy(study): return get_profile_db().count_documents(getClientQuery(study))
def get_matching_tokens(query): logging.debug("Getting tokens matching query %s" % query) ret_cursor = edb.get_profile_db().find(query, {"_id": False, "device_token": True}) mapped_list = map(lambda e: e.get("device_token"), ret_cursor) ret_list = [item for item in mapped_list if item is not None] return ret_list
def update(self, update_doc): logging.debug("Updating user %s with fields %s" % (self.uuid, update_doc)) get_profile_db().update_one({'user_id': self.uuid}, {'$set': update_doc})