def setUp(self):
   # Make sure we start with a clean slate every time
   self.serverName = 'localhost'
   common.dropAllCollections(get_db())
   logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" % 
     (get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count()))
   common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
def get_Alluser_work_end_time():
    list_of_time=[]
    Profiles=edb.get_profile_db()
    for user in Profiles.distinct("user_id"):
        for day in range(1,6):
            list_of_time.extend(get_work_end_time(user,day))
    return list_of_time
Example #3
0
 def createProfile(uuid, ts):
   initProfileObj = {'user_id': uuid,
                     'source':'Shankari',
                     'update_ts': ts,
                     'mpg_array': [defaultMpg]}
   writeResultProfile = get_profile_db().update_one(
       {'user_id': uuid},
       {'$set': initProfileObj},
       upsert=True)
   return writeResultProfile
Example #4
0
 def unsetStudy(self, study):
   # Here's what we want to do:
   # - if there is no profile entry, ignore
   # - if there is a profile entry and the study_list does not contain this study, ignore
   # - if there is a profile entry and the study_list contains the entry, remove it
   # The following mongodb statement is supposed to handle all of those cases correctly :)
   writeResult = get_profile_db().update({'user_id': self.uuid},
     {'$pullAll': {'study_list': [study]}})
   if 'err' in writeResult and writeResult['err'] is not None:
     logging.error("In setStudy, err = %s" % writeResult['err'])
     raise Exception()
Example #5
0
 def createProfile(uuid, ts, studyList):
   initProfileObj = {'user_id': uuid,
                     'source':'Shankari',
                     'update_ts': ts,
                     'mpg_array': [defaultMpg]}
   writeResultProfile = get_profile_db().update(
       {'user_id': uuid},
       {'$set': initProfileObj,
        '$addToSet': {'study_list': {'$each': studyList}}},
       upsert=True)
   return writeResultProfile
def _find_platform_users(platform):
    # Since all new clients register a profile with the server, we don't have
    # to run a 'distinct' query over the entire contents of the timeseries.
    # Instead, we can simply query from the profile users, which is
    # significantly faster
    # Use the commented out line instead for better performance.
    # Soon, we can move to the more performant option, because there will be
    # no users that don't have a profile
    # return edb.get_timeseries_db().find({'metadata.platform': platform}).distinct(
    #    'user_id')
   return edb.get_profile_db().find({"curr_platform": platform}).distinct("user_id")
Example #7
0
def get_userZipcode(user, valid_zip):
    location = detect_home_from_db(user)
    current_db = get_profile_db()
    user_pro = current_db.find_one({"$and":[{'source':'Shankari'},{'user_id':user}]})
    if valid_zip and user_pro.get('zip'):
        return user_pro['zip']
    elif location!='N/A':
        # Convert from our internal GeoJSON specific (lng, lat) to the (lat, lng)
        # format required by geocoder
        return _geocodeZipcode(user_pro, location)
    else:
        return 'N/A'
Example #8
0
  def setUp(self):
    # Make sure we start with a clean slate every time
    self.serverName = 'localhost'
    common.dropAllCollections(edb._get_current_db())

    import shutil
    self.config_path = "conf/clients/testclient.settings.json"
    shutil.copyfile("%s.sample" % self.config_path,
                    self.config_path)

    logging.info("After setup, client count = %d, profile count = %d, uuid count = %d" % 
      (get_client_db().find().count(), get_profile_db().count(), get_uuid_db().count()))
    common.loadTable(self.serverName, "Stage_Modes", "emission/tests/data/modes.json")
Example #9
0
def getZipcode():
    Profiles=get_profile_db()
    zips = Profiles.distinct("zip")
    # print(users)
    userZipcodes = []
    ZipDictCount={}
    ZipDictUser = defaultdict(list)
    for zip in zips:
        if zip!='N/A':
            ZipDictCount[zip] = Profiles.find({'zip':zip}).count()
    # print(set(userZipcodes))
    # print(list(set(userZipcodes)))
    return ZipDictCount
def carbon_by_zip(start,end):
    Profiles=get_profile_db()
    carbon_list=[]
    for zip in Profiles.distinct('zip'):
        # print(zip)
        if zip!='N/A':
            tempdict={}
            tempdict['weight']=Profiles.find({'zip':zip}).count()
            # print(Profiles.find({'zip':zip}).count())
            tempdict['carbon']=0
            for profile in Profiles.find({'zip':zip}):
                tempdict['loc']=profile['zip_centroid']
                user=profile['user_id']
                user_carbon=getModeCarbonFootprint(user,carbonFootprintForMode,start,end)
                tempdict['carbon']+=sum(list(user_carbon.values()))
            tempdict['carbon']=tempdict['carbon']/tempdict['weight']
            carbon_list.append(tempdict)
    return {"weightedLoc": carbon_list}
Example #11
0
def get_mode_share_by_Zipcode(zip,flag,start,end):
    Profiles=get_profile_db()
    user_list=[]
    Modeshare={}
    logging.debug("Called get_mode_share_by_Zipcode(%s)" % zip)
    for profile in Profiles.find({'zip':zip}):
        user_list.append(profile['user_id'])
    if len(user_list)!=0:
        # print(user_list)
        for user in user_list:
            user_share=get_user_mode_share_by_distance(user,flag,start,end)
            # print(user_share)
            for mode in user_share.keys():
                if mode not in Modeshare:
                    Modeshare[mode]=user_share[mode]
                else:
                    Modeshare[mode]+=user_share[mode]
        return Modeshare
    else:
        return 'N/A'
Example #12
0
  def setUp(self):
    self.testUsers = ["[email protected]", "[email protected]", "[email protected]",
                      "[email protected]", "[email protected]"]
    self.serverName = 'localhost'

    # Sometimes, we may have entries left behind in the database if one of the tests failed
    # or threw an exception, so let us start by cleaning up all entries
    emission.tests.common.dropAllCollections(edb._get_current_db())
    self.Profiles = get_profile_db()
    self.assertEquals(self.Profiles.find().count(), 0)
    emission.tests.common.loadTable(self.serverName, "Stage_Profiles", "emission/tests/data/profiles.json")
    self.assertEquals(self.Profiles.find().count(), 1)
    # Let's make sure that the users are registered so that they have profiles
    for userEmail in self.testUsers:
      User.register(userEmail)

    self.walkExpect = 1057.2524056424411
    self.busExpect = 2162.668467546699
    self.busCarbon = old_div(267.0,1609)

    self.now = datetime.now()
    self.dayago = self.now - timedelta(days=1)
    self.weekago = self.now - timedelta(weeks = 1)
Example #13
0
 def setMpgArray(self, mpg_array):
   logging.debug("Setting MPG array for user %s to : %s" % (self.uuid, mpg_array))
   get_profile_db().update_one({'user_id': self.uuid}, {'$set': {'mpg_array': mpg_array}})
def detect_daily_work_office_from_db(user_id,day):
    Profiles=edb.get_profile_db()
    user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]})
    return user_pro['work'+str(day)]
Example #15
0
def detect_home_from_db(user_id):
    Profiles=edb.get_profile_db()
    user_pro=Profiles.find_one({"$and":[{'source':'Shankari'},{'user_id':user_id}]})
    if user_pro is None or 'home' not in user_pro:
        return 'N/A'
    return user_pro['home']
def get_matching_user_ids(query):
    logging.debug("Getting tokens matching query %s" % query)
    ret_cursor = edb.get_profile_db().find(query, {"_id": False, "user_id": True})
    mapped_list = [e.get("user_id") for e in ret_cursor]
    ret_list = [item for item in mapped_list if item is not None]
    return ret_list
Example #17
0
import logging
from pygeocoder import Geocoder
import math

# Our imports
from emission.analysis.modelling.home import detect_home, detect_home_from_db
from zipcode import get_userZipcode
from emission.analysis.modelling.work_place import detect_work_office, detect_daily_work_office
from emission.core.get_database import get_trip_db, get_section_db,get_profile_db
from emission.core.common import calDistance
from emission.analysis.modelling.tour_model.trajectory_matching.route_matching import update_user_routeDistanceMatrix, update_user_routeClusters
from emission.analysis.modelling.tour_model.K_medoid import kmedoids, user_route_data
import emission.analysis.modelling.tour_model.cluster_pipeline as cp

TOLERANCE = 200 #How much movement we allow before updating zip codes again. Should be pretty large.. this is conservative
Profiles=get_profile_db()

def update_profiles(dummy_users=False):
    if dummy_users:
        user_list = ['1']
    else:
        user_list = get_section_db().distinct('user_id')
    for user in user_list:
        generate_user_home_work(user)
        generate_route_clusters(user)

def generate_user_home_work(user):
    user_home=detect_home(user)
    # print user_home
    zip_is_valid = _check_zip_validity(user_home, user)
    logging.debug('starting for %s' % user)
Example #18
0
def getUsersForClient(clientName):
  # Find all users for this client
  client_uuids = []
  for user in get_profile_db().find(getClientQuery(clientName)):
    client_uuids.append(user)
def get_matching_tokens(query):
    logging.debug("Getting tokens matching query %s" % query)
    ret_cursor = edb.get_profile_db().find(query, {"_id": False, "device_token": True})
    mapped_list = map(lambda e: e.get("device_token"), ret_cursor)
    ret_list = [item for item in mapped_list if item is not None]
    return ret_list
Example #20
0
def countForStudy(study):
  return get_profile_db().find(getClientQuery(study)).count()
Example #21
0
 def changeUpdateTs(self, timedelta):
   newTs = self.getUpdateTS() + timedelta
   get_profile_db().update({'user_id': self.uuid}, {'$set': {'update_ts': newTs}})
Example #22
0
 def unregister(userEmail):
   user = User.fromEmail(userEmail)
   uuid = user.uuid
   get_uuid_db().delete_one({'user_email': userEmail})
   get_profile_db().delete_one({'user_id': uuid})
   return uuid
 def getProfile(self):
   # is user_id a uuid?
   return edb.get_profile_db().find_one({'user_id': self.user_id})
# extract_timeline_for_day_range_and_user.py script
# The channel is stored in the "client" field of the profile

import emission.core.wrapper.user as ecwu

import sys
import argparse
import logging
import json
import bson.json_util as bju

import emission.core.get_database as edb

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    parser = argparse.ArgumentParser(prog="get_users_for_channel")

    parser.add_argument("channel", help="the channel that the users signed in to")
    parser.add_argument("-o", "--outfile", help="the output filename (default: stdout)")

    args = parser.parse_args()

    matched_profiles_it = edb.get_profile_db().find({"client": args.channel})
    matched_uuids_it = [p["user_id"] for p in matched_profiles_it]
    matched_email2uuid_it = [edb.get_uuid_db().find_one({"uuid": u}) for u in matched_uuids_it]

    logging.debug("Mapped %d entries for channel %s" % (len(matched_email2uuid_it), args.channel)) 

    out_fd = sys.stdout if args.outfile is None else open(args.outfile, "w")
    json.dump(matched_email2uuid_it, out_fd, default=bju.default)
def invalidate_entries(ret_tokens_list):
    for token_entry in ret_tokens_list:
        edb.get_profile_db().update({"device_token": token_entry["token"]}, {"$set": {
            "device_token_valid": token_entry["valid"],
            "device_token_invalidated": token_entry["invalidated"]
        }});
Example #26
0
            edb.get_uuid_db().update({"uuid" : user.uuid},
                                     {"$set": {"uuid" : new_uuid}})
            logging.debug("Resetting alternatives...")
            reset_collection(edb.get_alternatives_db(), user.uuid, new_uuid)
            logging.debug("Resetting analysis...")
            reset_collection(edb.get_analysis_timeseries_db(), user.uuid, new_uuid)
            logging.debug("Resetting client...")
            reset_collection(edb.get_client_db(), user.uuid, new_uuid)
            logging.debug("Resetting client_stats_backup...")
            reset_collection(edb.get_client_stats_db_backup(), user.uuid, new_uuid)
            logging.debug("Resetting server_stats_backup...")
            reset_collection(edb.get_server_stats_db_backup(), user.uuid, new_uuid)
            logging.debug("Resetting result_stats_backup...")
            reset_collection(edb.get_result_stats_db_backup(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_common_place_db...")
            reset_collection(edb.get_common_place_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_common_trip_db...")
            reset_collection(edb.get_common_trip_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_habitica_db...")
            reset_collection(edb.get_habitica_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_pipeline_state_db...")
            reset_collection(edb.get_pipeline_state_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_profile_db...")
            reset_collection(edb.get_profile_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_timeseries_db...")
            reset_collection(edb.get_timeseries_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_timeseries_error_db...")
            reset_collection(edb.get_timeseries_error_db(), user.uuid, new_uuid)
            logging.debug("Resetting edb.get_usercache_db...")
            reset_collection(edb.get_usercache_db(), user.uuid, new_uuid)
Example #27
0
 def getProfile(self):
   return get_profile_db().find_one({'user_id': self.uuid})
Example #28
0
 def setClientSpecificProfileFields(self, setQuery):
   logging.debug("Changing profile for user %s to %s" % (self.uuid, setQuery))
   get_profile_db().update({'user_id': self.uuid}, {'$set': setQuery})
Example #29
0
 def update(self, update_doc):
   logging.debug("Updating user %s with fields %s" % (self.uuid, update_doc))
   get_profile_db().update_one({'user_id': self.uuid}, {'$set': update_doc})
Example #30
0
def getDistinctUserCount():
    Profiles=get_profile_db()
    distinctUserCount = len(Profiles.distinct("user_id"))
    logging.debug("Found %s distinct users " % distinctUserCount)
    return distinctUserCount