def getOrgContribAndLogin(self, date, org, prev_days=31): keyname_log = "{}:{}" keyname_contrib = "{}:{}" data = [] for curDate in util.getXPrevDaysSpan(date, prev_days): log = self.serv_redis_db.zscore( keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org) log = 0 if log is None else 1 contrib = self.serv_redis_db.zscore( keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org) contrib = 0 if contrib is None else 1 data.append([log, contrib]) return data
def addContributionToCateg(self, date, categ, org, count=1): today_str = util.getDateStrFormat(date) keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ) self.serv_redis_db.zincrby(keyname, org, count) self.logger.debug( 'Added to redis: keyname={}, org={}, count={}'.format( keyname, org, count))
def getCoordsByRadius(self, dateStart, dateEnd, centerLat, centerLon, radius): dico_coord = {} to_return = [] delta = dateEnd - dateStart for i in range(delta.days+1): correctDatetime = dateStart + datetime.timedelta(days=i) date_str = util.getDateStrFormat(correctDatetime) keyname = "{}:{}".format(self.keyCategRad, date_str) res = self.serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True) #sum up really close coord for data, coord in res: flag_added = False coord = [coord[0], coord[1]] #list all coord for dicoCoordStr in dico_coord.keys(): dicoCoord = json.loads(dicoCoordStr) #if curCoord close to coord if self.isCloseTo(dicoCoord, coord): #add data to dico coord dico_coord[dicoCoordStr].append(data) flag_added = True break # coord not in dic if not flag_added: dico_coord[str(coord)] = [data] for dicoCoord, array in dico_coord.items(): dicoCoord = json.loads(dicoCoord) to_return.append([array, dicoCoord]) return to_return
def getDates(self, org, date=None): keyname = "{}:{}".format(self.keyTimestamp, org) timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) if date is None: to_return = [ datetime.datetime.fromtimestamp(float(t[1])) for t in timestamps ] else: to_return = [] for t in timestamps: t = datetime.datetime.fromtimestamp(float(t[1])) if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day to_return.append(t) elif util.getDateStrFormat(t) > util.getDateStrFormat(date): continue # timestamps should be sorted, skipping to reach wanted date else: break # timestamps should be sorted, no need to process anymore return to_return
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1): now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) self.serv_redis_db.zincrby(keyname, toAdd, count) self.logger.debug( 'Added to redis: keyname={}, toAdd={}, count={}'.format( keyname, toAdd, count))
def push_to_redis_geo(self, keyCateg, lon, lat, content): now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}".format(keyCateg, today_str) self.serv_redis_db.geoadd(keyname, lon, lat, content) self.logger.debug( 'Added to redis: keyname={}, lon={}, lat={}, content={}'.format( keyname, lon, lat, content))
def getAllLoggedInOrgs(self, date, prev_days=31): orgs = set() for curDate in util.getXPrevDaysSpan(date, prev_days): keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) for org in data: orgs.add(org.decode('utf8')) return list(orgs)
def getUserLogins(self, date): keyname = "{}:{}".format(self.keyTimestamp, util.getDateStrFormat(date)) timestamps = self.serv_redis_db.smembers(keyname) timestamps = [ int(timestamp.decode('utf8')) for timestamp in timestamps ] return timestamps
def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6): dico_hours_contrib = {} dico_hours = {} for curDate in util.getXPrevHoursSpan(date, prev_days * 24): dico_hours[util.getTimestamp( curDate)] = 0 # populate with empty data dico_hours_contrib[util.getTimestamp( curDate)] = 0 # populate with empty data for curDate in util.getXPrevDaysSpan(date, prev_days): if org is None: dates = self.getUserLogins(curDate) else: dates = self.getDates(org, date=curDate) keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate)) if org is None: orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) orgs_contri_num = 0 for _, count in orgs_contri: orgs_contri_num += count else: orgs_contri_num = self.serv_redis_db.zscore(keyname, org) orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0 for curDate in util.getHoursSpanOfDate( curDate, adaptToFitCurrentTime=True): #fill hole day dico_hours_contrib[util.getTimestamp( curDate)] = orgs_contri_num for d in dates: # sum occurence during the current hour dateTimestamp = d.replace(minute=0, second=0, microsecond=0) try: dico_hours[util.getTimestamp(dateTimestamp)] += 1 except KeyError: # timestamp out of bound (greater than 1 week) pass # Format data # login to_ret = {} data = [] for curDate, occ in dico_hours.items(): data.append([curDate, occ]) data.sort(key=lambda x: x[0]) to_ret['login'] = data # contrib data = [] for curDate, occ in dico_hours_contrib.items(): data.append([curDate, occ]) data.sort(key=lambda x: x[0]) to_ret['contrib'] = data return to_ret
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1): if not isinstance(toAdd, str): self.logger.warning('Can\'t add to redis, element is not of type String. {}'.format(type(toAdd))) return now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey) self.serv_redis_db.zincrby(keyname, count, toAdd) self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
def getOrgslogin(self, date, topNum=12): keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] return data
def handleContribution(self, zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False): if action in ['edit', None]: pass #return #not a contribution? now = datetime.datetime.now() nowSec = int(time.time()) pnts_to_add = self.default_pnts_per_contribution # Do not consider contribution as login anymore #self.users_helper.add_user_login(nowSec, org) # is a valid contribution if categ is not None: try: pnts_to_add = self.DICO_PNTS_REWARD[util.noSpaceLower(categ)] except KeyError: pnts_to_add = self.default_pnts_per_contribution pnts_to_add *= pntMultiplier util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add) #CONTRIB_CATEG retain the contribution per category, not the point earned in this categ util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ)) self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB) else: categ = "" self.serv_redis_db.sadd(self.keyAllOrg, org) keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now)) self.serv_redis_db.zadd(keyname, nowSec, org) self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org)) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ)) for award in awards_given: # update awards given keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now)) self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })) self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))) self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day # publish self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS)
def getOrgPntFromRedis(self, org, date): scoreSum = 0 for curDate in util.getMonthSpan(date): date_str = util.getDateStrFormat(curDate) keyname = "{}:{}".format(self.keyDay, date_str) data = self.serv_redis_db.zscore(keyname, org) if data is None: data = 0 scoreSum += data return scoreSum
def getTrendingSightings(self, dateS, dateE): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate)) sight = self.serv_redis_db.get(keyname) sight = 0 if sight is None else int(sight.decode('utf8')) keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate)) fp = self.serv_redis_db.get(keyname) fp = 0 if fp is None else int(fp.decode('utf8')) to_ret.append([ util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp } ]) return to_ret
def push_to_redis_geo(self, keyCateg, lon, lat, content): now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}".format(keyCateg, today_str) try: self.serv_redis_db.geoadd(keyname, lon, lat, content) except redis.exceptions.ResponseError as error: print(error) print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.") print("To test for support: echo \"help GEOADD\"| redis-cli") self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
def addGenericTrending(self, trendingType, data, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) keyname = "{}:{}".format(trendingType, timestampDate_str) if isinstance(data, OrderedDict): to_save = json.dumps(data) else: to_save = data self.serv_redis_db.zincrby(keyname, to_save, 1) self.logger.debug('Added to redis: keyname={}, content={}'.format( keyname, to_save))
def add_user_login(self, timestamp, org): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) if not self.hasAlreadyBeenAdded(org, timestamp): keyname_timestamp = "{}:{}".format(self.keyTimestamp, timestampDate_str) self.serv_redis_db.sadd(keyname_timestamp, timestamp) self.addTemporary(org, timestamp) keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) self.serv_redis_db.zincrby(keyname_org, org, 1)
def getOrgOvertime(self, org): overtime = [] today = datetime.datetime.today() today = today.replace(hour=0, minute=0, second=0, microsecond=0) for curDate in util.getXPrevDaysSpan(today, 7): timestamp = util.getTimestamp(curDate) keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate)) org_score = self.serv_redis_db.zscore(keyname, org) if org_score is None: org_score = 0 overtime.append([timestamp, org_score]) to_return = {'label': org, 'data': overtime} return to_return
def getLoginVSCOntribution(self, date): keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date)) orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False) orgs_contri = [ org.decode('utf8') for org in orgs_contri ] orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ] contributed_num = 0 non_contributed_num = 0 for org in orgs_login: if org in orgs_contri: contributed_num += 1 else: non_contributed_num +=1 return [contributed_num, non_contributed_num]
def getGenericTrendingOvertime(self, dateS, dateE, choice=None, topNum=0): if choice == 'categs': trendingType = self.keyCateg elif choice == 'tags': trendingType = self.keyTag else: trendingType = self.keyEvent dico_items = {} to_format = [] prev_days = (dateE - dateS).days # get data for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] data = data if data is not None else [] to_format.append([util.getTimestamp(curDate), data]) for timestamp, array in to_format: for item, _ in array: if item not in dico_items: dico_items[item] = [] dico_items[item].append(timestamp) # sort timestamps in correct order for item in dico_items.keys(): dico_items[item].sort() # dico_items have the form: {item: [t1,t2,t4], ...} to_ret = [] ONEDAY = 60 * 60 * 24 for item, timestamps in dico_items.items(): obj = { 'name': item, 'start': timestamps[0], 'end': timestamps[0] + ONEDAY } for t in timestamps: if t - obj['end'] > ONEDAY: #new entry to_ret.append(copy.deepcopy(obj)) obj['start'] = t obj['end'] = t + ONEDAY else: # contrinue entry obj['end'] = t + ONEDAY to_ret.append(obj) return to_ret
def add_user_login(self, timestamp, org): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) keyname_timestamp = "{}:{}".format(self.keyTimestamp, org) self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp)) keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str) self.serv_redis_db.zincrby(keyname_org, org, 1) self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org)) self.serv_redis_db.sadd(self.keyAllOrgLog, org) self.logger.debug('Added to redis: keyname={}, org={}'.format(self.keyAllOrgLog, org))
def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zscore(keyname, specificLabel) data = [[specificLabel, data]] if data is not None else [] to_ret.append([util.getTimestamp(curDate), data]) return to_ret
def getGenericTrending(self, trendingType, dateS, dateE, topNum=0): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] data = data if data is not None else [] to_ret.append([util.getTimestamp(curDate), data]) return to_ret
def getCategPerContribFromRedis(self, date): topNum = 0 # all contrib_org = self.getTopContributorFromRedis(date) for dic in contrib_org: org = dic['org'] for categ in self.categories_in_datatable: categ_score = 0 for curDate in util.getMonthSpan(date): keyname = "{}:{}:{}".format(self.keyCateg, util.getDateStrFormat(curDate), categ) temp = self.serv_redis_db.zscore(keyname, org) if temp is None: temp = 0 categ_score += temp dic[categ] = categ_score return contrib_org
def getTrendingTags(self, dateS, dateE, topNum=12): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] data = data if data is not None else [] temp = [] for jText, score in data: temp.append([json.loads(jText), score]) data = temp to_ret.append([util.getTimestamp(curDate), data]) return to_ret
def getTypeaheadData(self, dateS, dateE): to_ret = {} for trendingType in [self.keyEvent, self.keyCateg]: allSet = set() prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True) for elem in data: allSet.add(elem.decode('utf8')) to_ret[trendingType] = list(allSet) tags = self.getTrendingTags(dateS, dateE) tagSet = set() for item in tags: theDate, tagList = item for tag in tagList: tag = tag[0] tagSet.add(tag['name']) to_ret[self.keyTag] = list(tagSet) return to_ret
def addFalsePositive(self, timestamp): timestampDate = datetime.datetime.fromtimestamp(float(timestamp)) timestampDate_str = util.getDateStrFormat(timestampDate) keyname = "{}:{}".format(self.keyFalse, timestampDate_str) self.serv_redis_db.incrby(keyname, 1) self.logger.debug('Incrby: keyname={}'.format(keyname))
def push_to_redis_geo(self, keyCateg, lon, lat, content): now = datetime.datetime.now() today_str = util.getDateStrFormat(now) keyname = "{}:{}".format(keyCateg, today_str) self.serv_redis_db.geoadd(keyname, lon, lat, content)
def main(): if len(sys.argv) > 1: org = sys.argv[1] else: org = input('Enter the organisation name: ') printOrgInfo(org) ContributionStatus = chelper.getCurrentContributionStatus(org) OLD_org_c_status = ContributionStatus['status'] OLD_org_honor_badge = chelper.getOrgHonorBadges(org) OLD_org_trophy = chelper.getOrgTrophies(org) # ranks while True: org_pnts = chelper.getOrgContributionTotalPoints(org) org_c_rank = chelper.getOrgContributionRank(org) org_c_status = chelper.getCurrentContributionStatus(org) org_honor_badge = chelper.getOrgHonorBadges(org) org_trophy = chelper.getOrgTrophies(org) userRep = input( "Enter the organisation RANK to give/remove to {} (<ENTER> to finish): " .format(org)) if userRep == '': break else: # validate input try: #not int rankNum = int(userRep) except: print('Not an integer') continue if rankNum < 1 or rankNum > chelper.org_rank_maxLevel: print('Not a valid rank') continue if org_c_status['status'][rankNum] == 1: #remove rank chelper.removeContribRankFromOrg(org, rankNum) else: chelper.giveContribRankToOrg(org, rankNum) printOrgInfo(org) # badges while True: org_pnts = chelper.getOrgContributionTotalPoints(org) org_c_rank = chelper.getOrgContributionRank(org) org_c_status = chelper.getCurrentContributionStatus(org) org_honor_badge = chelper.getOrgHonorBadges(org) org_trophy = chelper.getOrgTrophies(org) userRep = input( "Enter the organisation BADGE to give/remove to {} (<ENTER> to finish): " .format(org)) if userRep == '': break else: # validate input try: #not int badgeNum = int(userRep) except: print('Not an integer') continue if badgeNum < 1 and badgeNum > chelper.honorBadgeNum: print('Not a valid rank') continue if badgeNum in org_honor_badge: #remove badge chelper.removeBadgeFromOrg(org, badgeNum) else: chelper.giveBadgeToOrg(org, badgeNum) printOrgInfo(org) # trophy while True: org_pnts = chelper.getOrgContributionTotalPoints(org) org_c_rank = chelper.getOrgContributionRank(org) org_c_status = chelper.getCurrentContributionStatus(org) org_honor_badge = chelper.getOrgHonorBadges(org) org_trophy = chelper.getOrgTrophies(org) print() for i, categ in enumerate(chelper.categories_in_trophy): print("{}. {}".format(i, categ)) userCateg = input( "Enter the CATEGORY in which to add/remove trophy points: ") if userCateg == '': break try: #not int userCateg = int(userCateg) except: print('Not an integer') continue if userCateg < 1 and userCateg > len(chelper.categories_in_trophy): print('Not a valid rank') continue categ = chelper.categories_in_trophy[userCateg] userRep = input( "Enter the TROPHY POINTS to give/remove to {} (<ENTER> to finish) in {}: " .format(org, categ)) if userRep == '': break else: # validate input try: #not int trophyPnts = int(userRep) except: print('Not an integer') continue chelper.giveTrophyPointsToOrg(org, categ, trophyPnts) printOrgInfo(org) now = datetime.datetime.now() nowSec = int(time.time()) ContributionStatus = chelper.getCurrentContributionStatus(org) NEW_org_c_status = ContributionStatus['status'] NEW_org_honor_badge = chelper.getOrgHonorBadges(org) NEW_org_trophy = chelper.getOrgTrophies(org) awards_given = [] for i in NEW_org_c_status.keys(): if OLD_org_c_status[i] < NEW_org_c_status[ i] and i != ContributionStatus['rank']: awards_given.append( ['contribution_status', ContributionStatus['rank']]) for badgeNum in NEW_org_honor_badge: if badgeNum not in OLD_org_honor_badge: awards_given.append(['badge', badgeNum]) temp = {} for item in OLD_org_trophy: categ = item['categ'] rank = item['trophy_true_rank'] temp[categ] = rank for item in NEW_org_trophy: categ = item['categ'] rank = item['trophy_true_rank'] if rank > temp[categ]: awards_given.append(['trophy', [categ, rank]]) for award in awards_given: # update awards given serv_redis_db.zadd( 'CONTRIB_LAST_AWARDS:' + util.getDateStrFormat(now), { json.dumps({ 'org': org, 'award': award, 'epoch': nowSec }): nowSec }) serv_redis_db.expire('CONTRIB_LAST_AWARDS:' + util.getDateStrFormat(now), ONE_DAY * 7) #expire after 7 day # publish publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', { 'org': org, 'award': award, 'epoch': nowSec }, CHANNEL_LASTAWARDS)
def addContributionToCateg(self, date, categ, org, count=1): today_str = util.getDateStrFormat(date) keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ) self.serv_redis_db.zincrby(keyname, org, count)