def getOrgContribAndLogin(self, date, org, prev_days=31):
keyname_log = "{}:{}"
keyname_contrib = "{}:{}"
data = []
for curDate in util.getXPrevDaysSpan(date, prev_days):
log = self.serv_redis_db.zscore(
keyname_log.format(self.keyOrgLog,
util.getDateStrFormat(curDate)), org)
log = 0 if log is None else 1
contrib = self.serv_redis_db.zscore(
keyname_contrib.format(self.keyContribDay,
util.getDateStrFormat(curDate)), org)
contrib = 0 if contrib is None else 1
data.append([log, contrib])
return data
def addContributionToCateg(self, date, categ, org, count=1):
today_str = util.getDateStrFormat(date)
keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
self.serv_redis_db.zincrby(keyname, org, count)
self.logger.debug(
'Added to redis: keyname={}, org={}, count={}'.format(
keyname, org, count))
def getCoordsByRadius(self, dateStart, dateEnd, centerLat, centerLon, radius):
dico_coord = {}
to_return = []
delta = dateEnd - dateStart
for i in range(delta.days+1):
correctDatetime = dateStart + datetime.timedelta(days=i)
date_str = util.getDateStrFormat(correctDatetime)
keyname = "{}:{}".format(self.keyCategRad, date_str)
res = self.serv_redis_db.georadius(keyname, centerLon, centerLat, radius, unit='km', withcoord=True)
#sum up really close coord
for data, coord in res:
flag_added = False
coord = [coord[0], coord[1]]
#list all coord
for dicoCoordStr in dico_coord.keys():
dicoCoord = json.loads(dicoCoordStr)
#if curCoord close to coord
if self.isCloseTo(dicoCoord, coord):
#add data to dico coord
dico_coord[dicoCoordStr].append(data)
flag_added = True
break
# coord not in dic
if not flag_added:
dico_coord[str(coord)] = [data]
for dicoCoord, array in dico_coord.items():
dicoCoord = json.loads(dicoCoord)
to_return.append([array, dicoCoord])
return to_return
def getDates(self, org, date=None):
keyname = "{}:{}".format(self.keyTimestamp, org)
timestamps = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True)
if date is None:
to_return = [ datetime.datetime.fromtimestamp(float(t[1])) for t in timestamps ]
else:
to_return = []
for t in timestamps:
t = datetime.datetime.fromtimestamp(float(t[1]))
if util.getDateStrFormat(t) == util.getDateStrFormat(date): #same day
to_return.append(t)
elif util.getDateStrFormat(t) > util.getDateStrFormat(date):
continue # timestamps should be sorted, skipping to reach wanted date
else:
break # timestamps should be sorted, no need to process anymore
return to_return
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
self.serv_redis_db.zincrby(keyname, toAdd, count)
self.logger.debug(
'Added to redis: keyname={}, toAdd={}, count={}'.format(
keyname, toAdd, count))
def push_to_redis_geo(self, keyCateg, lon, lat, content):
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}".format(keyCateg, today_str)
self.serv_redis_db.geoadd(keyname, lon, lat, content)
self.logger.debug(
'Added to redis: keyname={}, lon={}, lat={}, content={}'.format(
keyname, lon, lat, content))
def getAllLoggedInOrgs(self, date, prev_days=31):
orgs = set()
for curDate in util.getXPrevDaysSpan(date, prev_days):
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for org in data:
orgs.add(org.decode('utf8'))
return list(orgs)
def getUserLogins(self, date):
keyname = "{}:{}".format(self.keyTimestamp,
util.getDateStrFormat(date))
timestamps = self.serv_redis_db.smembers(keyname)
timestamps = [
int(timestamp.decode('utf8')) for timestamp in timestamps
]
return timestamps
def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6):
dico_hours_contrib = {}
dico_hours = {}
for curDate in util.getXPrevHoursSpan(date, prev_days * 24):
dico_hours[util.getTimestamp(
curDate)] = 0 # populate with empty data
dico_hours_contrib[util.getTimestamp(
curDate)] = 0 # populate with empty data
for curDate in util.getXPrevDaysSpan(date, prev_days):
if org is None:
dates = self.getUserLogins(curDate)
else:
dates = self.getDates(org, date=curDate)
keyname = "{}:{}".format(self.keyContribDay,
util.getDateStrFormat(curDate))
if org is None:
orgs_contri = self.serv_redis_db.zrange(keyname,
0,
-1,
desc=True,
withscores=True)
orgs_contri_num = 0
for _, count in orgs_contri:
orgs_contri_num += count
else:
orgs_contri_num = self.serv_redis_db.zscore(keyname, org)
orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0
for curDate in util.getHoursSpanOfDate(
curDate, adaptToFitCurrentTime=True): #fill hole day
dico_hours_contrib[util.getTimestamp(
curDate)] = orgs_contri_num
for d in dates: # sum occurence during the current hour
dateTimestamp = d.replace(minute=0, second=0, microsecond=0)
try:
dico_hours[util.getTimestamp(dateTimestamp)] += 1
except KeyError: # timestamp out of bound (greater than 1 week)
pass
# Format data
# login
to_ret = {}
data = []
for curDate, occ in dico_hours.items():
data.append([curDate, occ])
data.sort(key=lambda x: x[0])
to_ret['login'] = data
# contrib
data = []
for curDate, occ in dico_hours_contrib.items():
data.append([curDate, occ])
data.sort(key=lambda x: x[0])
to_ret['contrib'] = data
return to_ret
def push_to_redis_zset(self, keyCateg, toAdd, endSubkey="", count=1):
if not isinstance(toAdd, str):
self.logger.warning('Can\'t add to redis, element is not of type String. {}'.format(type(toAdd)))
return
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}{}".format(keyCateg, today_str, endSubkey)
self.serv_redis_db.zincrby(keyname, count, toAdd)
self.logger.debug('Added to redis: keyname={}, toAdd={}, count={}'.format(keyname, toAdd, count))
def getOrgslogin(self, date, topNum=12):
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(date))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
return data
def handleContribution(self, zmq_name, org, contribType, categ, action, pntMultiplier=1, eventTime=datetime.datetime.now(), isLabeled=False):
if action in ['edit', None]:
pass
#return #not a contribution?
now = datetime.datetime.now()
nowSec = int(time.time())
pnts_to_add = self.default_pnts_per_contribution
# Do not consider contribution as login anymore
#self.users_helper.add_user_login(nowSec, org)
# is a valid contribution
if categ is not None:
try:
pnts_to_add = self.DICO_PNTS_REWARD[util.noSpaceLower(categ)]
except KeyError:
pnts_to_add = self.default_pnts_per_contribution
pnts_to_add *= pntMultiplier
util.push_to_redis_zset(self.serv_redis_db, self.keyDay, org, count=pnts_to_add)
#CONTRIB_CATEG retain the contribution per category, not the point earned in this categ
util.push_to_redis_zset(self.serv_redis_db, self.keyCateg, org, count=1, endSubkey=':'+util.noSpaceLower(categ))
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'categ': categ, 'action': action, 'epoch': nowSec }, channel=self.CHANNEL_LASTCONTRIB)
else:
categ = ""
self.serv_redis_db.sadd(self.keyAllOrg, org)
keyname = "{}:{}".format(self.keyLastContrib, util.getDateStrFormat(now))
self.serv_redis_db.zadd(keyname, nowSec, org)
self.logger.debug('Added to redis: keyname={}, nowSec={}, org={}'.format(keyname, nowSec, org))
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
awards_given = self.updateOrgContributionRank(org, pnts_to_add, action, contribType, eventTime=datetime.datetime.now(), isLabeled=isLabeled, categ=util.noSpaceLower(categ))
for award in awards_given:
# update awards given
keyname = "{}:{}".format(self.keyLastAward, util.getDateStrFormat(now))
self.serv_redis_db.zadd(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec }))
self.logger.debug('Added to redis: keyname={}, nowSec={}, content={}'.format(keyname, nowSec, json.dumps({'org': org, 'award': award, 'epoch': nowSec })))
self.serv_redis_db.expire(keyname, util.ONE_DAY*7) #expire after 7 day
# publish
self.publish_log(zmq_name, 'CONTRIBUTION', {'org': org, 'award': award, 'epoch': nowSec }, channel=self.CHANNEL_LASTAWARDS)
def getOrgPntFromRedis(self, org, date):
scoreSum = 0
for curDate in util.getMonthSpan(date):
date_str = util.getDateStrFormat(curDate)
keyname = "{}:{}".format(self.keyDay, date_str)
data = self.serv_redis_db.zscore(keyname, org)
if data is None:
data = 0
scoreSum += data
return scoreSum
def getTrendingSightings(self, dateS, dateE):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keySigh,
util.getDateStrFormat(curDate))
sight = self.serv_redis_db.get(keyname)
sight = 0 if sight is None else int(sight.decode('utf8'))
keyname = "{}:{}".format(self.keyFalse,
util.getDateStrFormat(curDate))
fp = self.serv_redis_db.get(keyname)
fp = 0 if fp is None else int(fp.decode('utf8'))
to_ret.append([
util.getTimestamp(curDate), {
'sightings': sight,
'false_positive': fp
}
])
return to_ret
def push_to_redis_geo(self, keyCateg, lon, lat, content):
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}".format(keyCateg, today_str)
try:
self.serv_redis_db.geoadd(keyname, lon, lat, content)
except redis.exceptions.ResponseError as error:
print(error)
print("Please fix the above, and make sure you use a redis version that supports the GEOADD command.")
print("To test for support: echo \"help GEOADD\"| redis-cli")
self.logger.debug('Added to redis: keyname={}, lon={}, lat={}, content={}'.format(keyname, lon, lat, content))
def addGenericTrending(self, trendingType, data, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname = "{}:{}".format(trendingType, timestampDate_str)
if isinstance(data, OrderedDict):
to_save = json.dumps(data)
else:
to_save = data
self.serv_redis_db.zincrby(keyname, to_save, 1)
self.logger.debug('Added to redis: keyname={}, content={}'.format(
keyname, to_save))
def add_user_login(self, timestamp, org):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
if not self.hasAlreadyBeenAdded(org, timestamp):
keyname_timestamp = "{}:{}".format(self.keyTimestamp,
timestampDate_str)
self.serv_redis_db.sadd(keyname_timestamp, timestamp)
self.addTemporary(org, timestamp)
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
self.serv_redis_db.zincrby(keyname_org, org, 1)
def getOrgOvertime(self, org):
overtime = []
today = datetime.datetime.today()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
for curDate in util.getXPrevDaysSpan(today, 7):
timestamp = util.getTimestamp(curDate)
keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate))
org_score = self.serv_redis_db.zscore(keyname, org)
if org_score is None:
org_score = 0
overtime.append([timestamp, org_score])
to_return = {'label': org, 'data': overtime}
return to_return
def getLoginVSCOntribution(self, date):
keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(date))
orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=False)
orgs_contri = [ org.decode('utf8') for org in orgs_contri ]
orgs_login = [ org for org in self.getAllLoggedInOrgs(date, prev_days=0) ]
contributed_num = 0
non_contributed_num = 0
for org in orgs_login:
if org in orgs_contri:
contributed_num += 1
else:
non_contributed_num +=1
return [contributed_num, non_contributed_num]
def getGenericTrendingOvertime(self, dateS, dateE, choice=None, topNum=0):
if choice == 'categs':
trendingType = self.keyCateg
elif choice == 'tags':
trendingType = self.keyTag
else:
trendingType = self.keyEvent
dico_items = {}
to_format = []
prev_days = (dateE - dateS).days
# get data
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
data = data if data is not None else []
to_format.append([util.getTimestamp(curDate), data])
for timestamp, array in to_format:
for item, _ in array:
if item not in dico_items:
dico_items[item] = []
dico_items[item].append(timestamp)
# sort timestamps in correct order
for item in dico_items.keys():
dico_items[item].sort()
# dico_items have the form: {item: [t1,t2,t4], ...}
to_ret = []
ONEDAY = 60 * 60 * 24
for item, timestamps in dico_items.items():
obj = {
'name': item,
'start': timestamps[0],
'end': timestamps[0] + ONEDAY
}
for t in timestamps:
if t - obj['end'] > ONEDAY: #new entry
to_ret.append(copy.deepcopy(obj))
obj['start'] = t
obj['end'] = t + ONEDAY
else: # contrinue entry
obj['end'] = t + ONEDAY
to_ret.append(obj)
return to_ret
def add_user_login(self, timestamp, org):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname_timestamp = "{}:{}".format(self.keyTimestamp, org)
self.serv_redis_db.zadd(keyname_timestamp, timestamp, timestamp)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_timestamp, timestamp))
keyname_org = "{}:{}".format(self.keyOrgLog, timestampDate_str)
self.serv_redis_db.zincrby(keyname_org, org, 1)
self.logger.debug('Added to redis: keyname={}, org={}'.format(keyname_org, org))
self.serv_redis_db.sadd(self.keyAllOrgLog, org)
self.logger.debug('Added to redis: keyname={}, org={}'.format(self.keyAllOrgLog, org))
def getSpecificTrending(self,
trendingType,
dateS,
dateE,
specificLabel=''):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zscore(keyname, specificLabel)
data = [[specificLabel, data]] if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getGenericTrending(self, trendingType, dateS, dateE, topNum=0):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
data = data if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getCategPerContribFromRedis(self, date):
topNum = 0 # all
contrib_org = self.getTopContributorFromRedis(date)
for dic in contrib_org:
org = dic['org']
for categ in self.categories_in_datatable:
categ_score = 0
for curDate in util.getMonthSpan(date):
keyname = "{}:{}:{}".format(self.keyCateg, util.getDateStrFormat(curDate), categ)
temp = self.serv_redis_db.zscore(keyname, org)
if temp is None:
temp = 0
categ_score += temp
dic[categ] = categ_score
return contrib_org
def getTrendingTags(self, dateS, dateE, topNum=12):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keyTag,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
data = data if data is not None else []
temp = []
for jText, score in data:
temp.append([json.loads(jText), score])
data = temp
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getTypeaheadData(self, dateS, dateE):
to_ret = {}
for trendingType in [self.keyEvent, self.keyCateg]:
allSet = set()
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for elem in data:
allSet.add(elem.decode('utf8'))
to_ret[trendingType] = list(allSet)
tags = self.getTrendingTags(dateS, dateE)
tagSet = set()
for item in tags:
theDate, tagList = item
for tag in tagList:
tag = tag[0]
tagSet.add(tag['name'])
to_ret[self.keyTag] = list(tagSet)
return to_ret
def addFalsePositive(self, timestamp):
timestampDate = datetime.datetime.fromtimestamp(float(timestamp))
timestampDate_str = util.getDateStrFormat(timestampDate)
keyname = "{}:{}".format(self.keyFalse, timestampDate_str)
self.serv_redis_db.incrby(keyname, 1)
self.logger.debug('Incrby: keyname={}'.format(keyname))
def push_to_redis_geo(self, keyCateg, lon, lat, content):
now = datetime.datetime.now()
today_str = util.getDateStrFormat(now)
keyname = "{}:{}".format(keyCateg, today_str)
self.serv_redis_db.geoadd(keyname, lon, lat, content)
def main():
if len(sys.argv) > 1:
org = sys.argv[1]
else:
org = input('Enter the organisation name: ')
printOrgInfo(org)
ContributionStatus = chelper.getCurrentContributionStatus(org)
OLD_org_c_status = ContributionStatus['status']
OLD_org_honor_badge = chelper.getOrgHonorBadges(org)
OLD_org_trophy = chelper.getOrgTrophies(org)
# ranks
while True:
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
userRep = input(
"Enter the organisation RANK to give/remove to {} (<ENTER> to finish): "
.format(org))
if userRep == '':
break
else:
# validate input
try: #not int
rankNum = int(userRep)
except:
print('Not an integer')
continue
if rankNum < 1 or rankNum > chelper.org_rank_maxLevel:
print('Not a valid rank')
continue
if org_c_status['status'][rankNum] == 1: #remove rank
chelper.removeContribRankFromOrg(org, rankNum)
else:
chelper.giveContribRankToOrg(org, rankNum)
printOrgInfo(org)
# badges
while True:
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
userRep = input(
"Enter the organisation BADGE to give/remove to {} (<ENTER> to finish): "
.format(org))
if userRep == '':
break
else:
# validate input
try: #not int
badgeNum = int(userRep)
except:
print('Not an integer')
continue
if badgeNum < 1 and badgeNum > chelper.honorBadgeNum:
print('Not a valid rank')
continue
if badgeNum in org_honor_badge: #remove badge
chelper.removeBadgeFromOrg(org, badgeNum)
else:
chelper.giveBadgeToOrg(org, badgeNum)
printOrgInfo(org)
# trophy
while True:
org_pnts = chelper.getOrgContributionTotalPoints(org)
org_c_rank = chelper.getOrgContributionRank(org)
org_c_status = chelper.getCurrentContributionStatus(org)
org_honor_badge = chelper.getOrgHonorBadges(org)
org_trophy = chelper.getOrgTrophies(org)
print()
for i, categ in enumerate(chelper.categories_in_trophy):
print("{}. {}".format(i, categ))
userCateg = input(
"Enter the CATEGORY in which to add/remove trophy points: ")
if userCateg == '':
break
try: #not int
userCateg = int(userCateg)
except:
print('Not an integer')
continue
if userCateg < 1 and userCateg > len(chelper.categories_in_trophy):
print('Not a valid rank')
continue
categ = chelper.categories_in_trophy[userCateg]
userRep = input(
"Enter the TROPHY POINTS to give/remove to {} (<ENTER> to finish) in {}: "
.format(org, categ))
if userRep == '':
break
else:
# validate input
try: #not int
trophyPnts = int(userRep)
except:
print('Not an integer')
continue
chelper.giveTrophyPointsToOrg(org, categ, trophyPnts)
printOrgInfo(org)
now = datetime.datetime.now()
nowSec = int(time.time())
ContributionStatus = chelper.getCurrentContributionStatus(org)
NEW_org_c_status = ContributionStatus['status']
NEW_org_honor_badge = chelper.getOrgHonorBadges(org)
NEW_org_trophy = chelper.getOrgTrophies(org)
awards_given = []
for i in NEW_org_c_status.keys():
if OLD_org_c_status[i] < NEW_org_c_status[
i] and i != ContributionStatus['rank']:
awards_given.append(
['contribution_status', ContributionStatus['rank']])
for badgeNum in NEW_org_honor_badge:
if badgeNum not in OLD_org_honor_badge:
awards_given.append(['badge', badgeNum])
temp = {}
for item in OLD_org_trophy:
categ = item['categ']
rank = item['trophy_true_rank']
temp[categ] = rank
for item in NEW_org_trophy:
categ = item['categ']
rank = item['trophy_true_rank']
if rank > temp[categ]:
awards_given.append(['trophy', [categ, rank]])
for award in awards_given:
# update awards given
serv_redis_db.zadd(
'CONTRIB_LAST_AWARDS:' + util.getDateStrFormat(now), {
json.dumps({
'org': org,
'award': award,
'epoch': nowSec
}): nowSec
})
serv_redis_db.expire('CONTRIB_LAST_AWARDS:' +
util.getDateStrFormat(now),
ONE_DAY * 7) #expire after 7 day
# publish
publish_log('GIVE_HONOR_ZMQ', 'CONTRIBUTION', {
'org': org,
'award': award,
'epoch': nowSec
}, CHANNEL_LASTAWARDS)
def addContributionToCateg(self, date, categ, org, count=1):
today_str = util.getDateStrFormat(date)
keyname = "{}:{}:{}".format(self.keyCateg, today_str, categ)
self.serv_redis_db.zincrby(keyname, org, count)
