def getUserLoginsForPunchCard(self, date, prev_days=6):
week = {}
for curDate in util.getXPrevDaysSpan(date, prev_days):
timestamps = self.getUserLogins(curDate)
day = {}
for timestamp in timestamps:
date = datetime.datetime.fromtimestamp(float(timestamp))
if date.hour not in day:
day[date.hour] = 0
day[date.hour] += 1
week[curDate.weekday()] = day
# Format data
data = []
for d in range(7):
try:
to_append = []
for h in range(24):
try:
to_append.append(week[d][h])
except KeyError:
to_append.append(0)
data.append(to_append)
except KeyError: # no data
data.append([0 for x in range(24)])
# swap: punchcard day starts on monday
data = [data[6]] + data[:6]
return data
def getUserLoginsForPunchCard(self, date, org=None, prev_days=6):
week = {}
for curDate in util.getXPrevDaysSpan(date, prev_days):
if org is None:
dates = self.getUserLogins(curDate)
else:
dates = self.getDates(org, date=curDate)
day = {}
for date in dates:
if date.hour not in day:
day[date.hour] = 0
day[date.hour] += 1
week[curDate.weekday()] = day
# Format data
data = []
for d in range(7):
try:
to_append = []
for h in range(24):
try:
to_append.append(week[d][h])
except KeyError:
to_append.append(0)
data.append(to_append)
except KeyError: # no data
data.append([0 for x in range(24)])
# swap: punchcard day starts on sunday
data = [data[6]] + data[:6]
return data
def getAllLoggedInOrgs(self, date, prev_days=31):
orgs = set()
for curDate in util.getXPrevDaysSpan(date, prev_days):
keyname = "{}:{}".format(self.keyOrgLog, util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for org in data:
orgs.add(org.decode('utf8'))
return list(orgs)
def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6):
dico_hours_contrib = {}
dico_hours = {}
for curDate in util.getXPrevHoursSpan(date, prev_days * 24):
dico_hours[util.getTimestamp(
curDate)] = 0 # populate with empty data
dico_hours_contrib[util.getTimestamp(
curDate)] = 0 # populate with empty data
for curDate in util.getXPrevDaysSpan(date, prev_days):
if org is None:
dates = self.getUserLogins(curDate)
else:
dates = self.getDates(org, date=curDate)
keyname = "{}:{}".format(self.keyContribDay,
util.getDateStrFormat(curDate))
if org is None:
orgs_contri = self.serv_redis_db.zrange(keyname,
0,
-1,
desc=True,
withscores=True)
orgs_contri_num = 0
for _, count in orgs_contri:
orgs_contri_num += count
else:
orgs_contri_num = self.serv_redis_db.zscore(keyname, org)
orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0
for curDate in util.getHoursSpanOfDate(
curDate, adaptToFitCurrentTime=True): #fill hole day
dico_hours_contrib[util.getTimestamp(
curDate)] = orgs_contri_num
for d in dates: # sum occurence during the current hour
dateTimestamp = d.replace(minute=0, second=0, microsecond=0)
try:
dico_hours[util.getTimestamp(dateTimestamp)] += 1
except KeyError: # timestamp out of bound (greater than 1 week)
pass
# Format data
# login
to_ret = {}
data = []
for curDate, occ in dico_hours.items():
data.append([curDate, occ])
data.sort(key=lambda x: x[0])
to_ret['login'] = data
# contrib
data = []
for curDate, occ in dico_hours_contrib.items():
data.append([curDate, occ])
data.sort(key=lambda x: x[0])
to_ret['contrib'] = data
return to_ret
def getOrgContribAndLogin(self, date, org, prev_days=31):
keyname_log = "{}:{}"
keyname_contrib = "{}:{}"
data = []
for curDate in util.getXPrevDaysSpan(date, prev_days):
log = self.serv_redis_db.zscore(keyname_log.format(self.keyOrgLog, util.getDateStrFormat(curDate)), org)
log = 0 if log is None else 1
contrib = self.serv_redis_db.zscore(keyname_contrib.format(self.keyContribDay, util.getDateStrFormat(curDate)), org)
contrib = 0 if contrib is None else contrib
data.append([log, contrib])
return data
def getOrgOvertime(self, org):
overtime = []
today = datetime.datetime.today()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
for curDate in util.getXPrevDaysSpan(today, 7):
timestamp = util.getTimestamp(curDate)
keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate))
org_score = self.serv_redis_db.zscore(keyname, org)
if org_score is None:
org_score = 0
overtime.append([timestamp, org_score])
to_return = {'label': org, 'data': overtime}
return to_return
def getGenericTrendingOvertime(self, dateS, dateE, choice=None, topNum=0):
if choice == 'categs':
trendingType = self.keyCateg
elif choice == 'tags':
trendingType = self.keyTag
else:
trendingType = self.keyEvent
dico_items = {}
to_format = []
prev_days = (dateE - dateS).days
# get data
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
data = data if data is not None else []
to_format.append([util.getTimestamp(curDate), data])
for timestamp, array in to_format:
for item, _ in array:
if item not in dico_items:
dico_items[item] = []
dico_items[item].append(timestamp)
# sort timestamps in correct order
for item in dico_items.keys():
dico_items[item].sort()
# dico_items have the form: {item: [t1,t2,t4], ...}
to_ret = []
ONEDAY = 60 * 60 * 24
for item, timestamps in dico_items.items():
obj = {
'name': item,
'start': timestamps[0],
'end': timestamps[0] + ONEDAY
}
for t in timestamps:
if t - obj['end'] > ONEDAY: #new entry
to_ret.append(copy.deepcopy(obj))
obj['start'] = t
obj['end'] = t + ONEDAY
else: # contrinue entry
obj['end'] = t + ONEDAY
to_ret.append(obj)
return to_ret
def getSpecificTrending(self,
trendingType,
dateS,
dateE,
specificLabel=''):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zscore(keyname, specificLabel)
data = [[specificLabel, data]] if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getGenericTrending(self, trendingType, dateS, dateE, topNum=0):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
data = data if data is not None else []
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getTrendingSightings(self, dateS, dateE):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keySigh,
util.getDateStrFormat(curDate))
sight = self.serv_redis_db.get(keyname)
sight = 0 if sight is None else int(sight.decode('utf8'))
keyname = "{}:{}".format(self.keyFalse,
util.getDateStrFormat(curDate))
fp = self.serv_redis_db.get(keyname)
fp = 0 if fp is None else int(fp.decode('utf8'))
to_ret.append([
util.getTimestamp(curDate), {
'sightings': sight,
'false_positive': fp
}
])
return to_ret
def getTrendingTags(self, dateS, dateE, topNum=12):
to_ret = []
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(self.keyTag,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname,
0,
topNum - 1,
desc=True,
withscores=True)
data = [[record[0].decode('utf8'), record[1]] for record in data]
data = data if data is not None else []
temp = []
for jText, score in data:
temp.append([json.loads(jText), score])
data = temp
to_ret.append([util.getTimestamp(curDate), data])
return to_ret
def getLastAwardsFromRedis(self):
date = datetime.datetime.now()
keyname = self.keyLastAward
prev_days = 7
topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
addedOrg = []
data = []
for curDate in util.getXPrevDaysSpan(date, prev_days):
last_awards = getZrange(self.serv_redis_db, keyname, curDate, topNum)
for dico_award, sec in last_awards:
dico_award = json.loads(dico_award)
org = dico_award['org']
dic = {}
dic['orgRank'] = self.getOrgContributionRank(org)['final_rank']
dic['logo_path'] = self.getOrgLogoFromMISP(org)
dic['org'] = org
dic['epoch'] = sec
dic['award'] = dico_award['award']
data.append(dic)
return data
def getTypeaheadData(self, dateS, dateE):
to_ret = {}
for trendingType in [self.keyEvent, self.keyCateg]:
allSet = set()
prev_days = (dateE - dateS).days
for curDate in util.getXPrevDaysSpan(dateE, prev_days):
keyname = "{}:{}".format(trendingType,
util.getDateStrFormat(curDate))
data = self.serv_redis_db.zrange(keyname, 0, -1, desc=True)
for elem in data:
allSet.add(elem.decode('utf8'))
to_ret[trendingType] = list(allSet)
tags = self.getTrendingTags(dateS, dateE)
tagSet = set()
for item in tags:
theDate, tagList = item
for tag in tagList:
tag = tag[0]
tagSet.add(tag['name'])
to_ret[self.keyTag] = list(tagSet)
return to_ret
def getLastContributorsFromRedis(self):
date = datetime.datetime.now()
prev_days = 7
topNum = self.MAX_NUMBER_OF_LAST_CONTRIBUTOR # default Num
addedOrg = []
data = []
for curDate in util.getXPrevDaysSpan(date, prev_days):
last_contrib_org = getZrange(self.serv_redis_db, self.keyLastContrib, curDate, topNum)
for org, sec in last_contrib_org:
if org in addedOrg:
continue
dic = {}
dic['rank'] = self.getOrgRankFromRedis(org, date)
dic['orgRank'] = self.getOrgContributionRank(org)['final_rank']
dic['honorBadge'] = self.getOrgHonorBadges(org)
dic['logo_path'] = self.getOrgLogoFromMISP(org)
dic['org'] = org
dic['pnts'] = self.getOrgPntFromRedis(org, date)
dic['epoch'] = sec
data.append(dic)
addedOrg.append(org)
return data
