def connectToSource():
global conn
global counter
cfg = ConfigHelper.getConfig()
if counter <= 0:
conn = psycopg2.connect(host=cfg["DBConnectionString"]["host"],
port=cfg["DBConnectionString"]["port"],
user=cfg["DBConnectionString"]["username"],
password=cfg["DBConnectionString"]["password"],
database=cfg["DBConnectionString"]["database"])
conn.set_session(autocommit=True)
else:
try:
cursor = conn.cursor()
except:
print("It's closed but the Counter is %s" % (counter, ))
counter = 0
return connectToSource()
#conn = psycopg2.connect(host='localhost',port=5432 ,user='******', password='******', database = 'LaserScraper')
#conn = pyodbc.connect('Driver={SQL Server}; Server=CTRI-DESKTOP\SQLEXPRESS; Database=LaserScraper; Trusted_Connection=yes;')
counter = counter + 1
return conn
def buildAllForAllArenasSequentially(jobID=None, startIndex=None):
cfg = cfgH.getConfig()
if jobID == None:
jobID = jobStart(
"Render all blobs", 0,
"buildAllForAllArenasSequentially.buildAllForAllArenasSequentially",
None, len(cfg["configs"]))
if startIndex == None:
startIndex = 0
counter = 0
for config in cfg["configs"]:
if counter >= startIndex:
cfgH.setActive(counter)
BuildMonthlyScoresToJSON.executeMonthlyScoresBuild()
jobHeartbeat(jobID, counter)
BuildMonthlyStarQualityToJSON.executeBuildMonthlyStars()
jobHeartbeat(jobID, counter)
BuildAchievementScoresToJSON.executeAchievementBuild()
BuildPlayerBlob.executeBuildPlayerBlobs(jobID=jobID,
counter=counter)
#BuildHeadToHeadsToJSON.buildHeadToHeads()
BuildAnnualArenaMetrics.executeBuild()
jobHeartbeat(jobID, counter)
BuildAnnualTop3s.execute()
jobHeartbeat(jobID, counter)
BuildAnnualStars.executeBuild_AnnualStars()
jobHeartbeat(jobID, counter)
counter = counter + 1
jobEnd(jobID)
def queueWeekly():
cfg = ConfigHelper.getConfig()
#WEEKLY UPDATE
#66
params = {}
params["scope"] = "activePlayers"
params["arenaName"] = ConfigHelper.getConfigString("SiteNameReal")
targetIDs = SQLHelper.getInterestingPlayersRoster(
False,
ConfigHelper.getConfigString("StartDate"),
ConfigHelper.getConfigString("ChurnDuration"),
siteName=None)
gamesID = SQLHelper.jobStart("Fetch games, All arenas active players ",
0,
"FetchPlayerAndGames.executeQueryGames",
params,
len(targetIDs),
delay=-2)
#67
params = {}
params["scope"] = "activePlayers"
targetIDs = SQLHelper.getInterestingPlayersRoster(
False,
ConfigHelper.getConfigString("StartDate"),
ConfigHelper.getConfigString("ChurnDuration"),
siteName=None)
achievesID = SQLHelper.jobStart(
"Fetch achievements, players from the last 7 days",
0,
"FetchAchievements.executeFetchAchievements",
params,
len(targetIDs),
delay=-2)
SQLHelper.jobBlock(gamesID, achievesID)
#6
renderID = SQLHelper.jobStart(
"Render all blobs",
0,
"buildAllForAllArenasSequentially.buildAllForAllArenasSequentially",
None,
len(cfg["configs"]),
delay=-2)
SQLHelper.jobBlock(achievesID, renderID)
def executeQueryArena(initTargetID):
global config
config = cfg.getConfig() #cache the config
def queueMonthly():
cfg = ConfigHelper.getConfig()
#667 - find new players, this should happen first.
newPlayersIDs = []
for site in cfg["configs"]:
params = {}
params["siteName"] = site["SiteNameReal"]
newPlayersID = SQLHelper.jobStart(
" new players at [%s]" % params["siteName"],
0,
"FetchPlayerUpdatesAndNewPlayers.findNewPlayers",
params,
delay=-2)
newPlayersIDs.append(newPlayersID)
#661 - load summaries. This should happen after the new player updates, before the games updates.
targetIDs = SQLHelper.getPlayers(0)
summaryID = SQLHelper.jobStart(
"Fetch summaries, all known players",
0,
"FetchPlayerUpdatesAndNewPlayers.updateExistingPlayers",
None,
len(targetIDs),
delay=-2)
#666 - load details. This should happen after the summaries updates.
params = {}
params["scope"] = "full"
params["arenaName"] = ConfigHelper.getConfigString("SiteNameReal")
targetIDs = SQLHelper.getInterestingPlayersRoster(
True,
ConfigHelper.getConfigString("StartDate"),
ConfigHelper.getConfigString("ChurnDuration"),
offset=0)
gamesID = SQLHelper.jobStart("Fetch games, all players",
0,
"FetchPlayerAndGames.executeQueryGames",
params,
len(targetIDs),
delay=-2)
for newPlayerID in newPlayersIDs:
SQLHelper.jobBlock(newPlayerID, summaryID)
SQLHelper.jobBlock(summaryID, gamesID)
#677
targetIDs = SQLHelper.getInterestingPlayersRoster(
False,
ConfigHelper.getConfigString("StartDate"),
ConfigHelper.getConfigString("ChurnDuration"),
offset=0)
if len(targetIDs) > 0:
achievesID = SQLHelper.jobStart(
"Fetch achievements, active players",
0,
"FetchAchievements.executeFetchAchievements",
params,
len(targetIDs),
delay=-2)
SQLHelper.jobBlock(gamesID, achievesID)
else:
DBG.DBG(
"WARNING, no known active players at the time of queuing achievements. May need a re-run.",
2)
achievesID = gamesID #sets it so that the render job will be blocked by games, not achieves.
#6
renderID = SQLHelper.jobStart(
"Render all blobs",
0,
"buildAllForAllArenasSequentially.buildAllForAllArenasSequentially",
None,
len(cfg["configs"]),
delay=-2)
SQLHelper.jobBlock(achievesID, renderID)
def execute():
SQL = """
with starQuality as
(
select playerID, gamertag
,round(avg(starsforgame),2) as avgQualityPerGame
,gamemonth
,count(*)::int as gamesPlayed
from public."participationWithStars"
where gameTimestamp >= %s
and gameTimestamp < %s
and arenaName ilike %s
group by 1,2,4
),
GoldenTopX as
(
select PlayerID, gamertag
, ROW_NUMBER() over (partition by gameMonth order by avgQualityPerGame desc) as playerRank
, gameMonth
from StarQuality
where StarQuality.gamesPlayed >= 3
order by AvgQualityPerGame desc
--limit 3
),
GoldenTop3 as
(
select * from GoldenTopX
where playerRank <= 3
)
select * from GoldenTop3 g3 join StarQuality sq on
sq.playerID = g3.playerID and
sq.gameMonth = g3.gameMonth
order by g3.gameMonth asc, g3.playerRank asc
"""
#--startDate,endDate, siteNameReal, name (sen) (sen) (sen) name
cfg = ConfigHelper.getConfig()
startYear = int(cfg["StartDate"][0:4])
endYear = startYear + 1
startYear = "%s-01-01" % startYear
endYear = "%s-01-01" % endYear
#startYear = '2019-08-01'
parameters = (startYear, endYear, cfg['SiteNameReal'])
conn = connectToSource()
cursor = conn.cursor()
cursor.execute(SQL, parameters)
results = cursor.fetchall()
SQLdesc = cursor.description
descCount = 0
for desc in SQLdesc:
#print("%s %s" % (descCount,desc[0]))
descCount = descCount + 1
currentMonth = ""
months = []
for result in results:
if result[3] != currentMonth:
currentMonth = result[3]
#print("== New month = [%s]" % (currentMonth,))
month = {}
months.append(month)
month["month"] = result[3]
players = []
month["players"] = players
#print(result)
player = {}
player["playerName"] = result[5]
player["gamePlayed"] = result[8]
player["averageStars"] = "%s" % (result[6])
players.append(player)
playerName = "%s%s" % (result[5], " " * 15)
playerName = playerName[0:10]
#print("%s %s, %s games played \t %s stars per game (avg) " % (result[1],playerName , result[6], result[8]) )
#print(json.dumps(months,indent=4))
#playerID, rank, month,
#ID, gamertag, avgopponents
#gamesplayed, averagerank, avgqual
#totalqual, avgscore, achievementscore
#gamemonth
filepart = "AnnualTop3s"
if os.name == "nt":
divider = "\\"
elif os.name == "posix":
divider = "/"
f = open(
"JSONBlobs%s%s%s-%s.json" %
(divider, cfg["ID Prefix"], filepart, startYear[0:4]), "w+")
f.write(json.dumps(months, indent=4))
DBG("Annual top3s complete!", 3)
def executeBuildMonthlyStars():
cachedconfig = cfg.getConfig()
curMonth = cachedconfig["StartDate"][0:7]
lastMonth = cachedconfig["LastMonthStart"][0:7]
endDate = cachedconfig["EndDate"]
arenaName = cachedconfig["SiteNameReal"]
SQL = '''
with subsetOfData as
(
select playerID
, gamerTag
, gamemonth
, avg(playercount) as AveragePlayerCount
, count(*) as GamesPlayed
, avg(Starsforgame) as AverageStarQuality
, sum(Starsforgame) as TotalStarQuality
, avg(rank) as AverageRank
from public."participationWithStars"
where arenaName = %s --
and GameMonth in (%s,%s) --
group by 1,2,3
)
select r1.PlayerID, r1.GamerTag,
round (cast(r1.AverageStarQuality as numeric),2) as AverageStarQuality,
round (cast(r1.AverageStarQuality * r1.gamesPlayed as numeric),2) as TotalStarQuality,
round (cast(r1.AveragePlayerCount as numeric),2) as AveragePlayerCount,
round (cast(r1.AverageRank as numeric),2) as AverageRank,
r1.gamesPlayed as GamesPlayed,
round (cast(r2.AverageRank - r1.AverageRank as numeric),2) as changeInRank,
round (cast(r1.AveragePlayerCount-r2.AveragePlayerCount as numeric),2) as changeInPlayers,
round (cast(r1.AverageStarQuality - r2.AverageStarQuality as numeric),2) as changeInStars
from subsetOfData r1 left join subsetOfData r2
on r1.PlayerID = r2.PlayerID and r1.GameMonth != r2.GameMonth
where r1.GameMonth = %s --
order by AverageStarQuality desc
'''
conn = connectToSource()
cursor = conn.cursor()
cursor.execute(SQL, (arenaName, curMonth, lastMonth, curMonth))
JSON = {
'ScoreTitle':
"Star Quality for all known players, between {1} and {0}".format(
curMonth, lastMonth),
'ScoreGreaterOrEqualDate':
curMonth,
'ScoreLessDate':
lastMonth,
'Player': [{
# 'Name' : "C'tri",
# 'AverageScore' : -1,
# 'MissionsPlayed' : -1,
}],
}
breakdownSQL = """
with data as (
select pl.PlayerID, pl.GamerTag, GameName, GameTimestamp, p.score,
ROW_NUMBER() over (partition by GameTimestamp order by GameTimestamp desc, score desc) as rank,
count(p.PlayerID) over (partition by GameTimestamp order by GameTimestamp desc) as playerCount,
TO_CHAR(GameTimestamp,'YYYY-MM') as GameMonth
from Participation p join Games g on p.GameUUID = g.GameUUID
join Players pl on p.PlayerID = pl.PlayerID
where g.ArenaName = %s
)
select gamename, rank, playerCount
, round(cast((cast(playerCount as float) * cast(playerCount as float)) / cast(rank as float) as numeric),2) as stars
from data
where GameMonth = %s
and playerID ilike %s
order by gametimestamp desc
"""
SQResults = cursor.fetchall()
counter = 0
for result in SQResults:
#print (result)
ChangeInRank = None
ChangeInPlayers = None
ChangeInStars = None
if result[7] is not None:
ChangeInRank = "↑%s" % result[7] if result[7] > 0 else "↓%s" % abs(
result[7])
if result[8] is not None:
ChangeInPlayers = "↑%s" % result[8] if result[
8] > 0 else "↓%s" % abs(result[8])
if result[9] is not None:
ChangeInStars = "↑%s" % result[9] if result[
9] > 0 else "↓%s" % abs(result[9])
SQObject = {
'JSID': counter,
'Name': result[1],
'StarQualityPerGame': "%s" % result[2],
'TotalStarQuality': "%s" % result[3],
'AverageOpponents': "%s" % result[4],
'gamesPlayed': result[6],
'AverageRank': "%s" % result[5],
'ChangeInRank': ChangeInRank,
'ChangeInPlayers': ChangeInPlayers,
'ChangeInSQPerGame': ChangeInStars,
'breakdown': []
}
cursor.execute(breakdownSQL, (arenaName, curMonth, result[0]))
breakdownResults = cursor.fetchall()
for breakdownEntry in breakdownResults:
SQBreakdown = {
"gameName": breakdownEntry[0],
"rank": breakdownEntry[1],
"totalPlayers": breakdownEntry[2],
"starsForGame": "%s" % breakdownEntry[3]
}
SQObject['breakdown'].append(SQBreakdown)
JSON['Player'].append(SQObject)
counter = counter + 1
filepart = "StarQuality"
if os.name == "nt":
divider = "\\"
elif os.name == "posix":
divider = "/"
f = open(
"JSONBlobs%s%s%s.json" %
(divider, cfg.getConfigString("ID Prefix"), filepart), "w+")
f.write(json.dumps(JSON, indent=4))
DBG("Star Quality blobs written!", 3)
counter = 1
if len(playerJSON["centre"]) > 1:
for centre in playerJSON["centre"]:
print("[%s] %s" % (counter,centre["name"]))
counter = counter + 1
pickedArena = int(input ("Pick which of these is the main arena: \n"))
else:
pickedArena = 1
pickedArena = pickedArena - 1
print ("\nArena Name: \t\t%s" % (playerJSON["centre"][pickedArena]["name"],))
print ("Arena Prefix: \t\t%s-%s-###" % (matches.groups()[0],matches.groups()[1]))
print(" Leave blank to use Arena Name")
shortName = input ("Arena short name:\t" )
if shortName == "":
shortName = playerJSON["centre"][pickedArena]["name"]
print("Arena short name:\t%s" % (shortName,))
print("\n\n\n")
print(json.dumps(playerJSON,indent=2))
newEntry = {"ID Prefix":"%s-%s-" % (matches.groups()[0],matches.groups()[1]),
"SiteNameReal":"%s" % (playerJSON["centre"][pickedArena]["name"],),
"SiteNameShort" : "%s" % (shortName,)}
cfg.addNewSite(newEntry)
conf = cfg.getConfig()
newID = len(conf["configs"])
cfg.setActive (newID)
print("Run main application, 667, then 661, then 677")
def executeBuildPlayerBlobs(jobID=None, counter=None):
cachedconfig = cfg.getConfig()
targetIDs = getTop5PlayersRoster(cachedconfig["StartDate"],
cachedconfig["EndDate"],
cachedconfig["SiteNameReal"])
DBG("Building big 5 player blobs", 3)
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
#print ("Player profile blobs written!")
JSONobject = {}
if len(targetIDs) >= 1:
#fetchIndividualWithID(targetIDs[0][0])
JSONobject["GoldenPlayer"] = buildPlayerBlob(cachedconfig["StartDate"],
cachedconfig["EndDate"],
targetIDs[0][0])
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
if len(targetIDs) >= 2:
#fetchIndividualWithID(targetIDs[1][0])
JSONobject["SilverPlayer"] = buildPlayerBlob(cachedconfig["StartDate"],
cachedconfig["EndDate"],
targetIDs[1][0])
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
if len(targetIDs) >= 3:
#fetchIndividualWithID(targetIDs[2][0])
JSONobject["BronzePlayer"] = buildPlayerBlob(cachedconfig["StartDate"],
cachedconfig["EndDate"],
targetIDs[2][0])
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
if len(targetIDs) >= 4:
fetchIndividualWithID(targetIDs[3][0])
JSONobject["OtherPlayer1"] = buildPlayerBlob(cachedconfig["StartDate"],
cachedconfig["EndDate"],
targetIDs[3][0])
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
if jobID is not None and counter is not None:
jobHeartbeat(jobID, counter)
if len(targetIDs) >= 5:
fetchIndividualWithID(targetIDs[4][0])
JSONobject["OtherPlayer2"] = buildPlayerBlob(cachedconfig["StartDate"],
cachedconfig["EndDate"],
targetIDs[4][0])
if len(targetIDs) < 5:
DBGstring = "Big 5 returned %i: " % (len(targetIDs))
for target in targetIDs:
DBGstring = DBGstring + "[%i %s]," % (target[2], target[3])
DBG(DBGstring, 2)
JSONobject["SiteNameShort"] = cachedconfig["SiteNameShort"]
filepart = "playerBlob"
if os.name == "nt":
divider = "\\"
elif os.name == "posix":
divider = "/"
f = open(
"JSONBlobs%s%s%s.json" %
(divider, cfg.getConfigString("ID Prefix"), filepart), "w+")
f.write(json.dumps(JSONobject, indent=4))
f.close()
def buildPlayerBlob(startDate, endDate, targetID):
cachedconfig = cfg.getConfig()
infoQuery = """
with StarQuality as (
select playerID
, avg(rank) as AverageRank
, avg(playercount) as AverageOpponents
, round(avg(starsforgame),2) as AvgQualityPerGame
, round(sum(starsforgame),2) as TotalQualityScore
, count(*) as gamesPlayed
from public."participationWithStars"
where gametimestamp >= %s
and gametimestamp < %s
and arenaName = %s
and playerID = %s
group by playerID
),
totalAchievements as (
select sum ( case when achievedDate is null then 0 when achievedDate is not null then 1 end) as AchievementsCompleted, PlayerID
from PlayerAchievement pa join AllAchievements aa on pa.AchID = aa.AchID
where aa.ArenaName = %s
or aa.ArenaName = 'Global Achievements'
group by PlayerID
)
select Players.PlayerID, GamerTag,players.Level, Missions, round(cast(AverageOpponents as numeric),2) as AverageOpponents, gamesPlayed
, AverageRank
, AvgQualityPerGame
, TotalQualityScore
, ta.AchievementsCompleted
, ta.PlayerID as TaPID
, ph.arenaName
, pas.locallevel
, arl.rankName
from Players
join StarQuality sq on sq.playerID = players.playerID
join totalAchievements ta on ta.playerID = players.playerID
join public."playerHomes" ph
on Players.PlayerID = ph.PlayerID
and ph.month = %s
and ph.arenarow = 1
join PlayerArenaSummary pas on pas.playerID = ph.PlayerID and pas.ArenaName = ph.ArenaName
join ArenaRanksLookup arl on arl.ArenaName = pas.ArenaName and arl.ranknumber = pas.localLevel
where Players.playerID = %s
"""
goldenGameQuery = """
with PlayersInGame as (
SELECT
Count (Players.GamerTag) as playersInGame,
Games.GameUUID as gameID
FROM Games
join Participation on participation.GameUUID = Games.GameUUID
join Players on Participation.PlayerID = Players.PlayerID
where games.ArenaName = %s
group by Games.GameUUID
),
Ranks as
(
select GameTimestamp, GameName, Players.PlayerID, GamerTag, Score, Games.GameUUID,
ROW_NUMBER() over (partition by GameTimestamp order by score desc) as gamePosition
from Games
join Participation on Games.GameUUID = Participation.GameUUID
join Players on Participation.PlayerID = Players.PlayerID
where games.ArenaName = %s
),
GoldenGame as (
select r.Score, r.GamerTag,r.GameUUID, GameName, r.PlayerID, gamePosition, playersInGame, GameTimestamp
,round((playersInGame * (playersInGame/gamePosition)),2) as StarQuality
from Ranks r join PlayersInGame pig on r.GameUUID = pig.gameID
where PlayerID = %s
and GameTimestamp >= %s
and GameTimeStamp < %s
order by StarQuality desc, score desc
limit 1
),
Vanquished as (
select g.PlayerID, g.GameTimestamp, g.gamePosition victoryPos, g.GamerTag victorName, g.Score victorScore, g.GameName, g.StarQuality victorStarQuality, r.PlayerID vanquishedID, r.GamerTag vanquishedName ,r.gamePosition as vanquishedPos from
Ranks r inner join GoldenGame g on r.gameUUID = g.GameUUID
where g.PlayerID != r.PlayerID
and g.gamePosition < r.gamePosition
)
select * from Vanquished"""
goldenAchievementQuery = """ with firstEarned as (
select distinct min (achievedDate) over (partition by AchID) as firstAchieved, AchID
from PlayerAchievement
where achievedDate is not null
),
data as (
select count(*) playersEarned, pa.AchID, achName from PlayerAchievement pa join AllAchievements aa on pa.AchID = aa.AchID
where achievedDate is not null
group by AchName, pa.AchID
)
select PlayerID, data.AchName, Description, fe.firstAchieved, playersEarned from PlayerAchievement pa
join data on data.AchID = pa.AchID
join firstEarned fe on fe.AchID = data.AchID
join AllAchievements aa on pa.AchID = aa.AchID
where PlayerID = %s and pa.achievedDate is not null
and ArenaName = %s
order by playersEarned asc, firstAchieved asc
limit 10
"""
fallbackInfoQuery = """ with achs as (select count(*) as achCount
from playerAchievement pa join allachievements aa on aa.achID = pa.achID
where playerID = %s and arenaname = %s
and achieveddate is not null)
select
pl.playerID, Gamertag, '' , pl.missions, '', 0 --monthly missions played
, '', 0.00, '', achCount, '', arl.arenaname, ranknumber, rankname
from players pl join achs on true
join playerarenasummary pas on
pas.arenaname = %s
and pl.playerID = pas.playerID
join arenarankslookup arl on pas.arenaname = arl.arenaname and locallevel = ranknumber
where pl.playerID = %s
"""
conn = connectToSource()
cursor = conn.cursor()
#DBG("BuildPlayerBlob.buildPlayerBlob start[%s], end[%s], target[%s], arena[%s]" % (cachedconfig["StartDate"],cachedconfig["EndDate"],targetID,cachedconfig["SiteNameReal"]),3)
#startDate, endDate, arenaName, startDate, endDate, arenaName, startDate, endDate, arenaName, arenaName, PlayerID
endDate = cachedconfig["EndDate"]
startDate = cachedconfig["StartDate"]
targetArena = cachedconfig["SiteNameReal"]
currentMonth = cachedconfig["StartDate"][:7]
result = cursor.execute(infoQuery,
(startDate, endDate, targetArena, targetID,
targetArena, currentMonth, targetID))
row = cursor.fetchone()
if row == None:
DBG(
"BuildPlayerBlob info query returned Null. Did they play any games? SWITCHING TO FALLBACK. [%s]"
% (targetID), 1)
result = cursor.execute(fallbackInfoQuery,
(targetID, targetArena, targetArena, targetID))
row = cursor.fetchone()
if row == None:
DBG("Fallback infoquery failed. This should not happen, something is wack somewhere. "
)
return
#print(row)
#print ("Players.PlayerID, GamerTag, round(AverageOpponents,2) as AverageOpponents, gamesPlayed, AverageRank")
JSONobject = {}
JSONobject["PlayerName"] = row[1]
JSONobject["HomeArenaTrunc"] = row[11]
JSONobject["SkillLevelName"] = row[13]
JSONobject["MonthlyGamesPlayed"] = row[5]
JSONobject["AllGamesPlayed"] = row[3]
JSONobject["StarQuality"] = "%s" % row[7]
JSONobject["Achievements"] = row[9]
result = cursor.execute(
goldenGameQuery,
(targetArena, targetArena, targetID, startDate, endDate))
rows = cursor.fetchall()
if len(rows) > 0:
row = rows[0]
#print(row)
#print ("g.PlayerID, g.GameTimestamp, victoryPos, victorName, victorScore, g.GameName, victorStarQuality, vanquishedID, vanquishedName , vanquishedPos")
ordinalranks = [
"0th", "1st", "2nd", "3rd", "4th", "5th", "6th", "7th", "8th",
"9th", "10th"
]
JSONobject["GGName"] = row[5]
JSONobject["GGRank"] = ordinalranks[row[2]]
JSONobject["GGStars"] = "%i stars" % row[6]
JSONobject["GGVanq1"] = rows[0][8]
if len(rows) >= 2:
JSONobject["GGVanq2"] = rows[1][8]
if len(rows) >= 3:
JSONobject["GGVanq3"] = rows[2][8]
if len(rows) >= 4:
JSONobject["GGVanq4"] = '%i others' % (len(rows) - 3)
result = cursor.execute(goldenAchievementQuery, (targetID, targetArena))
row = cursor.fetchone()
if row == None:
DBG(
"BuildPlayerBlob GoldenAchievementQuery query returned Null. Aborting. [%s]"
% (targetID), 1)
return
#print (row)
JSONobject["GAName"] = row[1]
JSONobject["GADesc"] = row[2]
ordinalOthers = [
"No one else has", "Only one other person has", "Only two others have",
"%i others have" % row[4]
]
JSONobject["GAOthers"] = ordinalOthers[min(row[4] - 1, 3)]
return JSONobject
def executeBuild():
DBG("Building arena metrics", 3)
conn = connectToSource()
cursor = conn.cursor()
outputObject = {}
cfg = ConfigHelper.getConfig()
startYear = int(cfg["StartDate"][0:4])
endYear = startYear + 1
startYear = "%s-01-01" % startYear
endYear = "%s-01-01" % endYear
#topGames
#print (" == Which games were played")
SQL = """
with gamesbyname as (
select count(*) as gameCount, gamename from games g
where arenaName ilike %s
and gametimestamp >= %s
and gametimestamp < %s
group by 2
order by 1 desc
),
totalGames as (
select sum(gameCount) totalGames
from gamesbyname
)
select cast(gamecount as integer), gamename from gamesbyname full outer join totalgames on true
where (gamecount / totalgames) > 0.01
union
select cast(sum(gamecount) as integer), 'other' from gamesbyname full outer join totalgames on true
where (gamecount / totalgames) <= 0.01
order by 1 desc;
"""
parameters = (cfg['SiteNameReal'], startYear, endYear)
conn = connectToSource()
cursor = conn.cursor()
cursor.execute(SQL, parameters)
results = cursor.fetchall()
SQLdesc = cursor.description
descCount = 0
for desc in SQLdesc:
#print("%s %s" % (descCount,desc[0]))
descCount = descCount + 1
gamesPlayed = []
for result in results:
#print (result)
game = {'gameName': result[1], 'timesPlayed': result[0]}
gamesPlayed.append(game)
outputObject['gamesPlayed'] = gamesPlayed
#referrals
#print (" == Referrals and Welcomers")
SQL = """
with firstGames as
(
select gamerTag, p.playerID, min(gametimestamp) as firstGame
from players pl
join participation p on p.playerID = pl.playerID
join games g on p.gameuuid = g.gameuuid
group by 2,1
),
playersInGames as
(
select count(*) as countOfPlayersInGame, gametimestamp
from players pl
join participation p on p.playerID = pl.playerID
join games g on p.gameuuid = g.gameuuid
group by 2
),
gamesWithNewPlayers as
(
select count (case when gametimestamp = firstgame then 1 else null end) as newPlayers
, count (case when gametimestamp != firstgame then 1 else null end) as existingPlayers
, gametimestamp
, arenaname
from players pl
join participation p on p.playerID = pl.playerID
join games g on p.gameuuid = g.gameuuid
join firstGames fg on pl.playerID = fg.playerID
where gametimestamp >= %s /*StartDate*/ and gametimestamp < %s /*EndDate*/
and arenaname ilike %s /*SiteNameReal*/
group by 3,4
order by 3 desc
),
referredPlayers as
(
select pl.gamertag, g.gametimestamp, firstgame, newplayers, existingplayers,
case when firstgame = g.gametimestamp then 'REFERRED!' else 'referee' end as referalRole
from players pl
join participation p on p.playerID = pl.playerID
join games g on p.gameuuid = g.gameuuid
join firstGames fg on pl.playerID = fg.playerID
join gamesWithNewPlayers gwnp on gwnp.gametimestamp = g.gametimestamp and gwnp.arenaname = g.arenaname
where newplayers > 0 and existingPlayers > 0
order by g.gametimestamp desc
)
select count(distinct gamerTag), 'new player' as playerRole from referredPlayers
where referalrole = 'REFERRED!'
union
select count (distinct gamerTag), 'veteran' from referredPlayers
where referalrole = 'referee'
"""
parameters = (startYear, endYear, cfg['SiteNameReal'])
conn = connectToSource()
cursor = conn.cursor()
cursor.execute(SQL, parameters)
results = cursor.fetchall()
SQLdesc = cursor.description
descCount = 0
for desc in SQLdesc:
#print("%s %s" % (descCount,desc[0]))
descCount = descCount + 1
referrals = {"newPlayers": results[0][0], "welcomers": results[1][0]}
outputObject['referrals'] = referrals
#new and old players
#print (" == New and Departed players")
SQL = """
with playerMetrics as (
select gamertag
, pl.playerID
, arenaname
, min(gametimestamp) as firstGame
, max(gametimestamp) as lastGame
from participation p
join players pl on p.playerID = pl.playerID
join games g on p.gameuuid = g.gameuuid
where arenaName ilike %s --arenaName
group by gamerTag, pl.playerID, arenaName
),
playerMissions as (
select playerID, count(*) as gamesPlayedInPeriod from participation p join games g on p.gameuuid = g.gameuuid
where gametimestamp >= %s --startDate
and gametimestamp < %s --endDate
group by 1
)
select
count (case when gamesPlayedInPeriod > 0 then 1 else null end) as totalPlayingPlayers
, count (case when firstGame >= to_date(%s,'YYYY-MM-DD') then 1 else null end) as newPlayers
, count (case when ((lastGame < to_date(%s,'YYYY-MM-DD') - INTERVAL '%s days' ) and lastGame >= %s ) then 1 else null end ) as churnedPlayers
from playerMetrics pm1 join playerMissions pm2 on pm1.playerID = pm2.playerID
"""
parameters = (cfg['SiteNameReal'], startYear, endYear, startYear, endYear,
cfg['ChurnDuration'], startYear)
conn = connectToSource()
cursor = conn.cursor()
cursor.execute(SQL, parameters)
results = cursor.fetchall()
SQLdesc = cursor.description
descCount = 0
for desc in SQLdesc:
#print("%s %s" % (descCount,desc[0]))
descCount = descCount + 1
playerCounts = {
'activePlayers': results[0][0],
'newPlayers': results[0][1],
'churnedPlayers': results[0][2]
}
outputObject['playerCounts'] = playerCounts
#print(json.dumps(outputObject,indent=4))
#MEMBER PERCENTILES - visits
SQL = """with data as (select
percent_rank() over (order by count(distinct date_trunc('day',g.gametimestamp))) as percentile
,count(distinct date_trunc('day',g.gametimestamp)) as visits
, pl.playerID from players pl join participation p on pl.playerID = p.playerID
join games g on g.gameuuid = p.gameuuid
where g.arenaname ilike %s
and g.gametimestamp >= %s
and g.gametimestamp < %s
group by pl.playerID
),
finalresults as (
select
count (case when visits >= 1 and visits < 2 then 1 else null end) as one,
count (case when visits >= 2 and visits < 3 then 1 else null end) as twoVisits,
count (case when visits >= 3 and visits < 5 then 1 else null end) as LessThanFive,
count (case when visits >= 5 and visits < 11 then 1 else null end) as LessThanEleven,
count (case when visits >= 11 then 1 else null end) as MoreThanEleven
from data
)
select * from finalResults
"""
parameters = [cfg['SiteNameReal'], startYear, endYear]
cursor.execute(SQL, parameters)
result = cursor.fetchone()
visits = [{
"caption": "1 visit",
"players": result[0]
}, {
"caption": "2 visits",
"players": result[1]
}, {
"caption": "3-4 visits",
"players": result[2]
}, {
"caption": "5-10 visits",
"players": result[3]
}, {
"caption": "11+ visits",
"players": result[4]
}]
outputObject['regularsAggregateVisits'] = visits
#referrals
#print (" == Referrals and Welcomers")
#MEMBER PERCENTILES - GAMES
SQL = """with data as (select
percent_rank() over (order by count(distinct date_trunc('day',g.gametimestamp))) as percentile
,count(distinct date_trunc('day',g.gametimestamp)) as visits
, pl.playerID from players pl join participation p on pl.playerID = p.playerID
join games g on g.gameuuid = p.gameuuid
where g.arenaname ilike %s
and g.gametimestamp >= %s
and g.gametimestamp < %s
group by pl.playerID
),
finalresults as (
select
count (case when visits >= 1 and visits < 3 then 1 else null end) as max2,
count (case when visits >= 3 and visits < 7 then 1 else null end) as max6,
count (case when visits >= 7 and visits < 13 then 1 else null end) as max12,
count (case when visits >= 13 and visits < 36 then 1 else null end) as max35,
count (case when visits >= 36 then 1 else null end) as min36
from data
)
select * from finalResults"""
parameters = [cfg['SiteNameReal'], startYear, endYear]
cursor.execute(SQL, parameters)
result = cursor.fetchone()
games = [{
"caption": "1-2 games",
"players": result[0]
}, {
"caption": "3-6 games",
"players": result[1]
}, {
"caption": "7-12 games",
"players": result[2]
}, {
"caption": "13-35 games",
"players": result[3]
}, {
"caption": "36+ games",
"players": result[4]
}]
outputObject['regularsAggregateGames'] = games
#MEMBER PERCENTILES - RETENTION
SQL = """with preData as
(select p.playerid
, min(g.gametimestamp) as firstSeen
, max(g.gametimestamp) as lastSeen
, floor(extract(epoch from max(g.gametimestamp) - min(g.gametimestamp))/604800)::int as weeks
from games g join participation p on p.gameuuid = g.gameuuid
where arenaname ilike %s
and g.gametimestamp < %s
group by 1
),
data as
(select * from preData where lastSeen >= %s
),
finalresults as (
select count(case when weeks >= 0 and weeks <= 4 then 1 else null end) as max4
, count(case when weeks > 4 and weeks <= 12 then 1 else null end) as max12
, count(case when weeks > 12 and weeks <= 52 then 1 else null end) as max52
, count(case when weeks > 52 and weeks <= 104 then 1 else null end) as max104
, count(case when weeks > 104 and weeks <= 156 then 1 else null end) as max156
, count(case when weeks > 156 then 1 else null end) as min157
from data
)
select * from finalresults"""
parameters = [cfg['SiteNameReal'], endYear, startYear]
cursor.execute(SQL, parameters)
result = cursor.fetchone()
retention = [{
"caption": "0-4 weeks",
"players": result[0]
}, {
"caption": "5-12 weeks",
"players": result[1]
}, {
"caption": "13-52 weeks",
"players": result[2]
}, {
"caption": "1-2 years",
"players": result[3]
}, {
"caption": "3-4 years",
"players": result[4]
}, {
"caption": "5+ years",
"players": result[5]
}]
outputObject['regularsAggregateRetention'] = retention
filepart = "AnnualMetrics"
if os.name == "nt":
divider = "\\"
elif os.name == "posix":
divider = "/"
f = open(
"JSONBlobs%s%s%s-%s.json" %
(divider, cfg["ID Prefix"], filepart, startYear[0:4]), "w+")
f.write(json.dumps(outputObject, indent=4))
DBG("Annual metrics complete!", 3)