def QueryGamesLoad(scope,
interval="Null",
ArenaName=None,
offset=None,
ID=None):
params = {}
params["scope"] = scope
params["arenaName"] = cfg.getConfigString("SiteNameReal")
if scope == "full":
targetIDs = getInterestingPlayersRoster(
True,
cfg.getConfigString("StartDate"),
cfg.getConfigString("ChurnDuration"),
offset=offset)
if ID == None: #new job
ID = jobStart("Fetch games, all players",
0,
"FetchPlayerAndGames.executeQueryGames",
params,
len(targetIDs),
delay=-2)
elif scope == "activePlayers":
targetIDs = getInterestingPlayersRoster(
False,
cfg.getConfigString("StartDate"),
cfg.getConfigString("ChurnDuration"),
offset=offset,
siteName=None)
if ID == None: #new job
ID = jobStart("Fetch games, All arenas active players ",
0,
"FetchPlayerAndGames.executeQueryGames",
params,
len(targetIDs),
delay=-2)
else: #local
targetIDs = getInterestingPlayersRoster(
False,
cfg.getConfigString("StartDate"),
cfg.getConfigString("ChurnDuration"),
offset=offset,
siteName=params["arenaName"])
if ID == None: #new job
ID = jobStart("Fetch games, [%s] active players " %
(cfg.getConfigString("SiteNameShort")),
0,
"FetchPlayerAndGames.executeQueryGames",
params,
len(targetIDs),
delay=-2)
for targetID in targetIDs:
wpq.gamesQ.put(targetID)
return ID
def buildAllForAllArenasSequentially(jobID=None, startIndex=None):
cfg = cfgH.getConfig()
if jobID == None:
jobID = jobStart(
"Render all blobs", 0,
"buildAllForAllArenasSequentially.buildAllForAllArenasSequentially",
None, len(cfg["configs"]))
if startIndex == None:
startIndex = 0
counter = 0
for config in cfg["configs"]:
if counter >= startIndex:
cfgH.setActive(counter)
BuildMonthlyScoresToJSON.executeMonthlyScoresBuild()
jobHeartbeat(jobID, counter)
BuildMonthlyStarQualityToJSON.executeBuildMonthlyStars()
jobHeartbeat(jobID, counter)
BuildAchievementScoresToJSON.executeAchievementBuild()
BuildPlayerBlob.executeBuildPlayerBlobs(jobID=jobID,
counter=counter)
#BuildHeadToHeadsToJSON.buildHeadToHeads()
BuildAnnualArenaMetrics.executeBuild()
jobHeartbeat(jobID, counter)
BuildAnnualTop3s.execute()
jobHeartbeat(jobID, counter)
BuildAnnualStars.executeBuild_AnnualStars()
jobHeartbeat(jobID, counter)
counter = counter + 1
jobEnd(jobID)
def updateExistingPlayersExecute():
updateExistingPlayersLoad()
TotalEntries = wpq.summaryQ.qsize()
jobID = jobStart("Fetch summaries, all known players",
0,
"FetchPlayerUpdatesAndNewPlayers.updateExistingPlayers",
None,
TotalEntries,
delay=-2)
return jobID
def FetchAchievementsLoad(scope, jobID=None, offset=0):
params = {}
params["scope"] = scope
if scope == "full":
targetIDs = getInterestingPlayersRoster(
True,
cfg.getConfigString("StartDate"),
cfg.getConfigString("ChurnDuration"),
offset=offset)
if jobID == None:
jobID = jobStart("Fetch achievements, inactive players", 0,
"FetchAchievements.executeFetchAchievements",
params, len(targetIDs))
elif scope == "partial" or scope == "activePlayers":
targetIDs = getInterestingPlayersRoster(
False,
cfg.getConfigString("StartDate"),
cfg.getConfigString("ChurnDuration"),
offset=offset)
if jobID == None:
jobID = jobStart("Fetch achievements, active players", 0,
"FetchAchievements.executeFetchAchievements",
params, len(targetIDs))
elif scope == "recent":
targetIDs = getPlayersWhoMightNeedAchievementUpdates(scope,
offset=offset)
if jobID == None:
jobID = jobStart(
"Fetch achievements, players from the last 7 days", 0,
"FetchAchievements.executeFetchAchievements", params,
len(targetIDs))
print("Scope : %s" % (scope))
FetchAchievementsLoop(targetIDs, jobID=jobID)
return jobID
def fetchAllAchievements(targetIDs, jobID=None):
conn = connectToSource()
cursor = conn.cursor()
totalToUpdate = len(targetIDs)
offset = 0
if jobID == None:
jobID = jobStart("Fetch achievements, all known players", 0,
"FetchAchievements.fetchAllAchievements", None)
startTime = datetime.datetime.now()
else:
query = """select ID, started,lastheartbeat,resumeindex, methodname from jobslist
where finished is null and ID = %s and methodname = 'FetchAchievements.executeFetchAchievements'
order by started desc"""
cursor.execute(query, (jobID, ))
results = cursor.fetchone()
if results is None:
DBG(
"Could not find valid achievement job for ID [%s] , aborting!"
% jobID, 1)
return
if results[2] is not None:
startTime = results[2]
else:
startTime = results[1]
if results[3] is not None:
offset = results[3]
playerCounter = offset
totalPlayerCount = len(targetIDs)
lastHeartbeat = startTime
for ID in targetIDs:
__heartbeat(jobID, lastHeartbeat, playerCounter, startTime,
totalPlayerCount, ID)
fetchIndividualsAchievements(ID)
playerCounter = playerCounter + 1
jobEnd(jobID)
endTime = datetime.datetime.now()
f = open("Stats.txt", "a+")
f.write(
"Queried {0} players' achievements, operation completed after {1}. \t\n"
.format(len(targetIDs), endTime - startTime))
f.close()
closeConnection()
def findNewPlayers(siteName=None, jobID=None):
startTime = datetime.datetime.now()
lastHeartbeat = startTime
conn = connectToSource()
cursor = conn.cursor()
conn = connectToSource()
cursor = conn.cursor()
if siteName is not None:
targetID = cfg.findSiteIDFromName(siteName)
siteObj = cfg.getSiteWithoutActivatingByID(targetID)
sitePrefix = siteObj["ID Prefix"]
siteName = siteObj["SiteNameReal"]
else:
sitePrefix = cfg.getConfigString("ID Prefix")
siteName = cfg.getConfigString("SiteNameReal")
params = {}
params["siteName"] = siteName
if jobID is None:
jobID = jobStart(" new players at [%s]" % siteName, 0,
"FetchPlayerUpdatesAndNewPlayers.findNewPlayers",
params)
else:
jobHeartbeat(jobID, 0)
TickerIcon = ["|", "/", "-", '\\']
sitePrefixforSQL = sitePrefix + "%"
query = sql.SQL("""
with IDs as ( select
cast (split_part(pl.PlayerID,'-',3) as integer) as ID
from players pl
where playerID like %s
order by 1 desc
offset 5
)
select max (ID) from IDs
""")
cursor.execute(query, (sitePrefixforSQL, ))
result = cursor.fetchone()
if result == None or result[0] == None:
MaxPlayer = 199 #LaserForce seems to start numbering players at 100
else:
MaxPlayer = result[0]
region = sitePrefix.split("-")[0]
siteNumber = sitePrefix.split("-")[1]
ticker = 0
consecutiveMisses = 0
currentTarget = MaxPlayer - 100 #we've had situations where the system adds user IDs behind the maximum. This is a stopgap dragnet to catch trailing players.
AllowedMisses = 100
while consecutiveMisses <= AllowedMisses:
heartbeatDelta = (datetime.datetime.now() -
lastHeartbeat).total_seconds()
if heartbeatDelta > 30:
jobHeartbeat(jobID, 0)
lastHeartbeat = datetime.datetime.now()
conn.commit()
player = fetchPlayer_root('', region, siteNumber, currentTarget)
if 'centre' in player:
codeName = player["centre"][0]["codename"]
dateJoined = player["centre"][0]["joined"]
missionsPlayed = player["centre"][0]["missions"]
skillLevelNum = player["centre"][0]["skillLevelNum"]
addPlayer("%s%i" % (sitePrefix, currentTarget), codeName,
dateJoined, missionsPlayed)
_parseCentresAndAdd(
player["centre"],
'%s-%s-%s' % (region, siteNumber, currentTarget))
consecutiveMisses = 0
else:
DBG(
"DBG: FetchPlayerUpdatesAndNewPlayers.findNewPlayers - Missed a player 7-X-%s"
% (currentTarget), 3)
consecutiveMisses = consecutiveMisses + 1
wpq.updateQ(consecutiveMisses, AllowedMisses,
"Seeking new... %s" % TickerIcon[ticker % 4], "ETA ?")
currentTarget = currentTarget + 1
ticker = ticker + 1
endTime = datetime.datetime.now()
jobEnd(jobID)
f = open("Stats.txt", "a+")
f.write(
"searched for {0} players, operation completed after {1}. \t\n".format(
currentTarget - MaxPlayer, endTime - startTime))
wpq.updateQ(1, 1, "Seeking new... %s", "Complete")
f.close()
conn.commit()
closeConnection()
def updateExistingPlayersLoop(JobID=None):
startTime = datetime.datetime.now()
conn = connectToSource()
cursor = conn.cursor()
offset = 0
TotalEntries = wpq.summaryQ.qsize()
if JobID == None:
JobID = jobStart(
"Fetch summaries, all known players", 0,
"FetchPlayerUpdatesAndNewPlayers.updateExistingPlayers", None,
TotalEntries)
startTime = datetime.datetime.now()
global WorkerStatus
lastHeartbeat = startTime
counter = offset
jobHeartbeat(JobID, counter)
while wpq.summaryQ.empty() == False:
targetID = wpq.summaryQ.get()
targetID = targetID[0]
heartbeatDelta = (datetime.datetime.now() -
lastHeartbeat).total_seconds()
if heartbeatDelta > 30:
jobHeartbeat(JobID, counter)
lastHeartbeat = datetime.datetime.now()
counter = TotalEntries - wpq.summaryQ.qsize()
WorkerStatus = {}
WorkerStatus["CurEntry"] = counter
WorkerStatus["TotalEntries"] = TotalEntries
WorkerStatus["CurrentAction"] = "summary of %s" % (targetID)
delta = "[ Calculating ]"
if counter >= 20:
delta = ((datetime.datetime.now() - startTime).total_seconds() /
counter)
delta = (TotalEntries - counter) * delta #seconds remaining
seconds = round(delta, 0)
minutes = 0
hours = 0
if (seconds > 60):
minutes = math.floor(seconds / 60)
seconds = seconds % 60
if (minutes > 60):
hours = math.floor(minutes / 60)
minutes = minutes % 60
delta = "%ih, %im, %is" % (hours, minutes, seconds)
wpq.updateQ(counter, TotalEntries, "summary of %s" % (targetID), delta)
ID = targetID.split('-')
player = fetchPlayer_root('', ID[0], ID[1], ID[2])
datetime_list = []
missions = 0
level = 0
for i in player["centre"]:
datetime_list.append(str(i["joined"]))
missions += int(i["missions"])
level = max(level, int(i["skillLevelNum"]))
joined = min(datetime_list)
codeName = str(player["centre"][0]["codename"])
#DBG("Summary update for player %s-%s-%s, [%i/%i]" % (ID[0],ID[1],ID[2],counter,TotalEntries),3)
addPlayer(targetID, codeName, joined, missions)
_parseCentresAndAdd(player["centre"], targetID)
jobEnd(JobID)
endTime = datetime.datetime.now()
f = open("Stats.txt", "a+")
f.write(
"Queried {0} players' aggregates, operation completed after {1}. \t\n".
format(TotalEntries, endTime - startTime))
f.close()