def removeFromRumble(self, requests):
#global global_dict
global_dict = {}
if "version" not in requests or requests["version"] is not "1":
return "ERROR. bad/no version"
game = requests.get("game", None)
if game is None:
return "ERROR. no game specified"
name = requests.get("name", None)
if name is None:
return "ERROR. no name specified"
while name.count("%20") > 0:
name = rreplace(name, "%20", " ", 1)
rumble = global_dict.get(game, None)
if rumble is None:
rumble = memcache.get(game)
if rumble is None:
rumble = structures.Rumble.get_by_key_name(game)
entry = None
if name.count(" ") == 0:
num_underscores = name.count("_")
error_messages = []
for n in range(num_underscores):
check_name = nth_repl(name, "_", " ", n + 1)
keyhash = check_name + "|" + game
entry = global_dict.get(keyhash, None)
if entry is None:
entry = memcache.get(keyhash)
if entry is None:
entry = structures.BotEntry.get_by_key_name(keyhash)
if entry is None:
error_messages.append(
"ERROR. name/game does not exist: " + check_name +
"/" + game)
else:
entry = structures.CachedBotEntry(entry)
if entry is not None:
name = check_name
break
if entry is None:
return "\n".join(error_messages)
else:
return "ERROR. Bot name missing version."
if isinstance(entry, structures.BotEntry):
entry = structures.CachedBotEntry(entry)
global_dict.pop(keyhash, 0)
memcache.delete(keyhash)
entry.Active = False
try:
scores = pickle.loads(zlib.decompress(rumble.ParticipantsScores))
except:
scoresdicts = marshal.loads(zlib.decompress(rumble.ParticipantsScores))
scoreslist = [structures.LiteBot() for _ in scoresdicts]
for s, d in zip(scoreslist, scoresdicts):
s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
scores.pop(name, 1)
rumble.ParticipantsScores = zlib.compress(
pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 4)
memcache.delete("home")
global_dict.pop("home", 0)
memcache.set(entry.key_name, entry)
global_dict[entry.key_name] = entry
modelBot = structures.BotEntry(key_name=entry.key_name)
modelBot.init_from_cache(entry)
modelBot.put()
global_dict[game] = rumble
memcache.set(game, rumble)
rumble.put()
return "OK. " + name + " retired from " + game
def get(self):
#global global_dict
global_dict = {}
starttime = time.time()
query = self.request.query_string
query = query.replace("%20"," ")
parts = query.split("&")
requests = {}
if parts[0] != "":
for pair in parts:
ab = pair.split('=')
requests[ab[0]] = ab[1]
game = requests.get("game","meleerumble")
lim = int(requests.get("limit","10000000"))
#ofst = int(requests.get("offset","0"))
order = requests.get("order","APS")
timing = bool(requests.get("timing",False))
api = bool(requests.get("api",False))
extraArgs = ""
if timing:
extraArgs += "&timing=1"
if lim < 100000:
extraArgs += "&limit=" + str(lim)
reverseSort = True
if len(order) == 0:
order = "APS"
if order[0] == "-":
order = order[1:]
reverseSort = False
if order == "Latest Battle":
order = "LastUpload"
elif order == "Competitor":
order = "Name"
elif order == "Vote":
order = "VoteScore"
parsing = time.time() - starttime
#rumble = global_dict.get(game,None)
#if rumble is None:
rumble = memcache.get(game)
if rumble is None:
rumble = structures.Rumble.get_by_key_name(game)
if rumble is None:
self.response.out.write("RUMBLE NOT FOUND")
return
else:
#global_dict[game]=rumble
memcache.set(game,rumble)
#else:
#global_dict[game] = rumble
#flagmap = global_dict.get(structures.default_flag_map,None)
#if flagmap is None:
flagmap = memcache.get(structures.default_flag_map)
if flagmap is None:
flagmapholder = structures.FlagMap.get_by_key_name(structures.default_flag_map)
if flagmapholder is None:
flagmap = zlib.compress(marshal.dumps({}))
else:
flagmap = flagmapholder.InternalMap
memcache.set(structures.default_flag_map,flagmap)
#global_dict[structures.default_flag_map] = flagmap
# else:
# global_dict[structures.default_flag_map] = flagmap
try:
flagmap = pickle.loads(zlib.decompress(flagmap))
except:
flagmap = marshal.loads(zlib.decompress(flagmap))
try:
#print "try json"
botsdict = pickle.loads(zlib.decompress(rumble.ParticipantsScores))
bots = botsdict.values()
# try:
# self.response.out.write( " json worked")
except:
# self.response.out.write( "\ntry pickle")
scoresdicts = marshal.loads(zlib.decompress(rumble.ParticipantsScores))
scoreslist = [structures.LiteBot() for _ in scoresdicts]
for s,d in zip(scoreslist,scoresdicts):
s.__dict__.update(d)
#r = {s.Name:s for s in scoreslist}
bots = scoreslist
# print " pickle worked"
retrievetime = time.time() - starttime - parsing
#newbots = []
for b in bots:
b.PWIN = 50.0*float(b.PL)/b.Pairings + 50.0
if b.VoteScore is None:
b.VoteScore = 0
if b.ANPP is None:
b.ANPP = 0
package = string.split(b.Name,".")[0]
if package in flagmap:
b.Flag = flagmap[package]
else:
b.Flag = "NONE"
get_key = attrgetter(order)
bots.sort( key=lambda b: get_key(b), reverse=reverseSort)
if api:
headings = ["\"name\"",
"\"flag\"",
"\"rank\"",
"\"APS\"",
"\"PWIN\"",
"\"ANPP\"",
"\"vote\"",
"\"survival\"",
"\"pairings\"",
"\"battles\"",
"\"latest\""]
escapes = ["\"","\"","","","","","","","","","\""]
outs = ["[\n"]
count = 0
for bot in bots:
count += 1
if count > lim:
break
cells = [
bot.Name,bot.Flag,count,
round(100.0*bot.APS)*0.01,
round(100.0*bot.PWIN)*0.01,
round(100.0*bot.ANPP)*0.01,
round(100.0*bot.VoteScore)*0.01,
round(100.0*bot.Survival)*0.01,
bot.Pairings,bot.Battles,bot.LastUpload]
outs.append("{")
for i in range(len(cells)):
outs.append(headings[i])
outs.append(":")
outs.append(escapes[i])
outs.append(str(cells[i]))
outs.append(escapes[i])
outs.append(",")
outs[-1] = "},\n"
outs[-1] = ("}\n]")
self.response.out.write(''.join(outs))
else:
sorttime = time.time() - parsing - retrievetime - starttime
if order == "LastUpload":
order = "Latest Battle"
elif order == "Name":
order = "Competitor"
elif order == "VoteScore":
order = "Vote"
out = []
gameTitle = "RANKINGS - " + string.upper(game) + " WITH " + str(len(bots)) + " BOTS"
out.append(structures.html_header % (game,gameTitle))
pairVals = [b.Pairings for b in bots]
if max(pairVals) == min(pairVals) == (len(bots)-1):
out.append("<big>Rankings Stable</big>")
else:
out.append("<big>Rankings Not Stable</big>")
out.append("\n<table>\n<tr>");
headings = ["","Flag","Competitor","APS","PWIN","ANPP","Vote","Survival","Pairings","Battles","Latest Battle"]
for heading in headings:
sortedBy = order == heading
if order == heading and reverseSort:
heading = "-" + heading
orderl = []
orderl.append("<a href=\"Rankings?game=")
orderl.append(game)
orderl.append("&order=")
orderl.append(heading.replace(" ","%20"))
orderl.append(extraArgs)
orderl.append("\">")
orderl.append(heading)
orderl.append("</a>")
orderHref = ''.join(orderl)
if sortedBy:
out.append( "\n<th class=\"sortedby\">" + orderHref + "</th>")
else:
out.append( "\n<th>" + orderHref + "</th>")
out.append("\n</tr>")
rank = 1
for bot in bots:
if rank > lim:
break
botName=bot.Name
bnh = []
bnh.append("<a href=\"BotDetails?game=")
bnh.append(game)
bnh.append("&name=")
bnh.append(botName.replace(" ","%20"))
bnh.append(extraArgs)
bnh.append("\" >")
bnh.append(botName)
bnh.append("</a>")
botNameHref = ''.join(bnh) #"<a href=BotDetails?game="+game+"&name=" + botName.replace(" ","%20")+extraArgs+">"+botName+"</a>"
ft = []
ft.append("<img id='flag' src=\"/flags/")
ft.append(bot.Flag)
ft.append(".gif\">")
flagtag = ''.join(ft)
cells = [rank,flagtag,botNameHref,
round(100.0*bot.APS)*0.01,
round(100.0*bot.PWIN)*0.01,
round(100.0*bot.ANPP)*0.01,
round(100.0*bot.VoteScore)*0.01,
round(100.0*bot.Survival)*0.01,
bot.Pairings,bot.Battles,bot.LastUpload]
out.append("\n<tr>")
for cell in cells:
out.append( "\n<td>")
out.append(str(cell))
out.append("</td>")
out.append("\n</tr>")
del bot.PWIN
rank += 1
out.append( "</table>")
htmltime = time.time() - parsing - retrievetime - sorttime - starttime
elapsed = time.time() - starttime
if timing:
out.append("\n<br> Page served in " + str(int(round(elapsed*1000))) + "ms. ")# + str(len(missingHashes)) + " bots retrieved from datastore.")
out.append("\n<br> parsing: " + str(int(round(parsing*1000))) )
out.append("\n<br> retrieve: " + str(int(round(retrievetime*1000))) )
out.append("\n<br> sort: " + str(int(round(sorttime*1000))) )
out.append("\n<br> html generation: " + str(int(round(htmltime*1000))) )
out.append( "</body></html>")
outstr = ''.join(out)
self.response.out.write(outstr)
class HandleQueuedResults(webapp.RequestHandler):
def post(self):
global global_dict
#global_dict = {}
global global_sync
global locks
global last_write
#global sync_lock
# starttime = time.time()
results = json.loads(self.request.body)
rumble = results.get("game", "ERROR")
syncname = str(
bool(results.get("melee") == "YES")) + "|" + structures.sync
if syncname not in locks:
locks[syncname] = threading.Lock()
sync_lock = locks[syncname]
global_sync[syncname] = global_sync.get(syncname, {})
botsync = global_sync[syncname]
bota = results.get("fname")
botb = results.get("sname")
#bota = rreplace(bota,"_"," ",1)
#botb = rreplace(botb,"_"," ",1)
logging.debug("Bots : " + bota + " vs. " + botb)
bd = [[bota, rumble], [botb, rumble]]
botHashes = [string.join(a, "|").encode('ascii') for a in bd]
memget = [rumble]
memget.extend(botHashes) #botHashes.append(rumble)
memdict = memcache.get_multi(memget)
game = memdict.get(rumble, None)
game_future = None
if game is None:
#game = structures.Rumble.get_by_key_name(rumble)
game_future = db.get_async(db.Key.from_path('Rumble', rumble))
newBot = False
bots = [memdict.get(h, None) for h in botHashes]
pairingsarray = [[], []]
botFutures = [None, None]
for i in [0, 1]:
if bots[i] is None or bots[i].PairingsList is None:
botFutures[i] = db.get_async(
db.Key.from_path('BotEntry', botHashes[i]))
for i in [0, 1]:
if botFutures[i] is not None:
modelbot = botFutures[i].get_result()
if modelbot is not None:
bots[i] = structures.CachedBotEntry(modelbot)
#logging.debug("retrieved from database")
for i in [0, 1]:
if bots[i] is None:
modelbot = structures.BotEntry(key_name=botHashes[i],
Name=bd[i][0],
Battles=0,
Pairings=0,
APS=0.0,
Survival=0.0,
PL=0,
Rumble=rumble,
Active=False,
PairingsList=zlib.compress(
pickle.dumps([]), 1))
bots[i] = structures.CachedBotEntry(modelbot)
newBot = True
if isinstance(bots[i], structures.BotEntry):
bots[i] = structures.CachedBotEntry(bots[i])
try:
pairingsarray[i] = pickle.loads(
zlib.decompress(bots[i].PairingsList))
bots[i].PairingsList = None
except:
try:
pairsDicts = marshal.loads(
zlib.decompress(bots[i].PairingsList))
pairingsarray[i] = [
structures.ScoreSet() for _ in pairsDicts
]
for s, d in zip(pairingsarray[i], pairsDicts):
s.__dict__.update(d)
bots[i].PairingsList = None
except:
pairingsarray[i] = []
if game_future is not None:
game = game_future.get_result()
if game is None:
game = structures.Rumble(key_name=rumble,
Name=rumble,
Rounds=int(results["rounds"]),
Field=results["field"],
Melee=bool(results["melee"] == "YES"),
Teams=bool(results["teams"] == "YES"),
TotalUploads=0,
MeleeSize=10,
ParticipantsScores=db.Blob(
zlib.compress(pickle.dumps([]))))
self.response.out.write("CREATED NEW GAME TYPE " + rumble + "\n")
logging.info("Created new game type: " + rumble)
else:
field = game.Field == results["field"]
rounds = (game.Rounds == int(results["rounds"]))
teams = game.Teams == bool(results["teams"] == "YES")
melee = game.Melee == bool(results["melee"] == "YES")
allowed = field and rounds and teams and melee
if not allowed:
errstr = "OK. ERROR. Incorrect " + rumble + " config: "
errorReasons = []
if not field:
errorReasons.append("field size ")
if not rounds:
errorReasons.append("number of rounds ")
if not teams:
errorReasons.append("teams ")
if not melee:
errorReasons.append("melee ")
logging.error(errstr + string.join(errorReasons, ", ") +
" User: "******"user"])
return
scores = None
try:
scores = pickle.loads(zlib.decompress(game.ParticipantsScores))
game.ParticipantsScores = None
if len(scores) == 0:
scores = {}
except:
scoresdicts = marshal.loads(
zlib.decompress(game.ParticipantsScores))
game.ParticipantsScores = None
scoreslist = [structures.LiteBot(loadDict=d) for d in scoresdicts]
#for s,d in zip(scoreslist,scoresdicts):
# s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
if len(scores) == 0:
scores = {}
#logging.debug("uncompressed scores: " + str(len(s)) + " compressed: " + str(a))
for i in [0, 1]:
if not bots[i].Active or bots[i].Name not in scores:
bots[i].Active = True
scores[bots[i].Name] = structures.LiteBot(bots[i])
newBot = True
self.response.out.write("Added " + bd[i][0] + " to " + rumble +
"\n")
logging.info("added new bot!")
#retrievetime = time.time() - starttime
scorea = float(results["fscore"])
scoreb = float(results["sscore"])
if scorea + scoreb > 0:
APSa = 100 * scorea / (scorea + scoreb)
else:
APSa = 50 #register a 0/0 as 50%
#APSb = 100 - APSa
survivala = float(results["fsurvival"])
survivalb = float(results["ssurvival"])
survivala = 100.0 * survivala / game.Rounds
survivalb = 100.0 * survivalb / game.Rounds
for b, pairings in zip(bots, pairingsarray):
#b.PairingsList = zlib.compress(marshal.dumps([s.__dict__ for s in pairings]),1)
if len(pairings) > 0:
removes = []
for p in pairings:
try:
p.APS = float(p.APS)
p.Survival = float(p.Survival)
p.Battles = int(p.Battles)
except:
removes.append(pairings.index(p))
continue
removes.sort(reverse=True)
for i in removes:
pairings.pop(i)
apair = None
for p in pairingsarray[0]:
if p.Name == botb:
apair = p
if apair is None:
apair = structures.ScoreSet(name=botb)
pairingsarray[0].append(apair)
bpair = None
for p in pairingsarray[1]:
if p.Name == bota:
bpair = p
if bpair is None:
bpair = structures.ScoreSet(name=bota)
pairingsarray[1].append(bpair)
#participantsSet = set(game.Participants)
for b, pairings in zip(bots, pairingsarray):
i = 0
while i < len(pairings):
if pairings[i].Name == b.Name:
pairings.pop(i)
continue
if not hasattr(pairings[i], "Alive"):
pairings[i].Alive = True
if pairings[i].Alive and pairings[i].Name not in scores:
pairings[i].Alive = False
i += 1
#b.Pairings = i
aBattles = apair.Battles
#rolling average with a half-life of 10k
maxPerPair = 10000 / len(bots)
if aBattles > maxPerPair:
aBattles = maxPerPair
#bBattles = bpair.Battles
inv_ab = 1.0 / (aBattles + 1.0)
apair.APS *= float(aBattles) * inv_ab
apair.APS += APSa * inv_ab
apair.__dict__["Min_APS"] = min(APSa,
apair.__dict__.get("Min_APS", 100))
#bpair.APS *= float(bBattles)/(bBattles + 1)
#bpair.APS += APSb/(bBattles+1)
bpair.APS = 100 - apair.APS
bpair.__dict__["Min_APS"] = min(100 - APSa,
bpair.__dict__.get("Min_APS", 100))
apair.Survival *= float(aBattles) * inv_ab
apair.Survival += survivala * inv_ab
bpair.Survival *= float(aBattles) * inv_ab
bpair.Survival += survivalb * inv_ab
apair.Battles += 1
bpair.Battles = apair.Battles
apair.LastUpload = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
bpair.LastUpload = apair.LastUpload
for b, pairings in zip(bots, pairingsarray):
aps = 0.0
survival = 0.0
pl = 0
battles = 0
alivePairings = 0
if len(pairings) > 0:
for p in pairings:
if p.Alive:
aps += p.APS
survival += p.Survival
if p.APS > 50:
pl += 1
else:
pl -= 1
battles += p.Battles
alivePairings += 1
aps /= alivePairings
survival /= alivePairings
b.APS = aps
b.Survival = survival
b.PL = pl
b.Battles = battles
b.PairingsList = db.Blob(
zlib.compress(pickle.dumps(pairings, pickle.HIGHEST_PROTOCOL),
1))
b.LastUpload = apair.LastUpload
b.Pairings = alivePairings
game.TotalUploads += 1
#self.response.out.write("<" + str(bots[0].Battles) + " " + str(bots[1].Battles) + ">")
game.LastUpload = apair.LastUpload
game.AvgBattles = game.AvgBattles * 0.99 + 0.005 * (bots[0].Battles +
bots[1].Battles)
if game.Uploaders is None:
uploaders = None
else:
uploaders = pickle.loads(zlib.decompress(game.Uploaders))
if uploaders is None or len(uploaders) == 0:
uploaders = {}
uploaderName = results["user"]
try:
uploader = uploaders[uploaderName]
uploader.latest = apair.LastUpload
uploader.total += 1
except KeyError:
uploader = structures.User(name=uploaderName)
uploaders[uploaderName] = uploader
game.Uploaders = zlib.compress(pickle.dumps(uploaders, -1), 1)
for b in bots:
try:
bUploaders = b.Uploaders
if uploaderName not in bUploaders:
bUploaders.append(uploaderName)
except:
b.__dict__["Uploaders"] = [uploaderName]
with sync_lock:
for b in bots:
key = None
if isinstance(b, structures.BotEntry):
key = b.key().name()
else:
key = b.key_name
botsync[key] = botsync.get(key, 0) + 1
minSize = min(10, len(scores) / 2)
wrote = False
logging.debug("botsync: " + str(len(botsync)))
if len(botsync) > minSize:
syncset = botsync.keys()
syncbotsDict = memcache.get_multi(syncset)
syncbots = []
for sb in syncset:
b = syncbotsDict.get(sb, None)
if b is None or b.PairingsList is None:
syncbotsDict.pop(sb, 1)
else:
syncbots.append(b)
botsync.pop(sb, 1)
try:
thisput = []
while len(syncbots) > 0:
b = syncbots.pop()
key = None
if isinstance(b, structures.BotEntry):
key = b.key().name()
else:
key = b.key_name
putb = structures.BotEntry(key_name=key)
putb.init_from_cache(b)
thisput.append(putb)
db.put(thisput)
logging.info("wrote " + str(len(thisput)) +
" results to database")
for b in thisput:
s = b.key().name()
botsync.pop(s, 1)
syncbotsDict.pop(s, 1)
wrote = True
except Exception, e:
logging.error('Failed to write data: ' + str(e))
#self.response.out.write('Failed to write data: '+ str(e.__class__))
for b in bots:
if b.Name in scores:
lb = scores[b.Name]
b.ANPP = lb.ANPP
b.VoteScore = lb.VoteScore
scores.pop(b.Name, 1)
scores[b.Name] = structures.LiteBot(b)
game.ParticipantsScores = db.Blob(
zlib.compress(pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 1))
game.Participants = []
if game.BatchScoresAccurate:
game.BatchScoresAccurate = False
if not newBot:
game.put()
wrote = True
if newBot or wrote:
game.put()
#db.put(bots)
memcache.delete("home")
botsDict = {rumble: game}
global_dict[rumble] = game
for b in bots:
if isinstance(b, structures.BotEntry):
b = structures.CachedBotEntry(b)
key = b.key_name
botsDict[key] = b
memcache.set_multi(botsDict)
# puttime = time.time() - scorestime - retrievetime - starttime
scoreVals = scores.values()
maxPairs = len(scoreVals) - 1
if game.PriorityBattles and (
(not game.Melee) or
(random.random() < 0.0222 or bots[0].Pairings != maxPairs
or bots[1].Pairings != maxPairs
or min([bots[0].Battles, bots[1].Battles]) == min(
[b.Battles for b in scoreVals]))
): # or min(bots, key = lambda b: b.Battles) < game.AvgBattles):
#do a gradient descent to the lowest battled pairings:
#1: take the bot of this pair which has less pairings/battles
#2: find an empty pairing or take a low pairing
#3: ????
#4: PROFIT!!!
priobot = None
priopairs = None
#logging.debug("Maxpairs: " + str(maxPairs))
#let's do a direct search since we already have the data
#First look for bots missing pairings
missingPairings = filter(lambda b: b.Pairings != maxPairs,
scoreVals)
#logging.debug("missingPairings: " + str(len(missingPairings)))
priobot2 = None
if len(missingPairings) > 0:
total = 0
#weight bots based on how different they are from desired pairings
weighted = [(abs(maxPairs - b.Pairings), b)
for b in missingPairings]
#select them for battles according to that weight
for t in weighted:
total += t[0]
running = 0
point = random.randint(0, total - 1)
for t in weighted:
running += t[0]
if running > point:
priobot = t[1]
break
#check if we have the selected bot already loaded
if priobot.Name == bots[0].Name:
priobot = bots[0]
priopairs = pairingsarray[0]
elif priobot.Name == bots[1].Name:
priobot = bots[1]
priopairs = pairingsarray[1]
else:
#if not try to load them from memcache, but only if we need a special battle
if priobot.Pairings < maxPairs:
bhash = priobot.Name + "|" + rumble
fullPrioBot = memcache.get(bhash)
else:
fullPrioBot = None
if fullPrioBot:
priopairs = pickle.loads(
zlib.decompress(fullPrioBot.PairingsList))
logging.info(
"memcache lookup shortcut to local search")
else:
#if they aren't in memcache give them a generic battle
priobot2 = random.choice(scoreVals).Name
catch = 10
while priobot2 == priobot.Name and catch > 0:
priobot2 = random.choice(scoreVals).Name
catch -= 1
if catch == 0:
priobot2 = None
logging.info("repeatedly found same bot for prio")
else:
logging.info(
"global min search successful for non-paired bot"
)
if priobot != None and priobot2 == None:
#create the first battle of a new pairing
#cache pairings into dictionary to speed lookups against entire rumble
pairsdict = {}
for b in priopairs:
pairsdict[b.Name] = b
#select all unpaired bots
possPairs = []
for p in scores:
if p not in pairsdict and p != priobot.Name:
possPairs.append(p)
if len(possPairs) > 0:
#choose a random new pairing to prevent overlap
priobot2 = random.choice(possPairs)
logging.info("successful local search for new pair")
else:
logging.info("unsuccessful local search for new pair")
else:
minBattles = 1.1 * min(
[b.Battles for b in scoreVals if b.Active])
possBots = filter(
lambda b: b.Battles <= minBattles and b.Active, scoreVals)
names = [b.Name for b in possBots]
if bots[1].Name not in names and bots[0].Name not in names:
priobot = random.choice(possBots)
bhash = priobot.Name + "|" + rumble
fullPrioBot = memcache.get(bhash)
if fullPrioBot:
priopairs = pickle.loads(
zlib.decompress(fullPrioBot.PairingsList))
logging.info(
"memcache lookup shortcut to global search")
else:
priobot2 = random.choice(scoreVals).Name
catch = 10
while priobot2 == priobot.Name and catch > 0:
priobot2 = random.choice(scoreVals).Name
catch -= 1
if catch == 0:
priobot2 = None
logging.info("repeatedly found same bot for prio")
else:
logging.info(
"global min search successful for low-battled bot"
)
elif bots[0].Battles <= bots[1].Battles:
priobot = bots[0]
priopairs = pairingsarray[0]
else:
priobot = bots[1]
priopairs = pairingsarray[1]
if priobot2 is None and priopairs is not None:
#sort for lowest battled pairing
#priopairs = sorted(priopairs, key = lambda score: score.Battles)
minbat = min([
p.Battles for p in priopairs
if p.Name in scores and scores[p.Name].Active
])
possPairs = filter(
lambda p: p.Battles <= minbat and p.Name != priobot.Name
and p.Name in scores and scores[p.Name].Active, priopairs)
if len(possPairs) < min(50, 0.5 * len(scores)):
possPairs = filter(
lambda p: p.Battles <= minbat + 1 and p.Name != priobot
.Name and p.Name in scores and scores[p.Name].Active,
priopairs)
if len(possPairs) > 0:
priobot2 = random.choice(possPairs).Name
logging.info(
"successful local search for low-battled pair")
#choose low battles, but still random - prevents lots of duplicates
else:
logging.info(
"unsuccessful local search for low-battled pair")
if priobot is not None and priobot2 is not None:
priobots = [priobot.Name, priobot2]
priobots = [b.replace(' ', '_') for b in priobots]
prio_string = "[" + string.join(priobots, ",") + "]\n"
logging.info("adding priority battle: " + prio_string + ", " +
rumble)
rq_name = rumble + "|queue"
try:
rumble_queue = global_dict[rq_name]
rumble_queue.append(prio_string)
except KeyError:
logging.info("No queue for rumble " + rumble +
", adding one!")
global_dict[rq_name] = deque()
rumble_queue = global_dict[rq_name]
rumble_queue.append(prio_string)
else:
logging.info("no suitable priority battle found in " + rumble)
if priobot is None:
logging.info("priobot is None")
else:
logging.info("priobot2 is None")
else:
logging.info("no priority battle attempted for " + rumble)
# priotime = time.time() - puttime - scorestime - retrievetime - starttime
self.response.out.write("\nOK. " + bota + " vs " + botb + " received")
def post(self):
global global_dict
#global_dict = {}
global global_sync
global locks
global last_write
#global sync_lock
# starttime = time.time()
results = json.loads(self.request.body)
rumble = results.get("game", "ERROR")
syncname = str(
bool(results.get("melee") == "YES")) + "|" + structures.sync
if syncname not in locks:
locks[syncname] = threading.Lock()
sync_lock = locks[syncname]
global_sync[syncname] = global_sync.get(syncname, {})
botsync = global_sync[syncname]
bota = results.get("fname")
botb = results.get("sname")
#bota = rreplace(bota,"_"," ",1)
#botb = rreplace(botb,"_"," ",1)
logging.debug("Bots : " + bota + " vs. " + botb)
bd = [[bota, rumble], [botb, rumble]]
botHashes = [string.join(a, "|").encode('ascii') for a in bd]
memget = [rumble]
memget.extend(botHashes) #botHashes.append(rumble)
memdict = memcache.get_multi(memget)
game = memdict.get(rumble, None)
game_future = None
if game is None:
#game = structures.Rumble.get_by_key_name(rumble)
game_future = db.get_async(db.Key.from_path('Rumble', rumble))
newBot = False
bots = [memdict.get(h, None) for h in botHashes]
pairingsarray = [[], []]
botFutures = [None, None]
for i in [0, 1]:
if bots[i] is None or bots[i].PairingsList is None:
botFutures[i] = db.get_async(
db.Key.from_path('BotEntry', botHashes[i]))
for i in [0, 1]:
if botFutures[i] is not None:
modelbot = botFutures[i].get_result()
if modelbot is not None:
bots[i] = structures.CachedBotEntry(modelbot)
#logging.debug("retrieved from database")
for i in [0, 1]:
if bots[i] is None:
modelbot = structures.BotEntry(key_name=botHashes[i],
Name=bd[i][0],
Battles=0,
Pairings=0,
APS=0.0,
Survival=0.0,
PL=0,
Rumble=rumble,
Active=False,
PairingsList=zlib.compress(
pickle.dumps([]), 1))
bots[i] = structures.CachedBotEntry(modelbot)
newBot = True
if isinstance(bots[i], structures.BotEntry):
bots[i] = structures.CachedBotEntry(bots[i])
try:
pairingsarray[i] = pickle.loads(
zlib.decompress(bots[i].PairingsList))
bots[i].PairingsList = None
except:
try:
pairsDicts = marshal.loads(
zlib.decompress(bots[i].PairingsList))
pairingsarray[i] = [
structures.ScoreSet() for _ in pairsDicts
]
for s, d in zip(pairingsarray[i], pairsDicts):
s.__dict__.update(d)
bots[i].PairingsList = None
except:
pairingsarray[i] = []
if game_future is not None:
game = game_future.get_result()
if game is None:
game = structures.Rumble(key_name=rumble,
Name=rumble,
Rounds=int(results["rounds"]),
Field=results["field"],
Melee=bool(results["melee"] == "YES"),
Teams=bool(results["teams"] == "YES"),
TotalUploads=0,
MeleeSize=10,
ParticipantsScores=db.Blob(
zlib.compress(pickle.dumps([]))))
self.response.out.write("CREATED NEW GAME TYPE " + rumble + "\n")
logging.info("Created new game type: " + rumble)
else:
field = game.Field == results["field"]
rounds = (game.Rounds == int(results["rounds"]))
teams = game.Teams == bool(results["teams"] == "YES")
melee = game.Melee == bool(results["melee"] == "YES")
allowed = field and rounds and teams and melee
if not allowed:
errstr = "OK. ERROR. Incorrect " + rumble + " config: "
errorReasons = []
if not field:
errorReasons.append("field size ")
if not rounds:
errorReasons.append("number of rounds ")
if not teams:
errorReasons.append("teams ")
if not melee:
errorReasons.append("melee ")
logging.error(errstr + string.join(errorReasons, ", ") +
" User: "******"user"])
return
scores = None
try:
scores = pickle.loads(zlib.decompress(game.ParticipantsScores))
game.ParticipantsScores = None
if len(scores) == 0:
scores = {}
except:
scoresdicts = marshal.loads(
zlib.decompress(game.ParticipantsScores))
game.ParticipantsScores = None
scoreslist = [structures.LiteBot(loadDict=d) for d in scoresdicts]
#for s,d in zip(scoreslist,scoresdicts):
# s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
if len(scores) == 0:
scores = {}
#logging.debug("uncompressed scores: " + str(len(s)) + " compressed: " + str(a))
for i in [0, 1]:
if not bots[i].Active or bots[i].Name not in scores:
bots[i].Active = True
scores[bots[i].Name] = structures.LiteBot(bots[i])
newBot = True
self.response.out.write("Added " + bd[i][0] + " to " + rumble +
"\n")
logging.info("added new bot!")
#retrievetime = time.time() - starttime
scorea = float(results["fscore"])
scoreb = float(results["sscore"])
if scorea + scoreb > 0:
APSa = 100 * scorea / (scorea + scoreb)
else:
APSa = 50 #register a 0/0 as 50%
#APSb = 100 - APSa
survivala = float(results["fsurvival"])
survivalb = float(results["ssurvival"])
survivala = 100.0 * survivala / game.Rounds
survivalb = 100.0 * survivalb / game.Rounds
for b, pairings in zip(bots, pairingsarray):
#b.PairingsList = zlib.compress(marshal.dumps([s.__dict__ for s in pairings]),1)
if len(pairings) > 0:
removes = []
for p in pairings:
try:
p.APS = float(p.APS)
p.Survival = float(p.Survival)
p.Battles = int(p.Battles)
except:
removes.append(pairings.index(p))
continue
removes.sort(reverse=True)
for i in removes:
pairings.pop(i)
apair = None
for p in pairingsarray[0]:
if p.Name == botb:
apair = p
if apair is None:
apair = structures.ScoreSet(name=botb)
pairingsarray[0].append(apair)
bpair = None
for p in pairingsarray[1]:
if p.Name == bota:
bpair = p
if bpair is None:
bpair = structures.ScoreSet(name=bota)
pairingsarray[1].append(bpair)
#participantsSet = set(game.Participants)
for b, pairings in zip(bots, pairingsarray):
i = 0
while i < len(pairings):
if pairings[i].Name == b.Name:
pairings.pop(i)
continue
if not hasattr(pairings[i], "Alive"):
pairings[i].Alive = True
if pairings[i].Alive and pairings[i].Name not in scores:
pairings[i].Alive = False
i += 1
#b.Pairings = i
aBattles = apair.Battles
#rolling average with a half-life of 10k
maxPerPair = 10000 / len(bots)
if aBattles > maxPerPair:
aBattles = maxPerPair
#bBattles = bpair.Battles
inv_ab = 1.0 / (aBattles + 1.0)
apair.APS *= float(aBattles) * inv_ab
apair.APS += APSa * inv_ab
apair.__dict__["Min_APS"] = min(APSa,
apair.__dict__.get("Min_APS", 100))
#bpair.APS *= float(bBattles)/(bBattles + 1)
#bpair.APS += APSb/(bBattles+1)
bpair.APS = 100 - apair.APS
bpair.__dict__["Min_APS"] = min(100 - APSa,
bpair.__dict__.get("Min_APS", 100))
apair.Survival *= float(aBattles) * inv_ab
apair.Survival += survivala * inv_ab
bpair.Survival *= float(aBattles) * inv_ab
bpair.Survival += survivalb * inv_ab
apair.Battles += 1
bpair.Battles = apair.Battles
apair.LastUpload = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
bpair.LastUpload = apair.LastUpload
for b, pairings in zip(bots, pairingsarray):
aps = 0.0
survival = 0.0
pl = 0
battles = 0
alivePairings = 0
if len(pairings) > 0:
for p in pairings:
if p.Alive:
aps += p.APS
survival += p.Survival
if p.APS > 50:
pl += 1
else:
pl -= 1
battles += p.Battles
alivePairings += 1
aps /= alivePairings
survival /= alivePairings
b.APS = aps
b.Survival = survival
b.PL = pl
b.Battles = battles
b.PairingsList = db.Blob(
zlib.compress(pickle.dumps(pairings, pickle.HIGHEST_PROTOCOL),
1))
b.LastUpload = apair.LastUpload
b.Pairings = alivePairings
game.TotalUploads += 1
#self.response.out.write("<" + str(bots[0].Battles) + " " + str(bots[1].Battles) + ">")
game.LastUpload = apair.LastUpload
game.AvgBattles = game.AvgBattles * 0.99 + 0.005 * (bots[0].Battles +
bots[1].Battles)
if game.Uploaders is None:
uploaders = None
else:
uploaders = pickle.loads(zlib.decompress(game.Uploaders))
if uploaders is None or len(uploaders) == 0:
uploaders = {}
uploaderName = results["user"]
try:
uploader = uploaders[uploaderName]
uploader.latest = apair.LastUpload
uploader.total += 1
except KeyError:
uploader = structures.User(name=uploaderName)
uploaders[uploaderName] = uploader
game.Uploaders = zlib.compress(pickle.dumps(uploaders, -1), 1)
for b in bots:
try:
bUploaders = b.Uploaders
if uploaderName not in bUploaders:
bUploaders.append(uploaderName)
except:
b.__dict__["Uploaders"] = [uploaderName]
with sync_lock:
for b in bots:
key = None
if isinstance(b, structures.BotEntry):
key = b.key().name()
else:
key = b.key_name
botsync[key] = botsync.get(key, 0) + 1
minSize = min(10, len(scores) / 2)
wrote = False
logging.debug("botsync: " + str(len(botsync)))
if len(botsync) > minSize:
syncset = botsync.keys()
syncbotsDict = memcache.get_multi(syncset)
syncbots = []
for sb in syncset:
b = syncbotsDict.get(sb, None)
if b is None or b.PairingsList is None:
syncbotsDict.pop(sb, 1)
else:
syncbots.append(b)
botsync.pop(sb, 1)
try:
thisput = []
while len(syncbots) > 0:
b = syncbots.pop()
key = None
if isinstance(b, structures.BotEntry):
key = b.key().name()
else:
key = b.key_name
putb = structures.BotEntry(key_name=key)
putb.init_from_cache(b)
thisput.append(putb)
db.put(thisput)
logging.info("wrote " + str(len(thisput)) +
" results to database")
for b in thisput:
s = b.key().name()
botsync.pop(s, 1)
syncbotsDict.pop(s, 1)
wrote = True
except Exception, e:
logging.error('Failed to write data: ' + str(e))
def post(self):
try:
#global global_dict
#global_dict = {}
starttime = time.time()
cutoff_date = datetime.datetime.now() + datetime.timedelta(-365)
cutoff_date_string = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")
parts = self.request.body.split("&")
requests = {}
if parts is not None and parts[0] != "":
for pair in parts:
ab = pair.split('=')
requests[ab[0]] = ab[1]
force = bool(requests.get("force", False))
write = bool(requests.get("write", False))
minwrite = bool(requests.get("minwrite", False))
rpcList = []
client = memcache.Client()
q = structures.Rumble.all()
rumbles = []
for r in q.run():
memr = memcache.get(r.Name)
if memr is not None:
r = memr
if r.BatchScoresAccurate and not force:
continue
rumbles.append(r)
for r in rumbles:
scoresdicts = pickle.loads(
zlib.decompress(r.ParticipantsScores))
entries = len(scoresdicts)
r.__dict__["entries"] = entries
rumbles.sort(key=lambda r: -r.__dict__["entries"])
first = True
for r in rumbles:
if not first:
time.sleep(5)
gc.collect()
gc.collect(2)
first = False
logging.info("mem usage at start of " + r.Name + ": " +
str(runtime.memory_usage().current()) + "MB")
try:
scores = pickle.loads(zlib.decompress(
r.ParticipantsScores))
except:
scoresdicts = marshal.loads(
zlib.decompress(r.ParticipantsScores))
scoreslist = [structures.LiteBot() for _ in scoresdicts]
for s, d in zip(scoreslist, scoresdicts):
s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
if len(scores) == 0:
continue
r.ParticipantsScores = None
#gc.collect()
particHash = [p + "|" + r.Name for p in scores]
particSplit = list_split(particHash, 32)
ppDict = {}
for l in particSplit:
ppDict.update(memcache.get_multi(l))
time.sleep(0.1)
particSplit = None
bots = [ppDict.get(h, None) for h in particHash]
botsdict = {}
missingHashes = []
missingIndexes = []
for i in xrange(len(bots)):
if bots[i] is None:
missingHashes.append(particHash[i])
missingIndexes.append(i)
elif isinstance(bots[i], structures.BotEntry):
bots[i] = structures.CachedBotEntry(bots[i])
if len(missingHashes) > 0:
bmis = structures.BotEntry.get_by_key_name(missingHashes)
#lost = False
lostList = []
for i in xrange(len(missingHashes)):
if bmis[i] is not None:
cb = structures.CachedBotEntry(bmis[i])
bots[missingIndexes[i]] = cb
botsdict[missingHashes[i]] = cb
else:
bots[missingIndexes[i]] = None
lostList.append(missingHashes[i])
#lost = True
while len(particHash) > 0:
particHash.pop()
particHash = None
while len(missingHashes) > 0:
missingHashes.pop()
missingHashes = None
while len(missingIndexes) > 0:
missingIndexes.pop()
missingIndexes = None
logging.info("mem usage after loading bots: " +
str(runtime.memory_usage().current()) + "MB")
bots = filter(lambda b: b is not None, bots)
get_key = attrgetter("APS")
bots.sort(key=lambda b: get_key(b), reverse=True)
gc.collect()
botIndexes = {}
for i, b in enumerate(bots):
b.Name = b.Name.encode('ascii')
intern(b.Name)
botIndexes[b.Name] = i
b.VoteScore = 0.
botlen = len(bots)
APSs = numpy.empty([botlen, botlen])
APSs.fill(numpy.nan)
totalAlivePairs = 0
for i, b in enumerate(bots):
try:
pairings = pickle.loads(zlib.decompress(
b.PairingsList))
except:
pairsDicts = marshal.loads(
zlib.decompress(b.PairingsList))
pairings = [structures.ScoreSet() for _ in pairsDicts]
for s, d in zip(pairings, pairsDicts):
s.__dict__.update(d)
removes = []
alivePairings = 0
for q, p in enumerate(pairings):
j = botIndexes.get(p.Name, -1)
if j != -1:
APSs[j, i] = numpy.float64(p.APS)
p.Alive = True
alivePairings += 1
else:
removes.append(q)
b.Pairings = alivePairings
totalAlivePairs += alivePairings
removes.reverse()
removed = False
for q in removes:
p = pairings[q]
if p.LastUpload < cutoff_date_string:
removed = True
pairings.pop(q)
else:
if p.Alive:
removed = True
p.Alive = False
if removed:
b.PairingsList = zlib.compress(
pickle.dumps(pairings, -1), 1)
gc.collect()
APSs += numpy.float64(100) - APSs.transpose()
APSs *= numpy.float64(0.5)
numpy.fill_diagonal(APSs, numpy.nan)
gc.collect()
logging.info(
str(len(bots)) + " bots loaded, total of " +
str(totalAlivePairs) + " alive pairings")
logging.info("mem usage after unzipping pairings: " +
str(runtime.memory_usage().current()) + "MB")
#Vote
mins = numpy.nanmax(APSs, 1)
for i, minimum in enumerate(mins):
minIndexes = numpy.argwhere(APSs[i, ...] == minimum)
ties = len(minIndexes)
if ties > 0:
increment = 1. / ties
for minIndex in minIndexes:
bots[minIndex].VoteScore += increment
#inv_len = 1.0/botlen
for b in bots:
if b.Pairings > 0:
b.VoteScore = 100.0 * b.VoteScore / float(b.Pairings)
else:
b.VoteScore = 0
#KNN PBI
half_k = int(math.ceil(math.sqrt(botlen) / 2))
KNN_PBI = -numpy.ones((botlen, botlen))
for i in xrange(len(bots)):
low_bound = max([0, i - half_k])
high_bound = min([botlen - 1, i + half_k])
low_high_bound = min([i + 1, high_bound])
before = APSs[:, low_bound:i]
after = APSs[:, low_high_bound:high_bound]
compare = numpy.hstack((before, after))
mm = numpy.mean(numpy.ma.masked_array(
compare, numpy.isnan(compare)),
axis=1)
KNN_PBI[:, i] = APSs[:, i] - mm.filled(numpy.nan)
# a[i] = 0
# logging.info("mean error of transpose: " + str(numpy.mean(numpy.square(a))))
#KNN_PBI[KNN_PBI == numpy.nan] = -1
#logging.info("mem usage after KNNPBI: " + str(runtime.memory_usage().current()) + "MB")
# Avg Normalised Pairing Percentage
mins = numpy.nanmin(APSs, 1)
maxs = numpy.nanmax(APSs, 1)
inv_ranges = numpy.float64(1.0) / (maxs - mins)
NPPs = -numpy.ones((botlen, botlen))
for i in range(botlen):
if numpy.isfinite(inv_ranges[i]):
NPPs[i, :] = numpy.float64(100) * (
APSs[i, :] - mins[i]) * inv_ranges[i]
else:
NPPs[i, :] = numpy.float64(100)
#NPPs[NPPs] = -1
#logging.info("mem usage after ANPP: " + str(runtime.memory_usage().current()) + "MB")
changedBots = [] #bots with new pairings since last run
# save to cache
botsdict = {}
for i, b in enumerate(bots):
# try:
pairings = pickle.loads(zlib.decompress(b.PairingsList))
# except:
# pairsDicts = marshal.loads(zlib.decompress(b.PairingsList))
#
# pairings = [structures.ScoreSet() for _ in pairsDicts]
# for s,d in zip(pairings,pairsDicts):
# s.__dict__.update(d)
nppCount = 0
totalNPP = 0.0
apsCount = 0
totalAPS = 0.0
aliveCount = 0
changed = False
for p in pairings:
j = botIndexes.get(p.Name, -1)
if j != -1:
p.Alive = True
changePotential = (p.KNNPBI == 0.0 and p.NPP == -1)
aliveCount += 1
p.KNNPBI = float(KNN_PBI[j, i])
p.NPP = float(NPPs[j, i])
if not numpy.isnan(APSs[j, i]):
p.APS = float(APSs[j, i])
totalAPS += p.APS
apsCount += 1
if numpy.isnan(p.KNNPBI):
p.KNNPBI = 0
if numpy.isnan(p.NPP):
p.NPP = -1
else:
totalNPP += p.NPP
nppCount += 1
if changePotential and p.KNNPBI != 0.0 and p.NPP != -1:
changed = True
else:
p.Alive = False
p.KNNPBI = 0
p.NPP = -1
if nppCount > 0:
b.ANPP = float(totalNPP / nppCount)
else:
b.ANPP = -1.0
if apsCount > 0:
b.APS = float(totalAPS / apsCount)
else:
b.APS = -1.0
b.PairingsList = zlib.compress(pickle.dumps(pairings, -1),
1)
b.Pairings = aliveCount
if b.Pairings > 0:
botsdict[b.key_name] = b
if changed:
changedBots.append(b)
KNN_PBI = None
APSs = None
NPPs = None
logging.info("mem usage after zipping: " +
str(runtime.memory_usage().current()) + "MB")
gc.collect()
#logging.info("mem usage after gc: " + str(runtime.memory_usage().current()) + "MB")
if len(botsdict) > 0:
splitlist = dict_split(botsdict, 20)
logging.info("split bots into " + str(len(splitlist)) +
" sections")
for d in splitlist:
rpcList.append(client.set_multi_async(d))
time.sleep(.5) #throttle
logging.info("wrote " + str(len(botsdict)) +
" bots to memcache")
botsdict.clear()
botsdict = None
scores = {b.Name: structures.LiteBot(b) for b in bots}
# bots = None
r.ParticipantsScores = None
gc.collect()
r.ParticipantsScores = db.Blob(
zlib.compress(
pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 3))
logging.info("mem usage after participants zipping: " +
str(runtime.memory_usage().current()) + "MB")
#r.ParticipantsScores = zlib.compress(marshal.dumps([scores[s].__dict__ for s in scores]),4)
scores = None
if write:
writebots = [None] * len(bots)
for i, b in enumerate(bots):
putb = structures.BotEntry(key_name=b.key_name)
putb.init_from_cache(b)
writebots[i] = putb
write_lists = list_split(writebots, 50)
for subset in write_lists:
db.put(subset)
time.sleep(0.1) #throttle
logging.info("wrote " + str(len(writebots)) +
" bots to database")
while len(bots) > 0:
bots.pop()
bots = None
if minwrite:
writebots = [None] * len(changedBots)
for i, b in enumerate(changedBots):
putb = structures.BotEntry(key_name=b.key_name)
putb.init_from_cache(b)
writebots[i] = putb
write_lists = list_split(writebots, 50)
for subset in write_lists:
db.put(subset)
time.sleep(0.1)
logging.info("wrote " + str(len(writebots)) +
" changed bots to database")
while len(changedBots) > 0:
changedBots.pop()
changedBots = None
gc.collect()
if write or minwrite:
r.BatchScoresAccurate = True
rpcList.append(client.set_multi_async({r.Name: r}))
db.put([r])
#gc.collect()
r = None
logging.info("mem usage after write: " +
str(runtime.memory_usage().current()) + "MB")
for rpc in rpcList:
rpc.get_result()
elapsed = time.time() - starttime
logging.info("Success in " + str(round(1000 * elapsed) / 1000) +
"s")
self.response.out.write("Success in " +
str(round(1000 * elapsed)) + "ms")
except:
logging.exception('')
elapsed = time.time() - starttime
logging.info("Error in " + str(round(1000 * elapsed) / 1000) + "s")
self.response.out.write("Error in " + str(round(1000 * elapsed)) +
"ms")