def post(self):
global global_dict
#global_dict = {}
global global_sync
global locks
global last_write
#global sync_lock
# starttime = time.time()
results = json.loads(self.request.body)
rumble = results.get("game", "ERROR")
syncname = str(
bool(results.get("melee") == "YES")) + "|" + structures.sync
if syncname not in locks:
locks[syncname] = threading.Lock()
sync_lock = locks[syncname]
global_sync[syncname] = global_sync.get(syncname, {})
botsync = global_sync[syncname]
bota = results.get("fname")
botb = results.get("sname")
#bota = rreplace(bota,"_"," ",1)
#botb = rreplace(botb,"_"," ",1)
logging.debug("Bots : " + bota + " vs. " + botb)
bd = [[bota, rumble], [botb, rumble]]
botHashes = [string.join(a, "|").encode('ascii') for a in bd]
memget = [rumble]
memget.extend(botHashes) #botHashes.append(rumble)
memdict = memcache.get_multi(memget)
game = memdict.get(rumble, None)
game_future = None
if game is None:
#game = structures.Rumble.get_by_key_name(rumble)
game_future = db.get_async(db.Key.from_path('Rumble', rumble))
newBot = False
bots = [memdict.get(h, None) for h in botHashes]
pairingsarray = [[], []]
botFutures = [None, None]
for i in [0, 1]:
if bots[i] is None or bots[i].PairingsList is None:
botFutures[i] = db.get_async(
db.Key.from_path('BotEntry', botHashes[i]))
for i in [0, 1]:
if botFutures[i] is not None:
modelbot = botFutures[i].get_result()
if modelbot is not None:
bots[i] = structures.CachedBotEntry(modelbot)
#logging.debug("retrieved from database")
for i in [0, 1]:
if bots[i] is None:
modelbot = structures.BotEntry(key_name=botHashes[i],
Name=bd[i][0],
Battles=0,
Pairings=0,
APS=0.0,
Survival=0.0,
PL=0,
Rumble=rumble,
Active=False,
PairingsList=zlib.compress(
pickle.dumps([]), 1))
bots[i] = structures.CachedBotEntry(modelbot)
newBot = True
if isinstance(bots[i], structures.BotEntry):
bots[i] = structures.CachedBotEntry(bots[i])
try:
pairingsarray[i] = pickle.loads(
zlib.decompress(bots[i].PairingsList))
bots[i].PairingsList = None
except:
try:
pairsDicts = marshal.loads(
zlib.decompress(bots[i].PairingsList))
pairingsarray[i] = [
structures.ScoreSet() for _ in pairsDicts
]
for s, d in zip(pairingsarray[i], pairsDicts):
s.__dict__.update(d)
bots[i].PairingsList = None
except:
pairingsarray[i] = []
if game_future is not None:
game = game_future.get_result()
if game is None:
game = structures.Rumble(key_name=rumble,
Name=rumble,
Rounds=int(results["rounds"]),
Field=results["field"],
Melee=bool(results["melee"] == "YES"),
Teams=bool(results["teams"] == "YES"),
TotalUploads=0,
MeleeSize=10,
ParticipantsScores=db.Blob(
zlib.compress(pickle.dumps([]))))
self.response.out.write("CREATED NEW GAME TYPE " + rumble + "\n")
logging.info("Created new game type: " + rumble)
else:
field = game.Field == results["field"]
rounds = (game.Rounds == int(results["rounds"]))
teams = game.Teams == bool(results["teams"] == "YES")
melee = game.Melee == bool(results["melee"] == "YES")
allowed = field and rounds and teams and melee
if not allowed:
errstr = "OK. ERROR. Incorrect " + rumble + " config: "
errorReasons = []
if not field:
errorReasons.append("field size ")
if not rounds:
errorReasons.append("number of rounds ")
if not teams:
errorReasons.append("teams ")
if not melee:
errorReasons.append("melee ")
logging.error(errstr + string.join(errorReasons, ", ") +
" User: "******"user"])
return
scores = None
try:
scores = pickle.loads(zlib.decompress(game.ParticipantsScores))
game.ParticipantsScores = None
if len(scores) == 0:
scores = {}
except:
scoresdicts = marshal.loads(
zlib.decompress(game.ParticipantsScores))
game.ParticipantsScores = None
scoreslist = [structures.LiteBot(loadDict=d) for d in scoresdicts]
#for s,d in zip(scoreslist,scoresdicts):
# s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
if len(scores) == 0:
scores = {}
#logging.debug("uncompressed scores: " + str(len(s)) + " compressed: " + str(a))
for i in [0, 1]:
if not bots[i].Active or bots[i].Name not in scores:
bots[i].Active = True
scores[bots[i].Name] = structures.LiteBot(bots[i])
newBot = True
self.response.out.write("Added " + bd[i][0] + " to " + rumble +
"\n")
logging.info("added new bot!")
#retrievetime = time.time() - starttime
scorea = float(results["fscore"])
scoreb = float(results["sscore"])
if scorea + scoreb > 0:
APSa = 100 * scorea / (scorea + scoreb)
else:
APSa = 50 #register a 0/0 as 50%
#APSb = 100 - APSa
survivala = float(results["fsurvival"])
survivalb = float(results["ssurvival"])
survivala = 100.0 * survivala / game.Rounds
survivalb = 100.0 * survivalb / game.Rounds
for b, pairings in zip(bots, pairingsarray):
#b.PairingsList = zlib.compress(marshal.dumps([s.__dict__ for s in pairings]),1)
if len(pairings) > 0:
removes = []
for p in pairings:
try:
p.APS = float(p.APS)
p.Survival = float(p.Survival)
p.Battles = int(p.Battles)
except:
removes.append(pairings.index(p))
continue
removes.sort(reverse=True)
for i in removes:
pairings.pop(i)
apair = None
for p in pairingsarray[0]:
if p.Name == botb:
apair = p
if apair is None:
apair = structures.ScoreSet(name=botb)
pairingsarray[0].append(apair)
bpair = None
for p in pairingsarray[1]:
if p.Name == bota:
bpair = p
if bpair is None:
bpair = structures.ScoreSet(name=bota)
pairingsarray[1].append(bpair)
#participantsSet = set(game.Participants)
for b, pairings in zip(bots, pairingsarray):
i = 0
while i < len(pairings):
if pairings[i].Name == b.Name:
pairings.pop(i)
continue
if not hasattr(pairings[i], "Alive"):
pairings[i].Alive = True
if pairings[i].Alive and pairings[i].Name not in scores:
pairings[i].Alive = False
i += 1
#b.Pairings = i
aBattles = apair.Battles
#rolling average with a half-life of 10k
maxPerPair = 10000 / len(bots)
if aBattles > maxPerPair:
aBattles = maxPerPair
#bBattles = bpair.Battles
inv_ab = 1.0 / (aBattles + 1.0)
apair.APS *= float(aBattles) * inv_ab
apair.APS += APSa * inv_ab
apair.__dict__["Min_APS"] = min(APSa,
apair.__dict__.get("Min_APS", 100))
#bpair.APS *= float(bBattles)/(bBattles + 1)
#bpair.APS += APSb/(bBattles+1)
bpair.APS = 100 - apair.APS
bpair.__dict__["Min_APS"] = min(100 - APSa,
bpair.__dict__.get("Min_APS", 100))
apair.Survival *= float(aBattles) * inv_ab
apair.Survival += survivala * inv_ab
bpair.Survival *= float(aBattles) * inv_ab
bpair.Survival += survivalb * inv_ab
apair.Battles += 1
bpair.Battles = apair.Battles
apair.LastUpload = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
bpair.LastUpload = apair.LastUpload
for b, pairings in zip(bots, pairingsarray):
aps = 0.0
survival = 0.0
pl = 0
battles = 0
alivePairings = 0
if len(pairings) > 0:
for p in pairings:
if p.Alive:
aps += p.APS
survival += p.Survival
if p.APS > 50:
pl += 1
else:
pl -= 1
battles += p.Battles
alivePairings += 1
aps /= alivePairings
survival /= alivePairings
b.APS = aps
b.Survival = survival
b.PL = pl
b.Battles = battles
b.PairingsList = db.Blob(
zlib.compress(pickle.dumps(pairings, pickle.HIGHEST_PROTOCOL),
1))
b.LastUpload = apair.LastUpload
b.Pairings = alivePairings
game.TotalUploads += 1
#self.response.out.write("<" + str(bots[0].Battles) + " " + str(bots[1].Battles) + ">")
game.LastUpload = apair.LastUpload
game.AvgBattles = game.AvgBattles * 0.99 + 0.005 * (bots[0].Battles +
bots[1].Battles)
if game.Uploaders is None:
uploaders = None
else:
uploaders = pickle.loads(zlib.decompress(game.Uploaders))
if uploaders is None or len(uploaders) == 0:
uploaders = {}
uploaderName = results["user"]
try:
uploader = uploaders[uploaderName]
uploader.latest = apair.LastUpload
uploader.total += 1
except KeyError:
uploader = structures.User(name=uploaderName)
uploaders[uploaderName] = uploader
game.Uploaders = zlib.compress(pickle.dumps(uploaders, -1), 1)
for b in bots:
try:
bUploaders = b.Uploaders
if uploaderName not in bUploaders:
bUploaders.append(uploaderName)
except:
b.__dict__["Uploaders"] = [uploaderName]
with sync_lock:
for b in bots:
key = None
if isinstance(b, structures.BotEntry):
key = b.key().name()
else:
key = b.key_name
botsync[key] = botsync.get(key, 0) + 1
minSize = min(10, len(scores) / 2)
wrote = False
logging.debug("botsync: " + str(len(botsync)))
if len(botsync) > minSize:
syncset = botsync.keys()
syncbotsDict = memcache.get_multi(syncset)
syncbots = []
for sb in syncset:
b = syncbotsDict.get(sb, None)
if b is None or b.PairingsList is None:
syncbotsDict.pop(sb, 1)
else:
syncbots.append(b)
botsync.pop(sb, 1)
try:
thisput = []
while len(syncbots) > 0:
b = syncbots.pop()
key = None
if isinstance(b, structures.BotEntry):
key = b.key().name()
else:
key = b.key_name
putb = structures.BotEntry(key_name=key)
putb.init_from_cache(b)
thisput.append(putb)
db.put(thisput)
logging.info("wrote " + str(len(thisput)) +
" results to database")
for b in thisput:
s = b.key().name()
botsync.pop(s, 1)
syncbotsDict.pop(s, 1)
wrote = True
except Exception, e:
logging.error('Failed to write data: ' + str(e))
def removeFromRumble(self, requests):
#global global_dict
global_dict = {}
if "version" not in requests or requests["version"] is not "1":
return "ERROR. bad/no version"
game = requests.get("game", None)
if game is None:
return "ERROR. no game specified"
name = requests.get("name", None)
if name is None:
return "ERROR. no name specified"
while name.count("%20") > 0:
name = rreplace(name, "%20", " ", 1)
rumble = global_dict.get(game, None)
if rumble is None:
rumble = memcache.get(game)
if rumble is None:
rumble = structures.Rumble.get_by_key_name(game)
entry = None
if name.count(" ") == 0:
num_underscores = name.count("_")
error_messages = []
for n in range(num_underscores):
check_name = nth_repl(name, "_", " ", n + 1)
keyhash = check_name + "|" + game
entry = global_dict.get(keyhash, None)
if entry is None:
entry = memcache.get(keyhash)
if entry is None:
entry = structures.BotEntry.get_by_key_name(keyhash)
if entry is None:
error_messages.append(
"ERROR. name/game does not exist: " + check_name +
"/" + game)
else:
entry = structures.CachedBotEntry(entry)
if entry is not None:
name = check_name
break
if entry is None:
return "\n".join(error_messages)
else:
return "ERROR. Bot name missing version."
if isinstance(entry, structures.BotEntry):
entry = structures.CachedBotEntry(entry)
global_dict.pop(keyhash, 0)
memcache.delete(keyhash)
entry.Active = False
try:
scores = pickle.loads(zlib.decompress(rumble.ParticipantsScores))
except:
scoresdicts = marshal.loads(zlib.decompress(rumble.ParticipantsScores))
scoreslist = [structures.LiteBot() for _ in scoresdicts]
for s, d in zip(scoreslist, scoresdicts):
s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
scores.pop(name, 1)
rumble.ParticipantsScores = zlib.compress(
pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 4)
memcache.delete("home")
global_dict.pop("home", 0)
memcache.set(entry.key_name, entry)
global_dict[entry.key_name] = entry
modelBot = structures.BotEntry(key_name=entry.key_name)
modelBot.init_from_cache(entry)
modelBot.put()
global_dict[game] = rumble
memcache.set(game, rumble)
rumble.put()
return "OK. " + name + " retired from " + game
def post(self):
try:
#global global_dict
#global_dict = {}
starttime = time.time()
cutoff_date = datetime.datetime.now() + datetime.timedelta(-365)
cutoff_date_string = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")
parts = self.request.body.split("&")
requests = {}
if parts is not None and parts[0] != "":
for pair in parts:
ab = pair.split('=')
requests[ab[0]] = ab[1]
force = bool(requests.get("force", False))
write = bool(requests.get("write", False))
minwrite = bool(requests.get("minwrite", False))
rpcList = []
client = memcache.Client()
q = structures.Rumble.all()
rumbles = []
for r in q.run():
memr = memcache.get(r.Name)
if memr is not None:
r = memr
if r.BatchScoresAccurate and not force:
continue
rumbles.append(r)
for r in rumbles:
scoresdicts = pickle.loads(
zlib.decompress(r.ParticipantsScores))
entries = len(scoresdicts)
r.__dict__["entries"] = entries
rumbles.sort(key=lambda r: -r.__dict__["entries"])
first = True
for r in rumbles:
if not first:
time.sleep(5)
gc.collect()
gc.collect(2)
first = False
logging.info("mem usage at start of " + r.Name + ": " +
str(runtime.memory_usage().current()) + "MB")
try:
scores = pickle.loads(zlib.decompress(
r.ParticipantsScores))
except:
scoresdicts = marshal.loads(
zlib.decompress(r.ParticipantsScores))
scoreslist = [structures.LiteBot() for _ in scoresdicts]
for s, d in zip(scoreslist, scoresdicts):
s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
if len(scores) == 0:
continue
r.ParticipantsScores = None
#gc.collect()
particHash = [p + "|" + r.Name for p in scores]
particSplit = list_split(particHash, 32)
ppDict = {}
for l in particSplit:
ppDict.update(memcache.get_multi(l))
time.sleep(0.1)
particSplit = None
bots = [ppDict.get(h, None) for h in particHash]
botsdict = {}
missingHashes = []
missingIndexes = []
for i in xrange(len(bots)):
if bots[i] is None:
missingHashes.append(particHash[i])
missingIndexes.append(i)
elif isinstance(bots[i], structures.BotEntry):
bots[i] = structures.CachedBotEntry(bots[i])
if len(missingHashes) > 0:
bmis = structures.BotEntry.get_by_key_name(missingHashes)
#lost = False
lostList = []
for i in xrange(len(missingHashes)):
if bmis[i] is not None:
cb = structures.CachedBotEntry(bmis[i])
bots[missingIndexes[i]] = cb
botsdict[missingHashes[i]] = cb
else:
bots[missingIndexes[i]] = None
lostList.append(missingHashes[i])
#lost = True
while len(particHash) > 0:
particHash.pop()
particHash = None
while len(missingHashes) > 0:
missingHashes.pop()
missingHashes = None
while len(missingIndexes) > 0:
missingIndexes.pop()
missingIndexes = None
logging.info("mem usage after loading bots: " +
str(runtime.memory_usage().current()) + "MB")
bots = filter(lambda b: b is not None, bots)
get_key = attrgetter("APS")
bots.sort(key=lambda b: get_key(b), reverse=True)
gc.collect()
botIndexes = {}
for i, b in enumerate(bots):
b.Name = b.Name.encode('ascii')
intern(b.Name)
botIndexes[b.Name] = i
b.VoteScore = 0.
botlen = len(bots)
APSs = numpy.empty([botlen, botlen])
APSs.fill(numpy.nan)
totalAlivePairs = 0
for i, b in enumerate(bots):
try:
pairings = pickle.loads(zlib.decompress(
b.PairingsList))
except:
pairsDicts = marshal.loads(
zlib.decompress(b.PairingsList))
pairings = [structures.ScoreSet() for _ in pairsDicts]
for s, d in zip(pairings, pairsDicts):
s.__dict__.update(d)
removes = []
alivePairings = 0
for q, p in enumerate(pairings):
j = botIndexes.get(p.Name, -1)
if j != -1:
APSs[j, i] = numpy.float64(p.APS)
p.Alive = True
alivePairings += 1
else:
removes.append(q)
b.Pairings = alivePairings
totalAlivePairs += alivePairings
removes.reverse()
removed = False
for q in removes:
p = pairings[q]
if p.LastUpload < cutoff_date_string:
removed = True
pairings.pop(q)
else:
if p.Alive:
removed = True
p.Alive = False
if removed:
b.PairingsList = zlib.compress(
pickle.dumps(pairings, -1), 1)
gc.collect()
APSs += numpy.float64(100) - APSs.transpose()
APSs *= numpy.float64(0.5)
numpy.fill_diagonal(APSs, numpy.nan)
gc.collect()
logging.info(
str(len(bots)) + " bots loaded, total of " +
str(totalAlivePairs) + " alive pairings")
logging.info("mem usage after unzipping pairings: " +
str(runtime.memory_usage().current()) + "MB")
#Vote
mins = numpy.nanmax(APSs, 1)
for i, minimum in enumerate(mins):
minIndexes = numpy.argwhere(APSs[i, ...] == minimum)
ties = len(minIndexes)
if ties > 0:
increment = 1. / ties
for minIndex in minIndexes:
bots[minIndex].VoteScore += increment
#inv_len = 1.0/botlen
for b in bots:
if b.Pairings > 0:
b.VoteScore = 100.0 * b.VoteScore / float(b.Pairings)
else:
b.VoteScore = 0
#KNN PBI
half_k = int(math.ceil(math.sqrt(botlen) / 2))
KNN_PBI = -numpy.ones((botlen, botlen))
for i in xrange(len(bots)):
low_bound = max([0, i - half_k])
high_bound = min([botlen - 1, i + half_k])
low_high_bound = min([i + 1, high_bound])
before = APSs[:, low_bound:i]
after = APSs[:, low_high_bound:high_bound]
compare = numpy.hstack((before, after))
mm = numpy.mean(numpy.ma.masked_array(
compare, numpy.isnan(compare)),
axis=1)
KNN_PBI[:, i] = APSs[:, i] - mm.filled(numpy.nan)
# a[i] = 0
# logging.info("mean error of transpose: " + str(numpy.mean(numpy.square(a))))
#KNN_PBI[KNN_PBI == numpy.nan] = -1
#logging.info("mem usage after KNNPBI: " + str(runtime.memory_usage().current()) + "MB")
# Avg Normalised Pairing Percentage
mins = numpy.nanmin(APSs, 1)
maxs = numpy.nanmax(APSs, 1)
inv_ranges = numpy.float64(1.0) / (maxs - mins)
NPPs = -numpy.ones((botlen, botlen))
for i in range(botlen):
if numpy.isfinite(inv_ranges[i]):
NPPs[i, :] = numpy.float64(100) * (
APSs[i, :] - mins[i]) * inv_ranges[i]
else:
NPPs[i, :] = numpy.float64(100)
#NPPs[NPPs] = -1
#logging.info("mem usage after ANPP: " + str(runtime.memory_usage().current()) + "MB")
changedBots = [] #bots with new pairings since last run
# save to cache
botsdict = {}
for i, b in enumerate(bots):
# try:
pairings = pickle.loads(zlib.decompress(b.PairingsList))
# except:
# pairsDicts = marshal.loads(zlib.decompress(b.PairingsList))
#
# pairings = [structures.ScoreSet() for _ in pairsDicts]
# for s,d in zip(pairings,pairsDicts):
# s.__dict__.update(d)
nppCount = 0
totalNPP = 0.0
apsCount = 0
totalAPS = 0.0
aliveCount = 0
changed = False
for p in pairings:
j = botIndexes.get(p.Name, -1)
if j != -1:
p.Alive = True
changePotential = (p.KNNPBI == 0.0 and p.NPP == -1)
aliveCount += 1
p.KNNPBI = float(KNN_PBI[j, i])
p.NPP = float(NPPs[j, i])
if not numpy.isnan(APSs[j, i]):
p.APS = float(APSs[j, i])
totalAPS += p.APS
apsCount += 1
if numpy.isnan(p.KNNPBI):
p.KNNPBI = 0
if numpy.isnan(p.NPP):
p.NPP = -1
else:
totalNPP += p.NPP
nppCount += 1
if changePotential and p.KNNPBI != 0.0 and p.NPP != -1:
changed = True
else:
p.Alive = False
p.KNNPBI = 0
p.NPP = -1
if nppCount > 0:
b.ANPP = float(totalNPP / nppCount)
else:
b.ANPP = -1.0
if apsCount > 0:
b.APS = float(totalAPS / apsCount)
else:
b.APS = -1.0
b.PairingsList = zlib.compress(pickle.dumps(pairings, -1),
1)
b.Pairings = aliveCount
if b.Pairings > 0:
botsdict[b.key_name] = b
if changed:
changedBots.append(b)
KNN_PBI = None
APSs = None
NPPs = None
logging.info("mem usage after zipping: " +
str(runtime.memory_usage().current()) + "MB")
gc.collect()
#logging.info("mem usage after gc: " + str(runtime.memory_usage().current()) + "MB")
if len(botsdict) > 0:
splitlist = dict_split(botsdict, 20)
logging.info("split bots into " + str(len(splitlist)) +
" sections")
for d in splitlist:
rpcList.append(client.set_multi_async(d))
time.sleep(.5) #throttle
logging.info("wrote " + str(len(botsdict)) +
" bots to memcache")
botsdict.clear()
botsdict = None
scores = {b.Name: structures.LiteBot(b) for b in bots}
# bots = None
r.ParticipantsScores = None
gc.collect()
r.ParticipantsScores = db.Blob(
zlib.compress(
pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 3))
logging.info("mem usage after participants zipping: " +
str(runtime.memory_usage().current()) + "MB")
#r.ParticipantsScores = zlib.compress(marshal.dumps([scores[s].__dict__ for s in scores]),4)
scores = None
if write:
writebots = [None] * len(bots)
for i, b in enumerate(bots):
putb = structures.BotEntry(key_name=b.key_name)
putb.init_from_cache(b)
writebots[i] = putb
write_lists = list_split(writebots, 50)
for subset in write_lists:
db.put(subset)
time.sleep(0.1) #throttle
logging.info("wrote " + str(len(writebots)) +
" bots to database")
while len(bots) > 0:
bots.pop()
bots = None
if minwrite:
writebots = [None] * len(changedBots)
for i, b in enumerate(changedBots):
putb = structures.BotEntry(key_name=b.key_name)
putb.init_from_cache(b)
writebots[i] = putb
write_lists = list_split(writebots, 50)
for subset in write_lists:
db.put(subset)
time.sleep(0.1)
logging.info("wrote " + str(len(writebots)) +
" changed bots to database")
while len(changedBots) > 0:
changedBots.pop()
changedBots = None
gc.collect()
if write or minwrite:
r.BatchScoresAccurate = True
rpcList.append(client.set_multi_async({r.Name: r}))
db.put([r])
#gc.collect()
r = None
logging.info("mem usage after write: " +
str(runtime.memory_usage().current()) + "MB")
for rpc in rpcList:
rpc.get_result()
elapsed = time.time() - starttime
logging.info("Success in " + str(round(1000 * elapsed) / 1000) +
"s")
self.response.out.write("Success in " +
str(round(1000 * elapsed)) + "ms")
except:
logging.exception('')
elapsed = time.time() - starttime
logging.info("Error in " + str(round(1000 * elapsed) / 1000) + "s")
self.response.out.write("Error in " + str(round(1000 * elapsed)) +
"ms")