def test_player_class_distribution(_mock_lock):
redis = fakeredis.FakeStrictRedis()
distribution = get_player_class_distribution("FT_STANDARD", redis)
current_ts = datetime.utcnow()
# t_0 is 5 minutes in the past
t_0 = current_ts - timedelta(seconds=300,
microseconds=current_ts.microsecond)
actual_games = defaultdict(int)
actual_wins = defaultdict(int)
# First populate the distribution for the last 5 minutes
t_i = None
for i in range(301):
t_i = t_0 + timedelta(seconds=i)
games_per_second = randrange(3, 14)
for game_num in range(games_per_second):
win = bool(randrange(0, 2))
player_class = CardClass(randrange(2, 11))
actual_games[player_class.name] += 1
if win:
actual_wins[player_class.name] += 1
distribution.increment(key=player_class.name, win=win, as_of=t_i)
# Then validate the data distribution
data = distribution.distribution(start_ts=t_0, end_ts=t_i)
for i in range(2, 11):
player_class = CardClass(i)
assert player_class.name in data
player_class_data = data[player_class.name]
assert player_class_data["games"] == actual_games[player_class.name]
assert player_class_data["wins"] == actual_wins[player_class.name]
def setup_game():
"""
initializes a game between two players
Returns:
game: A game entity representing the start of the game after the mulligan phase
"""
#choose classes (priest, rogue, shaman, warlock)
p1 = random.randint(6, 9)
p2 = random.randint(6, 9)
#initialize players and randomly draft decks
#pdb.set_trace()
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
player1 = Player("Player1", deck1, CardClass(p1).default_hero)
player2 = Player("Player2", deck2, CardClass(p2).default_hero)
#begin the game
game = Game(players=(player1, player2))
game.start()
#Skip mulligan for now
for player in game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
return game
def getInitGame(self):
"""
Returns:
startBoard: a representation of the board (ideally this is the form
that will be the input to your neural network)
"""
if self.isolate:
self.isolateSet()
cards.db.initialize()
if self.is_basic: #create quick simple game
# with open('notbasic.data', 'rb') as f:
# extra_set = pickle.load(f)
extra_set = cards.filter(card_set=[
CardSet.EXPERT1, CardSet.HOF, CardSet.NAXX, CardSet.GVG,
CardSet.BRM, CardSet.TGT, CardSet.LOE, CardSet.OG, CardSet.
KARA, CardSet.GANGS, CardSet.UNGORO, CardSet.ICECROWN, CardSet.
LOOTAPALOOZA, CardSet.GILNEAS, CardSet.BOOMSDAY, CardSet.TROLL
])
# LOOTAPALOOZA = Kobolds and Catacombs # GILNEAS = Witchwood # TROLL = Rasthakan's Rumble
# p1 = 6 #priest
p1 = 7 #rogue
p2 = 7 #rogue
# p1 = 4 # mage
# p2 = 4 # mage
# deck1 = random_draft(CardClass(p1), exclude=extra_set)
# deck2 = random_draft(CardClass(p2), exclude=extra_set)
deck1 = self.roguebasic_draft(
) # use same shuffled rogue AI basic decks for now
deck2 = self.roguebasic_draft()
else:
p1 = random.randint(1, 9)
p2 = random.randint(1, 9)
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
self.players[0] = Player("Player1", deck1, CardClass(p1).default_hero)
self.players[1] = Player("Player2", deck2, CardClass(p2).default_hero)
game = Game(players=self.players)
game.start()
# Skip mulligan for now (only mulligan expensive cards)
for player in game.players:
# if player.name == 'Player1':
# cards_to_mulligan = [c for c in player.choice.cards if c.cost > 3]
# else:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
# track played card list
self.players[0].playedcards = []
self.players[1].playedcards = []
#game.player_to_start = game.current_player # obsolete?
self.game = game
return game
def chartifyData(self, theDateUpdated=""):
result = []
i = 1
for aCC in self.myClassClusters:
plt.figure(i)
myXs = []
myYs = []
myLabels = []
clusters= {}
for cluster in aCC.clusters:
if cluster.name not in clusters:
clusters[cluster.name] = []
for dp in cluster.decks:
myXs.append(dp["x"])
myYs.append(dp["y"])
myLabels.append(dp.classification)
clusters[cluster.name].append(tuple((dp["x"], dp["y"])))
#print(clusters)
for c in clusters:
first = [t[0] for t in clusters[c]]
second = [t[1] for t in clusters[c]]
plt.scatter(first, second, label=c)
plt.title(CardClass(aCC.inGameClass).name)
plt.ylabel("y")
plt.xlabel("x")
plt.legend()
i+=1
result.append(tuple((myXs, myYs, myLabels)))
plt.show()
return result
def setup_basic_game():
p1 = 6 #priest
p2 = 7 #rogue
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
player1 = Player("Player1", deck1, CardClass(p1).default_hero)
player2 = Player("Player2", deck2, CardClass(p2).default_hero)
game = Game(players=(player1, player2))
game.start()
#Skip mulligan
for player in game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
return game
def handle(self, *args, **options): cursor = connection.cursor() lookback = options["lookback"] cursor.execute(QUERY % lookback) with open(options["out"], mode="wt") as out: for row in cursor.fetchall(): record = "%s:%s\n" % (CardClass(row[0]).name, row[1]) out.write(record)
def handle(self, *args, **options):
conn = redshift.get_new_redshift_connection()
lookback_val = options["look_back"]
if lookback_val:
lookback = int(lookback_val[0])
else:
lookback = 14
end_ts = date.today() - timedelta(days=1)
start_ts = end_ts - timedelta(days=lookback)
redis_host = options["redis_host"]
if redis_host:
redis_client = redis.StrictRedis(host=redis_host[0])
pipeline = redis_client.pipeline(transaction=False)
else:
pipeline = None
params = {"start_date": start_ts, "end_date": end_ts}
compiled_statement = REDSHIFT_QUERY.params(params).compile(bind=conn)
start_ts = time.time()
for row in conn.execute(compiled_statement):
as_of = row["match_start"]
deck_id = row["deck_id"]
dbf_map = {
dbf_id: count
for dbf_id, count in json.loads(row["deck_list"])
}
player_class = CardClass(row["player_class"])
format = FormatType.FT_STANDARD if row[
"game_type"] == 2 else FormatType.FT_WILD
played_cards = json.loads(row["played_cards"])
tree = deck_prediction_tree(player_class,
format,
redis_client=pipeline)
min_played_cards = tree.max_depth - 1
played_card_dbfs = played_cards[:min_played_cards]
deck_size = sum(dbf_map.values())
if deck_size == 30:
tree.observe(deck_id, dbf_map, played_card_dbfs, as_of=as_of)
if len(pipeline) >= 8000:
pipeline.execute()
if len(pipeline):
pipeline.execute()
end_ts = time.time()
duration_seconds = round(end_ts - start_ts)
print("Took: %i Seconds" % duration_seconds)
def getInitGame(self):
"""
Returns:
startBoard: a representation of the board (ideally this is the form
that will be the input to your neural network)
"""
if self.isolate:
self.isolateSet()
cards.db.initialize()
if self.is_basic: #create quick simple game
with open('notbasic.data', 'rb') as f:
extra_set = pickle.load(f)
p1 = 6 #priest
p2 = 7 #rogue
deck1 = random_draft(CardClass(p1), exclude=extra_set)
deck2 = random_draft(CardClass(p2), exclude=extra_set)
else:
p1 = random.randint(1, 9)
p2 = random.randint(1, 9)
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
self.players[0] = Player("Player1", deck1, CardClass(p1).default_hero)
self.players[1] = Player("Player2", deck2, CardClass(p2).default_hero)
game = Game(players=self.players)
game.start()
# Skip mulligan for now
for player in game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
game.player_to_start = game.current_player
self.game = game
return game
def initGame(self):
cards.db.initialize()
if self.is_basic: #create quick simple game
with open('notbasic.data', 'rb') as f:
extra_set = pickle.load(f)
p1 = 6 #priest
p2 = 7 #rogue
deck1 = random_draft(CardClass(p1), exclude=extra_set)
deck2 = random_draft(CardClass(p2), exclude=extra_set)
else:
p1 = random.randint(1, 9)
p2 = random.randint(1, 9)
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
Board.players[0] = Player("Player1", deck1, CardClass(p1).default_hero)
Board.players[1] = Player("Player2", deck2, CardClass(p2).default_hero)
game = Game(players=self.players)
game.start()
# Skip mulligan for now
for player in game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
# self.start_player = game.current_player
game.player_to_start = game.current_player
Board.game = game
return game
def initGame(self):
self.init_envi()
if self.is_basic: #create quick simple game
p1 = 6 #priest
p2 = 7 #rogue
else:
p1 = random.randint(1, 9)
p2 = random.randint(1, 9)
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
self.players[0] = Player("Player1", deck1, CardClass(p1).default_hero)
self.players[1] = Player("Player2", deck2, CardClass(p2).default_hero)
game = Game(players=self.players)
game.start()
#Skip mulligan for now
for player in self.game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
return game
def inverse_lookup_table(game_format: FormatType,
player_class: CardClass,
redis_client=None) -> BaseInverseLookupTable:
if not redis_client:
from django.core.cache import caches
redis_client = caches["ilt_deck_prediction"].client.get_client()
game_format = FormatType(int(game_format))
player_class = CardClass(int(player_class))
required_cards = ClusterSnapshot.objects.get_required_cards_for_player_class(
game_format, player_class)
return RedisInverseLookupTable(redis_client, game_format, player_class,
required_cards)
def init_game(self):
"""
initializes a game between two players
Returns:
game: A game entity representing the start of the game after the mulligan phase
"""
if self.is_basic: #create quick simple game
p1 = 6 #priest
p2 = 7 #rogue
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
self.players[0] = Player("Player1", deck1,
CardClass(p1).default_hero)
self.players[1] = Player("Player2", deck2,
CardClass(p2).default_hero)
self.game = Game(players=self.players)
self.game.start()
#Skip mulligan
for player in self.game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
return self.game
else:
p1 = random.randint(1, 9)
p2 = random.randint(1, 9)
#initialize players and randomly draft decks
#pdb.set_trace()
deck1 = random_draft(CardClass(p1))
deck2 = random_draft(CardClass(p2))
self.players[0] = Player("Player1", deck1,
CardClass(p1).default_hero)
self.players[1] = Player("Player2", deck2,
CardClass(p2).default_hero)
#begin the game
self.game = Game(players=self.players)
self.game.start()
#Skip mulligan for now
for player in self.game.players:
cards_to_mulligan = random.sample(player.choice.cards, 0)
player.choice.choose(*cards_to_mulligan)
return self.game
def deck_prediction_tree(player_class, game_format, redis_client=None):
from django.core.cache import caches
player_class = CardClass(int(player_class))
game_format = FormatType(int(game_format))
if redis_client is None:
redis_primary = caches["deck_prediction_primary"]
redis_replica = _get_random_cache(
caches, "deck_prediction_replica") or redis_primary
redis_primary = redis_primary.client.get_client()
redis_replica = redis_replica.client.get_client()
else:
redis_primary = redis_client
redis_replica = redis_primary
return DeckPredictionTree(player_class, game_format, redis_primary,
redis_replica)
def canonical_decks(request):
result = []
archetypes = Archetype.objects.prefetch_related(
"canonical_decks__deck__includes").all()
for archetype in archetypes:
record = {
"name": archetype.name,
"archetype_id": archetype.id,
"player_class_id": archetype.player_class,
"player_class_name": CardClass(archetype.player_class).name
}
canonical_deck = archetype.canonical_decks.order_by("-created").first()
if canonical_deck:
record["representative_deck"] = {
"card_ids": canonical_deck.deck.card_id_list(),
"digest": canonical_deck.deck.digest
}
result.append(record)
return JsonResponse(result, safe=False)
def random_class():
return CardClass(random.randint(2, 10))
def handle(self, *args, **options):
conn = redshift.get_new_redshift_connection()
is_dry_run = options["dry_run"]
verbosity = options["verbosity"]
end_ts = date.today()
start_ts = end_ts - timedelta(days=options["lookback"])
params = {"start_date": start_ts, "end_date": end_ts}
compiled_statement = REDSHIFT_QUERY.params(params).compile(bind=conn)
for card_class in CardClass:
if 2 <= card_class <= 10:
for a in Archetype.objects.live().filter(
player_class=card_class):
self.archetype_map[a.id] = a
# Standard Signature Weights
standard_weight_values = ClusterSnapshot.objects.get_signature_weights(
FormatType.FT_STANDARD, card_class)
if len(standard_weight_values):
self.signature_weights[FormatType.FT_STANDARD][
card_class] = standard_weight_values
# Wild Signature Weights
wild_weight_values = ClusterSnapshot.objects.get_signature_weights(
FormatType.FT_WILD, card_class)
if len(wild_weight_values):
self.signature_weights[FormatType.FormatType.FT_WILD][
card_class] = wild_weight_values
result_set = list(conn.execute(compiled_statement))
total_rows = len(result_set)
self.stdout.write("%i decks to update" % (total_rows))
if is_dry_run:
self.stdout.write("Dry run, will not flush to databases")
for counter, row in enumerate(result_set):
deck_id = row["deck_id"]
if not is_dry_run and counter % 100000 == 0:
self.flush_db_buffer()
self.flush_firehose_buffer()
if deck_id is None:
self.stderr.write("Got deck_id %r ... skipping" % (deck_id))
continue
current_archetype_id = row["archetype_id"]
player_class = CardClass(row["player_class"])
if player_class == CardClass.NEUTRAL:
# Most likely noise
self.stderr.write("Found and skipping NEUTRAL data: %r" %
(row))
continue
format = FormatType.FT_STANDARD if row[
"game_type"] == 2 else FormatType.FT_WILD
dbf_map = {
dbf_id: count
for dbf_id, count in json.loads(row["deck_list"])
}
if player_class not in self.signature_weights[format]:
raise RuntimeError(
"%r not found for %r. Are signatures present?" %
(player_class, format))
if self.signature_weights[format][player_class]:
new_archetype_id = classify_deck(
dbf_map, self.signature_weights[format][player_class])
if new_archetype_id == current_archetype_id:
if verbosity > 1:
self.stdout.write("Deck %r - Nothing to do." %
(deck_id))
continue
current_name = self.get_archetype_name(current_archetype_id)
new_name = self.get_archetype_name(new_archetype_id)
pct_complete = str(math.floor(100.0 * counter / total_rows))
self.stdout.write(
"\t[%s%%] Reclassifying deck %r: %s => %s\n" %
(pct_complete, deck_id, current_name, new_name))
if not is_dry_run:
self.buffer_archetype_update(deck_id, new_archetype_id)
if not is_dry_run:
self.flush_db_buffer()
self.flush_firehose_buffer()
else:
self.stdout.write("Dry run complete")
def update_global_players(global_game, entity_tree, meta, upload_event,
exporter):
# Fill the player metadata and objects
players = {}
played_cards = exporter.export_played_cards()
is_spectated_replay = meta.get("spectator_mode", False)
is_dungeon_run = meta.get("scenario_id", 0) == 2663
for player in entity_tree.players:
is_friendly_player = player.player_id == meta["friendly_player"]
player_meta = meta.get("player%i" % (player.player_id), {})
decklist_from_meta = player_meta.get("deck")
decklist_from_replay = [
c.initial_card_id for c in player.initial_deck if c.card_id
]
meta_decklist_is_superset = _is_decklist_superset(
decklist_from_meta, decklist_from_replay)
# We disregard the meta decklist if it's not matching the replay decklist
# We always want to use it in dungeon run though, since the initial deck is garbage
disregard_meta = not meta_decklist_is_superset and (
not is_dungeon_run or not is_friendly_player)
if not decklist_from_meta or is_spectated_replay or disregard_meta:
# Spectated replays never know more than is in the replay data
# But may have erroneous data from the spectator's client's memory
# Read from before they entered the spectated game
decklist = decklist_from_replay
else:
decklist = decklist_from_meta
name, real_name = get_player_names(player)
player_hero_id = player._hero.card_id
try:
deck, _ = Deck.objects.get_or_create_from_id_list(
decklist,
hero_id=player_hero_id,
game_type=global_game.game_type,
classify_archetype=True)
log.debug("Prepared deck %i (created=%r)", deck.id, _)
except IntegrityError as e:
# This will happen if cards in the deck are not in the DB
# For example, during a patch release
influx_metric(
"replay_deck_create_failure", {
"count": 1,
"build": meta["build"],
"global_game_id": global_game.id,
"server_ip": meta.get("server_ip", ""),
"upload_ip": upload_event.upload_ip,
"error": str(e),
})
log.exception("Could not create deck for player %r", player)
global_game.tainted_decks = True
# Replace with an empty deck
deck, _ = Deck.objects.get_or_create_from_id_list([])
capture_played_card_stats(
global_game, [c.dbf_id for c in played_cards[player.player_id]],
is_friendly_player)
eligible_formats = [FormatType.FT_STANDARD, FormatType.FT_WILD]
is_eligible_format = global_game.format in eligible_formats
deck_prediction_enabled = getattr(settings,
"FULL_DECK_PREDICTION_ENABLED", True)
if deck_prediction_enabled and is_eligible_format and settings.ENV_AWS:
try:
player_class = Deck.objects._convert_hero_id_to_player_class(
player_hero_id)
tree = deck_prediction_tree(player_class, global_game.format)
played_cards_for_player = played_cards[player.player_id]
# 5 played cards partitions a 14 day window into buckets of ~ 500 or less
# We can search through ~ 2,000 decks in 100ms so that gives us plenty of headroom
min_played_cards = tree.max_depth - 1
# We can control via settings the minumum number of cards we need
# To know about in the deck list before we attempt to guess the full deck
min_observed_cards = settings.DECK_PREDICTION_MINIMUM_CARDS
played_card_dbfs = [c.dbf_id for c in played_cards_for_player
][:min_played_cards]
played_card_names = [c.name for c in played_cards_for_player
][:min_played_cards]
if deck.size is not None:
deck_size = deck.size
else:
deck_size = sum(i.count for i in deck.includes.all())
has_enough_observed_cards = deck_size >= min_observed_cards
has_enough_played_cards = len(
played_card_dbfs) >= min_played_cards
if deck_size == 30:
tree.observe(deck.id, deck.dbf_map(), played_card_dbfs)
# deck_id == proxy_deck_id for complete decks
deck.guessed_full_deck = deck
deck.save()
elif has_enough_observed_cards and has_enough_played_cards:
res = tree.lookup(
deck.dbf_map(),
played_card_dbfs,
)
predicted_deck_id = res.predicted_deck_id
fields = {
"actual_deck_id":
deck.id,
"deck_size":
deck_size,
"game_id":
global_game.id,
"sequence":
"->".join("[%s]" % c for c in played_card_names),
"predicted_deck_id":
res.predicted_deck_id,
"match_attempts":
res.match_attempts,
"tie":
res.tie
}
if settings.DETAILED_PREDICTION_METRICS:
fields["actual_deck"] = repr(deck)
if res.predicted_deck_id:
predicted_deck = Deck.objects.get(
id=res.predicted_deck_id)
fields["predicted_deck"] = repr(predicted_deck)
if res.node:
fields["depth"] = res.node.depth
if settings.DETAILED_PREDICTION_METRICS:
node_labels = []
for path_dbf_id in res.path():
if path_dbf_id == "ROOT":
path_str = path_dbf_id
else:
path_card = Card.objects.get(
dbf_id=path_dbf_id)
path_str = path_card.name
node_labels.append("[%s]" % path_str)
fields["node"] = "->".join(node_labels)
popularity = res.popularity_distribution.popularity(
res.predicted_deck_id)
fields["predicted_deck_popularity"] = popularity
deck_count = res.popularity_distribution.size()
fields["distribution_deck_count"] = deck_count
observation_count = res.popularity_distribution.observations(
)
fields[
"distribution_observation_count"] = observation_count
tree_depth = res.node.depth if res.node else None
influx_metric(
"deck_prediction",
fields,
missing_cards=30 - deck_size,
player_class=CardClass(int(player_class)).name,
format=FormatType(int(global_game.format)).name,
tree_depth=tree_depth,
made_prediction=predicted_deck_id is not None)
if predicted_deck_id:
deck.guessed_full_deck = Deck.objects.get(
id=predicted_deck_id)
deck.save()
except Exception as e:
error_handler(e)
# Create the BlizzardAccount first
defaults = {
"region": BnetRegion.from_account_hi(player.account_hi),
"battletag": name,
}
if not is_spectated_replay and not player.is_ai and is_friendly_player:
user = upload_event.token.user if upload_event.token else None
if user and not user.is_fake:
# and user.battletag and user.battletag.startswith(player.name):
defaults["user"] = user
blizzard_account, created = BlizzardAccount.objects.get_or_create(
account_hi=player.account_hi,
account_lo=player.account_lo,
defaults=defaults)
if not created and not blizzard_account.user and "user" in defaults:
# Set BlizzardAccount.user if it's an available claim for the user
influx_metric(
"pegasus_account_claimed", {
"count": 1,
"account": str(blizzard_account.id),
"region": str(blizzard_account.region),
"account_lo": str(blizzard_account.account_lo),
"game": str(global_game.id)
})
blizzard_account.user = defaults["user"]
blizzard_account.save()
log.debug("Prepared BlizzardAccount %r", blizzard_account)
# Now create the GlobalGamePlayer object
common = {
"game": global_game,
"player_id": player.player_id,
}
defaults = {
"is_first": player.tags.get(GameTag.FIRST_PLAYER, False),
"is_ai": player.is_ai,
"hero_id": player_hero_id,
"hero_premium": player._hero.tags.get(GameTag.PREMIUM, False),
"final_state": player.tags.get(GameTag.PLAYSTATE, 0),
"extra_turns": player.tags.get(GameTag.EXTRA_TURNS_TAKEN_THIS_GAME,
0),
"deck_list": deck,
}
update = {
"name": name,
"real_name": real_name,
"pegasus_account": blizzard_account,
"rank": player_meta.get("rank"),
"legend_rank": player_meta.get("legend_rank"),
"stars": player_meta.get("stars"),
"wins": player_meta.get("wins"),
"losses": player_meta.get("losses"),
"deck_id": player_meta.get("deck_id") or None,
"cardback_id": player_meta.get("cardback"),
}
defaults.update(update)
game_player, created = GlobalGamePlayer.objects.get_or_create(
defaults=defaults, **common)
log.debug("Prepared player %r (%i) (created=%r)", game_player,
game_player.id, created)
if not created:
# Go through the update dict and update values on the player
# This gets us extra data we might not have had when the player was first created
updated = False
for k, v in update.items():
if v and getattr(game_player, k) != v:
setattr(game_player, k, v)
updated = True
# Skip updating the deck if we already have a bigger one
# TODO: We should make deck_list nullable and only create it here
if game_player.deck_list.size is None or len(
decklist) > game_player.deck_list.size:
# XXX: Maybe we should also check friendly_player_id for good measure
game_player.deck_list = deck
updated = True
if updated:
log.debug("Saving updated player to the database.")
game_player.save()
players[player.player_id] = game_player
return players
def getInGameClass(self): return CardClass(self.inGameClass).name
