def retrieveLot(lotID):
lot = lotID.get()
container = LOTContainer()
container.lot = lot
container.games = list(Game.query(Game.lotID == lotID.id()))
#Load all players that are referenced by the LOT. First, get their player IDs and remove dupes
pids = set(flatten([g.players for g in container.games])) | set(lot.playersParticipating) | set(lot.playerRanks)
#logging.info("Loading lot " + str(lotID) + " got " + str(len(pids)) + " pids")
#Turn the player IDs into Player instances
players = list(ndb.get_multi([ndb.Key(Player, p) for p in pids]))
#Store them as a dictionary so we can easily look them up by id later
container.players = dict([(p.key.id(), p) for p in players if p is not None])
return container
def test_list_input(self):
"""
Test to check if a json file with arrays is rejected
"""
with open('files/input_list.json', 'r') as f:
json_data = json.load(f)
with self.assertRaises(Exception):
flatten(json_data)
def test_simple_input(self):
"""
Test to check results with simple one-level hierarchical json
"""
with open('files/input.json', 'r') as f:
json_data = json.load(f)
result = flatten(json_data)
expected = {'a': 1, 'b': True, 'c.d': 3, 'c.e': 'test'}
self.assertEqual(result, expected)
def createGames(request, container):
"""This is called periodically to check for new games that need to be created.
You should replace this with your own logic for how games are to be created.
Right now, this function just randomly pairs up players who aren't in a game."""
# Read configuration settings for ladder
readConfigForMDLadder()
#Recent games. All players who have played each other recently, will not be paired together.
recentGames = []
#Active games also count as recent.
activeGames = []
for g in container.games:
if g.dateEnded is None:
recentGames.append(g)
activeGames.append(g)
else:
delta = (datetime.now() - g.dateEnded)
timeElapsed = delta.total_seconds()
if int(timeElapsed) < timeBetweenGamesInHours * 60 * 60 :
recentGames.append(g)
#Retrieve player pairs from all games that are ongoing
activeGameIDs = dict([[g.key.id(), g] for g in activeGames])
logging.info("Active games: " + unicode(activeGameIDs))
#Throw all of the player IDs that are in these ongoing games into a list
playerIDsInActiveGames = flatten([g.players for g in activeGames])
# Map of {player:gameCount}
activeGameCountForPlayer = {}
for p in playerIDsInActiveGames:
activeGameCountForPlayer[p] = activeGameCountForPlayer.get(p, 0) + 1
# Map of {player:NumOfGamesToBeAllotted}
playersToBeAllocatedNewGames = {}
for p in container.lot.playersParticipating:
player = container.players[p]
newGames = player.numberOfGamesAtOnce - activeGameCountForPlayer.get(p, 0)
if newGames < 0 :
newGames = 0
playersToBeAllocatedNewGames[p] = newGames
logging.info("Games to be allotted: " + str(playersToBeAllocatedNewGames))
#Create a game for everyone not in a game.
#From a list of templates, a random one is picked for each game
gamesCreated = []
for pair in createPlayerPairs(container.lot.playerRanks, playersToBeAllocatedNewGames, recentGames):
chosenTemplateId = int(choice(templates))
overriddenBonuses = getOverriddenBonuses(chosenTemplateId)
players = [container.players[p] for p in pair]
g = createGame(request, container, players, chosenTemplateId, overriddenBonuses)
gamesCreated.append(g)
logging.info("Created games " + unicode(','.join([unicode(g) for g in gamesCreated])))
def createGames(request, container):
"""This is called periodically to check for new games that need to be created.
You should replace this with your own logic for how games are to be created.
Right now, this function just randomly pairs up players who aren't in a game."""
# Read configuration settings for ladder
readConfigForMDLadder()
# Recent games. All players who have played each other recently, will not be paired together.
recentGames = []
# Active games also count as recent.
activeGames = []
for g in container.games:
if g.dateEnded is None:
recentGames.append(g)
activeGames.append(g)
else:
delta = datetime.now() - g.dateEnded
timeElapsed = delta.total_seconds()
if int(timeElapsed) < timeBetweenGamesInHours * 60 * 60:
recentGames.append(g)
# Retrieve player pairs from all games that are ongoing
activeGameIDs = dict([[g.key.id(), g] for g in activeGames])
logging.info("Active games: " + unicode(activeGameIDs))
# Throw all of the player IDs that are in these ongoing games into a list
playerIDsInActiveGames = flatten([g.players for g in activeGames])
# Map of {player:gameCount}
activeGameCountForPlayer = {}
for p in playerIDsInActiveGames:
activeGameCountForPlayer[p] = activeGameCountForPlayer.get(p, 0) + 1
# Map of {player:NumOfGamesToBeAllotted}
playersToBeAllocatedNewGames = {}
for p in container.lot.playersParticipating:
player = container.players[p]
newGames = player.numberOfGamesAtOnce - activeGameCountForPlayer.get(p, 0)
if newGames < 0:
newGames = 0
playersToBeAllocatedNewGames[p] = newGames
logging.info("Games to be allotted: " + str(playersToBeAllocatedNewGames))
# Create a game for everyone not in a game.
# From a list of templates, a random one is picked for each game
gamesCreated = []
for pair in createPlayerPairs(container.lot.playerRanks, playersToBeAllocatedNewGames, recentGames):
chosenTemplateId = int(choice(templates))
overriddenBonuses = getOverriddenBonuses(chosenTemplateId)
players = [container.players[p] for p in pair]
g = createGame(request, container, players, chosenTemplateId, overriddenBonuses)
gamesCreated.append(g)
logging.info("Created games " + unicode(",".join([unicode(g) for g in gamesCreated])))
def test(self):
self.assertEqual(main.flatten([]), [])
self.assertEqual(main.flatten([1, 2, 3, 4, 5]), [1, 2, 3, 4, 5])
self.assertEqual(main.flatten([[1, 2], [3, 4], [5]]), [1, 2, 3, 4, 5])
self.assertEqual(main.flatten([[1], [2], [3], [4], [5]]),
[1, 2, 3, 4, 5])
self.assertEqual(main.flatten([[1, 2, 3, 4, 5]]), [1, 2, 3, 4, 5])
self.assertEqual(main.flatten([[[1, 2, 3, 4, 5]]]), [1, 2, 3, 4, 5])
self.assertEqual(main.flatten([1, [2, [3, 4], 5]]), [1, 2, 3, 4, 5])
def generate_results(results_jayson):
flattened_j_ts = []
for metric in results_jayson['metrics']:
if 'results' in metric.keys():
flattened_results = []
for ts in metric['results']:
flattened_timeseries = []
ts['upload_ts'] = str(datetime.utcnow())
flattened_timeseries.append(flatten(ts, {}, ''))
# Replace old 'metrics' with new 'flattened_results'
update_metrics = populating_vals(outer_dict=metric, inner_flattened_list=flattened_timeseries, destination_key='results')
flattened_results.extend(flatten_dupe_vals(vals=update_metrics, key='results'))
else:
flattened_results = [flatten(results_jayson, {}, '')]
update_new_j_ts = populating_vals(outer_dict=results_jayson, inner_flattened_list=flattened_results, destination_key='metrics')
flattened_j_ts.extend(flatten_dupe_vals(vals=update_new_j_ts, key='metrics'))
return flattened_j_ts
def test_advanced_input(self):
"""
Test to check results with advanced two-level hierarchical json
"""
with open('files/input_2.json', 'r') as f:
json_data = json.load(f)
result = flatten(json_data)
expected = {
'a': 1,
'b': True,
'c.d': 3,
'c.e': 'test',
'f.g.h': 'deep test',
'f.g.i': 10
}
self.assertEqual(result, expected)
def generate_experiments(exp_list):
###### Game plan is to separate nested fields from single layer fields, upload them to separate table, then do joins on BQ level
all_singles = []
metrics_table = []
variations_table = []
for exp in exp_list:
print(f"Processing experiment {exp['id']}")
# single layer fields:
nested_key_list = []
for k,v in exp.items():
if isinstance(v, list):
nested_key_list.append(k)
single_layer_experiment = {}
for k,v in exp.items():
if k not in nested_key_list:
k = k.replace('-', '_')
single_layer_experiment[k] = exp[k]
single_layer_experiment['upload_ts'] = str(datetime.utcnow())
all_singles.append(flatten(single_layer_experiment, {}, ''))
# nested part into separate tables:
# metrics table:
flattened_metric = []
for element in exp['metrics']:
flattened_metric.append(element)
updated_metric = populating_vals(outer_dict=exp, inner_flattened_list=flattened_metric, destination_key='metrics')
new_flattened_metric = flatten_dupe_vals(vals=updated_metric, key='metrics')
metric_list = []
for metric in new_flattened_metric:
metric_dict = {}
metric_dict['metrics_aggregator'] = metric['metrics_aggregator']
if 'metrics_event_id' in metric.keys():
metric_dict['metrics_event_id'] = metric['metrics_event_id']
metric_dict['metrics_scope'] = metric['metrics_scope']
metric_dict['metrics_winning_direction'] = metric['metrics_winning_direction']
metric_dict['experiment_id'] = exp['id']
metric_dict['upload_ts'] = str(datetime.utcnow())
metric_list.append(metric_dict)
metrics_table.extend(metric_list)
# variations table:
variations = {}
variations['experiment_id'] = exp['id']
variations['variations'] = exp['variations']
variations['upload_ts'] = str(datetime.utcnow())
flattened_variations = []
for var in exp['variations']:
flattened_actions = []
if len(var['actions']) > 0:
for action in var['actions']:
flattened_changes = []
if action['changes'] != []:
for element in action['changes']:
flattened_changes.append(element)
# Replace old 'changes' with new 'flattened_changes'
updated_changes = populating_vals(outer_dict=action, inner_flattened_list=flattened_changes, destination_key='changes')
new_flattened_changes = flatten_dupe_vals(vals=updated_changes, key='changes')
update_actions = populating_vals(outer_dict=var, inner_flattened_list=new_flattened_changes, destination_key='actions')
flat = flatten_dupe_vals(vals=update_actions, key='actions')
flattened_actions.extend(flat)
else:
new_actions = flatten_dupe_vals(vals=var, key='actions')
flattened_actions.extend(new_actions)
else:
other_flat = {}
for k,v in var.items():
if k != 'actions':
other_flat['actions'] = []
other_flat[k] = v
flat = [other_flat]
flattened_actions.extend(flat)
update_variations = populating_vals(outer_dict=variations, inner_flattened_list=flattened_actions, destination_key='variations')
flattened_variations.extend(flatten_dupe_vals(vals=update_variations, key='variations'))
variations_table.extend(flattened_variations)
return all_singles, metrics_table, variations_table
def test_flatten_ok():
result = list(main.flatten([0, [1, 2, [2, 3], [2, 3]], 5]))
assert result == [0, 1, 2, 2, 3, 2, 3, 5]