def calculate_goals(games,
goals_col,
goals_error_col,
year_month_day,
goals_to_check=None):
""" Analyze games for goals and insert those found into the MongoDB.
games: List of games to analyze, each in dict format
goals_col: Destination MongoDB collection
goals_error_col: MongoDB collection for goals analysis errors (for
potential reflow later). This is a TODO for now.
year_month_day: string in yyyymmdd format encoding date
goals_to_check: List of goals to analyze for. If passed, only the
listed goals will be calculated or re-calculated. Otherwise, all
goals are calculated.
"""
log.debug('Beginning to analyze %d games for goals, from %s', len(games),
year_month_day)
total_checked = 0
checker_output = collections.defaultdict(int)
for g in games:
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = goals_col.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict(dict)
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
goals_col.save(mongo_val)
print_totals(checker_output, total_checked)
return total_checked
def main(args):
db = utils.get_mongo_database()
games = db.games
game_stats = db.game_stats
for player_name in args.players:
log.debug("Processing top level player name %s", player_name)
norm_target_player = norm_name(player_name)
games_coll = games.find({keys.PLAYERS: norm_target_player})
calculate_game_stats(list(games_coll), game_stats)
def main(args):
db = utils.get_mongo_database()
games = db.games
game_stats = db.game_stats
for player_name in args.players:
log.debug("Processing top level player name %s", player_name)
norm_target_player = norm_name(player_name)
games_coll = games.find({keys.PLAYERS: norm_target_player})
calculate_game_stats(list(games_coll), game_stats)
def calculate_goals(games, goals_col, goals_error_col, year_month_day, goals_to_check=None):
""" Analyze games for goals and insert those found into the MongoDB.
games: List of games to analyze, each in dict format
goals_col: Destination MongoDB collection
goals_error_col: MongoDB collection for goals analysis errors (for
potential reflow later). This is a TODO for now.
year_month_day: string in yyyymmdd format encoding date
goals_to_check: List of goals to analyze for. If passed, only the
listed goals will be calculated or re-calculated. Otherwise, all
goals are calculated.
"""
log.debug('Beginning to analyze %d games for goals, from %s', len(games), year_month_day)
total_checked = 0
checker_output = collections.defaultdict(int)
for g in games:
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = goals_col.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict( dict )
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
goals_col.save(mongo_val)
print_totals(checker_output, total_checked)
return total_checked
def main(args):
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
collection = database.optimal_card_ratios
db_tracker = None
scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios',
database)
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
log.info("Starting run: %s", scanner.status_msg())
for ind, game in enumerate(utils.progress_meter(scanner.scan(games, {}))):
if not db_tracker:
log.debug("Initializing db tracker manager")
db_tracker = DBCardRatioTrackerManager(collection,
args.incremental)
log.debug("DB tracker manager initialized")
result = process_game(Game(game))
for final_ratio_dict, progressive_ratio_dict, win_points in result:
db_tracker.integrate_results('final', final_ratio_dict, win_points)
db_tracker.integrate_results('progressive', progressive_ratio_dict,
win_points)
if args.max_games >= 0 and ind >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if ind % commit_after == 0 and ind > 0:
start = time.time()
db_tracker.save()
scanner.save()
log.info("Committed calculations to the DB in %5.2fs",
time.time() - start)
log.info("Ending run: %s", scanner.status_msg())
if db_tracker:
db_tracker.save()
scanner.save()
def main(args):
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
collection = database.optimal_card_ratios
db_tracker = None
scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios', database)
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
log.info("Starting run: %s", scanner.status_msg())
for ind, game in enumerate(
utils.progress_meter(scanner.scan(games, {}))):
if not db_tracker:
log.debug("Initializing db tracker manager")
db_tracker = DBCardRatioTrackerManager(collection, args.incremental)
log.debug("DB tracker manager initialized")
result = process_game(Game(game))
for final_ratio_dict, progressive_ratio_dict, win_points in result:
db_tracker.integrate_results('final', final_ratio_dict, win_points)
db_tracker.integrate_results('progressive', progressive_ratio_dict, win_points)
if args.max_games >= 0 and ind >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if ind % commit_after == 0 and ind > 0:
start = time.time()
db_tracker.save()
scanner.save()
log.info("Committed calculations to the DB in %5.2fs", time.time() - start)
log.info("Ending run: %s", scanner.status_msg())
if db_tracker:
db_tracker.save()
scanner.save()
def calculate_game_stats(games, game_stats):
""" Save game statistics for each game and player
games: List of games to analyze, each in dict format
game_stats: Destination MongoDB collection
"""
log.debug("Beginning to analyze game statistics for %d games", len(games))
game_stats.ensure_index(keys.NAME)
added = 0
for game_dict in games:
game_val = game.Game(game_dict)
g_id = game_dict['_id']
date = game_dict['game_date']
supply = game_dict[keys.SUPPLY]
for p in get_game_stat_entries(game_val, g_id, date, supply):
game_stats.save(p, w=1)
added += 1
log.debug("Done analyzing game statistics, inserted %d", added)
return added
def calculate_game_stats(games, game_stats):
""" Save game statistics for each game and player
games: List of games to analyze, each in dict format
game_stats: Destination MongoDB collection
"""
log.debug("Beginning to analyze game statistics for %d games", len(games))
game_stats.ensure_index(keys.NAME)
added = 0
for game_dict in games:
game_val = game.Game(game_dict)
g_id = game_dict['_id']
date = game_dict['game_date']
supply = game_dict[keys.SUPPLY]
for p in get_game_stat_entries(game_val, g_id, date, supply):
game_stats.save(p, w=1)
added += 1
log.debug("Done analyzing game statistics, inserted %d", added)
return added
def main(args):
db = utils.get_mongo_database()
scanner = incremental_scanner.IncrementalScanner('analyze2', db)
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
for collection_name, _ in event_detectors:
db[collection_name].drop()
log.info("Starting run: %s", scanner.status_msg())
games_stream = analysis_util.games_stream(scanner, db.games)
accumulator = EventAccumulator()
accumulate_card_stats(games_stream, accumulator, args.max_games)
log.info('saving to database')
log.debug('saving accumulated stats')
accumulator.update_db(db)
log.info('saving the game scanner state')
scanner.save()
log.info("Ending run: %s", scanner.status_msg())
def main(args):
db = utils.get_mongo_database()
scanner = incremental_scanner.IncrementalScanner('analyze2', db)
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
for collection_name, _ in event_detectors:
db[collection_name].drop()
log.info("Starting run: %s", scanner.status_msg())
games_stream = analysis_util.games_stream(scanner, db.games)
accumulator = EventAccumulator()
accumulate_card_stats(games_stream, accumulator, args.max_games)
log.info('saving to database')
log.debug('saving accumulated stats')
accumulator.update_db(db)
log.info('saving the game scanner state')
scanner.save()
log.info("Ending run: %s", scanner.status_msg())
def update_db(self, mongo_db_inst):
for event_type_name, stats_dict in self.event_stats.iteritems():
log.debug('Updating database for event type %s, %d stats',
event_type_name, len(stats_dict))
mongo_collection = mongo_db_inst[event_type_name]
inserts = 0
updates = 0
for full_key, gain_stats_obj in sorted(stats_dict.iteritems()):
existing_raw_obj = mongo_collection.find_one({'_id': full_key})
if existing_raw_obj:
updates += 1
existing_stat = SmallGainStat()
existing_stat.from_primitive_object(
existing_raw_obj['vals'])
gain_stats_obj.merge(existing_stat)
else:
inserts += 1
key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()}
utils.write_object_to_db(key_wrap_obj, mongo_collection,
full_key)
log.debug('Database update results for %s: %d inserts, %d updates',
event_type_name, inserts, updates)
def update_db(self, mongo_db_inst):
for event_type_name, stats_dict in self.event_stats.iteritems():
log.debug('Updating database for event type %s, %d stats',
event_type_name, len(stats_dict))
mongo_collection = mongo_db_inst[event_type_name]
inserts = 0
updates = 0
for full_key, gain_stats_obj in sorted(stats_dict.iteritems()):
existing_raw_obj = mongo_collection.find_one(
{'_id': full_key})
if existing_raw_obj:
updates += 1
existing_stat = SmallGainStat()
existing_stat.from_primitive_object(
existing_raw_obj['vals'])
gain_stats_obj.merge(existing_stat)
else:
inserts += 1
key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()}
utils.write_object_to_db(key_wrap_obj, mongo_collection,
full_key)
log.debug('Database update results for %s: %d inserts, %d updates',
event_type_name, inserts, updates)
