def calculate_goals(games, goals_col, goals_error_col, year_month_day, goals_to_check=None): """ Analyze games for goals and insert those found into the MongoDB. games: List of games to analyze, each in dict format goals_col: Destination MongoDB collection goals_error_col: MongoDB collection for goals analysis errors (for potential reflow later). This is a TODO for now. year_month_day: string in yyyymmdd format encoding date goals_to_check: List of goals to analyze for. If passed, only the listed goals will be calculated or re-calculated. Otherwise, all goals are calculated. """ log.debug('Beginning to analyze %d games for goals, from %s', len(games), year_month_day) total_checked = 0 checker_output = collections.defaultdict(int) for g in games: total_checked += 1 game_val = game.Game(g) # Get existing goal set (if exists) game_id = game_val.get_id() mongo_val = goals_col.find_one({'_id': game_id}) if mongo_val is None: mongo_val = collections.defaultdict(dict) mongo_val['_id'] = game_id mongo_val['goals'] = [] # If rechecking, delete old values if goals_to_check is not None: goals = mongo_val['goals'] for ind in range(len(goals) - 1, -1, -1): goal = goals[ind] if goal['goal_name'] in goals_to_check: del goals[ind] # Get new values goals = check_goals(game_val, goals_to_check) # Write new values for goal in goals: goal_name = goal['goal_name'] mongo_val['goals'].append(goal) checker_output[goal_name] += 1 mongo_val = dict(mongo_val) goals_col.save(mongo_val) print_totals(checker_output, total_checked) return total_checked
def main(args): db = utils.get_mongo_database() games = db.games game_stats = db.game_stats for player_name in args.players: log.debug("Processing top level player name %s", player_name) norm_target_player = norm_name(player_name) games_coll = games.find({keys.PLAYERS: norm_target_player}) calculate_game_stats(list(games_coll), game_stats)
def calculate_goals(games, goals_col, goals_error_col, year_month_day, goals_to_check=None): """ Analyze games for goals and insert those found into the MongoDB. games: List of games to analyze, each in dict format goals_col: Destination MongoDB collection goals_error_col: MongoDB collection for goals analysis errors (for potential reflow later). This is a TODO for now. year_month_day: string in yyyymmdd format encoding date goals_to_check: List of goals to analyze for. If passed, only the listed goals will be calculated or re-calculated. Otherwise, all goals are calculated. """ log.debug('Beginning to analyze %d games for goals, from %s', len(games), year_month_day) total_checked = 0 checker_output = collections.defaultdict(int) for g in games: total_checked += 1 game_val = game.Game(g) # Get existing goal set (if exists) game_id = game_val.get_id() mongo_val = goals_col.find_one({'_id': game_id}) if mongo_val is None: mongo_val = collections.defaultdict( dict ) mongo_val['_id'] = game_id mongo_val['goals'] = [] # If rechecking, delete old values if goals_to_check is not None: goals = mongo_val['goals'] for ind in range(len(goals) - 1, -1, -1): goal = goals[ind] if goal['goal_name'] in goals_to_check: del goals[ind] # Get new values goals = check_goals(game_val, goals_to_check) # Write new values for goal in goals: goal_name = goal['goal_name'] mongo_val['goals'].append(goal) checker_output[goal_name] += 1 mongo_val = dict(mongo_val) goals_col.save(mongo_val) print_totals(checker_output, total_checked) return total_checked
def main(args): commit_after = 25000 database = utils.get_mongo_database() games = database.games collection = database.optimal_card_ratios db_tracker = None scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios', database) if not args.incremental: log.warning('resetting scanner and db') scanner.reset() log.info("Starting run: %s", scanner.status_msg()) for ind, game in enumerate(utils.progress_meter(scanner.scan(games, {}))): if not db_tracker: log.debug("Initializing db tracker manager") db_tracker = DBCardRatioTrackerManager(collection, args.incremental) log.debug("DB tracker manager initialized") result = process_game(Game(game)) for final_ratio_dict, progressive_ratio_dict, win_points in result: db_tracker.integrate_results('final', final_ratio_dict, win_points) db_tracker.integrate_results('progressive', progressive_ratio_dict, win_points) if args.max_games >= 0 and ind >= args.max_games: log.info("Reached max_games of %d", args.max_games) break if ind % commit_after == 0 and ind > 0: start = time.time() db_tracker.save() scanner.save() log.info("Committed calculations to the DB in %5.2fs", time.time() - start) log.info("Ending run: %s", scanner.status_msg()) if db_tracker: db_tracker.save() scanner.save()
def main(args): commit_after = 25000 database = utils.get_mongo_database() games = database.games collection = database.optimal_card_ratios db_tracker = None scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios', database) if not args.incremental: log.warning('resetting scanner and db') scanner.reset() log.info("Starting run: %s", scanner.status_msg()) for ind, game in enumerate( utils.progress_meter(scanner.scan(games, {}))): if not db_tracker: log.debug("Initializing db tracker manager") db_tracker = DBCardRatioTrackerManager(collection, args.incremental) log.debug("DB tracker manager initialized") result = process_game(Game(game)) for final_ratio_dict, progressive_ratio_dict, win_points in result: db_tracker.integrate_results('final', final_ratio_dict, win_points) db_tracker.integrate_results('progressive', progressive_ratio_dict, win_points) if args.max_games >= 0 and ind >= args.max_games: log.info("Reached max_games of %d", args.max_games) break if ind % commit_after == 0 and ind > 0: start = time.time() db_tracker.save() scanner.save() log.info("Committed calculations to the DB in %5.2fs", time.time() - start) log.info("Ending run: %s", scanner.status_msg()) if db_tracker: db_tracker.save() scanner.save()
def calculate_game_stats(games, game_stats): """ Save game statistics for each game and player games: List of games to analyze, each in dict format game_stats: Destination MongoDB collection """ log.debug("Beginning to analyze game statistics for %d games", len(games)) game_stats.ensure_index(keys.NAME) added = 0 for game_dict in games: game_val = game.Game(game_dict) g_id = game_dict['_id'] date = game_dict['game_date'] supply = game_dict[keys.SUPPLY] for p in get_game_stat_entries(game_val, g_id, date, supply): game_stats.save(p, w=1) added += 1 log.debug("Done analyzing game statistics, inserted %d", added) return added
def main(args): db = utils.get_mongo_database() scanner = incremental_scanner.IncrementalScanner('analyze2', db) if not args.incremental: log.warning('resetting scanner and db') scanner.reset() for collection_name, _ in event_detectors: db[collection_name].drop() log.info("Starting run: %s", scanner.status_msg()) games_stream = analysis_util.games_stream(scanner, db.games) accumulator = EventAccumulator() accumulate_card_stats(games_stream, accumulator, args.max_games) log.info('saving to database') log.debug('saving accumulated stats') accumulator.update_db(db) log.info('saving the game scanner state') scanner.save() log.info("Ending run: %s", scanner.status_msg())
def update_db(self, mongo_db_inst): for event_type_name, stats_dict in self.event_stats.iteritems(): log.debug('Updating database for event type %s, %d stats', event_type_name, len(stats_dict)) mongo_collection = mongo_db_inst[event_type_name] inserts = 0 updates = 0 for full_key, gain_stats_obj in sorted(stats_dict.iteritems()): existing_raw_obj = mongo_collection.find_one({'_id': full_key}) if existing_raw_obj: updates += 1 existing_stat = SmallGainStat() existing_stat.from_primitive_object( existing_raw_obj['vals']) gain_stats_obj.merge(existing_stat) else: inserts += 1 key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()} utils.write_object_to_db(key_wrap_obj, mongo_collection, full_key) log.debug('Database update results for %s: %d inserts, %d updates', event_type_name, inserts, updates)
def update_db(self, mongo_db_inst): for event_type_name, stats_dict in self.event_stats.iteritems(): log.debug('Updating database for event type %s, %d stats', event_type_name, len(stats_dict)) mongo_collection = mongo_db_inst[event_type_name] inserts = 0 updates = 0 for full_key, gain_stats_obj in sorted(stats_dict.iteritems()): existing_raw_obj = mongo_collection.find_one( {'_id': full_key}) if existing_raw_obj: updates += 1 existing_stat = SmallGainStat() existing_stat.from_primitive_object( existing_raw_obj['vals']) gain_stats_obj.merge(existing_stat) else: inserts += 1 key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()} utils.write_object_to_db(key_wrap_obj, mongo_collection, full_key) log.debug('Database update results for %s: %d inserts, %d updates', event_type_name, inserts, updates)