def main(parsed_args): """ Scan and update buy data""" start = time.time() db = utils.get_mongo_database() games = db.games output_db = db overall_stats = DeckBuyStats() scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db) buy_collection = output_db[BUYS_COL_NAME] if not parsed_args.incremental: log.warning('resetting scanner and db') scanner.reset() buy_collection.drop() start_size = scanner.get_num_games() log.info("Starting run: %s", scanner.status_msg()) do_scan(scanner, games, overall_stats, parsed_args.max_games) log.info("Ending run: %s", scanner.status_msg()) end_size = scanner.get_num_games() if parsed_args.incremental: existing_overall_data = DeckBuyStats() utils.read_object_from_db(existing_overall_data, buy_collection, '') overall_stats.merge(existing_overall_data) def deck_freq(data_set): return data_set[dominioncards.Estate].available.frequency() log.info('existing %s decks', deck_freq(existing_overall_data)) log.info('after merge %s decks', deck_freq(overall_stats)) utils.write_object_to_db(overall_stats, buy_collection, '') scanner.save()
def main(args): """ Update analysis statistics. By default, do so incrementally, unless --noincremental argument is given.""" commit_after = 25000 database = utils.get_mongo_database() games = database.games output_collection_name = 'analysis' output_collection = database[output_collection_name] game_analysis = GamesAnalysis() scanner = incremental_scanner.IncrementalScanner(output_collection_name, database) if args.incremental: utils.read_object_from_db(game_analysis, output_collection, '') else: log.warning('resetting scanner and db') scanner.reset() output_file_name = 'static/output/all_games_card_stats.js' if not os.path.exists('static/output'): os.makedirs('static/output') log.info("Starting run: %s", scanner.status_msg()) for idx, raw_game in enumerate( utils.progress_meter(scanner.scan(games, {}))): try: game_analysis.analyze_game(Game(raw_game)) if args.max_games >= 0 and idx >= args.max_games: log.info("Reached max_games of %d", args.max_games) break if idx % commit_after == 0 and idx > 0: start = time.time() game_analysis.max_game_id = scanner.get_max_game_id() game_analysis.num_games = scanner.get_num_games() utils.write_object_to_db(game_analysis, output_collection, '') scanner.save() log.info("Committed calculations to the DB in %5.2fs", time.time() - start) except int, exception: log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game) raise
def main(args): """ Update analysis statistics. By default, do so incrementally, unless --noincremental argument is given.""" commit_after = 25000 database = utils.get_mongo_database() games = database.games output_collection_name = 'analysis' output_collection = database[output_collection_name] game_analysis = GamesAnalysis() scanner = incremental_scanner.IncrementalScanner(output_collection_name, database) if args.incremental: utils.read_object_from_db(game_analysis, output_collection, '') else: log.warning('resetting scanner and db') scanner.reset() output_file_name = 'static/output/all_games_card_stats.js' if not os.path.exists('static/output'): os.makedirs('static/output') log.info("Starting run: %s", scanner.status_msg()) for idx, raw_game in enumerate(utils.progress_meter(scanner.scan(games, {}))): try: game_analysis.analyze_game(Game(raw_game)) if args.max_games >= 0 and idx >= args.max_games: log.info("Reached max_games of %d", args.max_games) break if idx % commit_after == 0 and idx > 0: start = time.time() game_analysis.max_game_id = scanner.get_max_game_id() game_analysis.num_games = scanner.get_num_games() utils.write_object_to_db(game_analysis, output_collection, '') scanner.save() log.info("Committed calculations to the DB in %5.2fs", time.time() - start) except int, exception: log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game) raise
def main(): """ Scan and update buy data""" start = time.time() con = pymongo.Connection() games = con.test.games output_db = con.test parser = utils.incremental_parser() parser.add_argument('--max_games', default=-1, type=int) args = parser.parse_args() overall_stats = DeckBuyStats() scanner = incremental_scanner.IncrementalScanner('buys', output_db) buy_collection = output_db['buys'] if not args.incremental: print 'resetting scanner and db' scanner.Reset() buy_collection.drop() start_size = scanner.NumGames() print scanner.StatusMsg() do_scan(scanner, games, overall_stats, args.max_games) print scanner.StatusMsg() end_size = scanner.NumGames() if args.incremental: existing_overall_data = DeckBuyStats() utils.read_object_from_db(existing_overall_data, buy_collection, '') overall_stats.Merge(existing_overall_data) def deck_freq(data_set): return data_set['Estate'].available.Frequency() print 'existing', deck_freq(existing_overall_data), 'decks' print 'after merge', deck_freq(overall_stats), 'decks' utils.write_object_to_db(overall_stats, buy_collection, '') scanner.Save() time_diff = time.time() - start games_diff = end_size - start_size print 'took', time_diff, 'seconds for', games_diff, 'games for a rate' \ ' of', games_diff / time_diff, 'games/sec'
def main(): """ Update analysis statistics. By default, do so incrementally, unless --noincremental argument is given.""" parser = utils.incremental_max_parser() parser.add_argument('--output_collection_name', default='analysis') args = parser.parse_args() conn = pymongo.Connection() database = conn.test games = database.games output_collection_name = args.output_collection_name output_collection = database[output_collection_name] game_analysis = GamesAnalysis() scanner = incremental_scanner.IncrementalScanner(output_collection_name, database) if args.incremental: utils.read_object_from_db(game_analysis, output_collection, '') else: scanner.Reset() output_file_name = 'static/output/all_games_card_stats.js' if not os.path.exists('static/output'): os.makedirs('static/output') print scanner.StatusMsg() for idx, raw_game in enumerate(scanner.Scan(games, {})): try: if idx % 1000 == 0: print idx game_analysis.analyze_game(Game(raw_game)) if idx == args.max_games: break except int, exception: print Game(raw_game).IsotropicUrl() print exception print raw_game raise
def GET(self): import count_buys web.header("Content-Type", "text/html; charset=utf-8") query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING'])) db = utils.get_mongo_database() stats = count_buys.DeckBuyStats() utils.read_object_from_db(stats, db.buys, '') player_buy_summary = None if 'player' in query_dict: targ_name = norm_name(query_dict['player']) games = map(game.Game, list(db.games.find({'players': targ_name}))) player_buy_summary = count_buys.DeckBuyStats() match_name = lambda g, name: norm_name(name) == targ_name count_buys.accum_buy_stats(games, player_buy_summary, match_name) count_buys.add_effectiveness(player_buy_summary, stats) render = web.template.render('', globals={'round': round}) return render.buy_template(stats, player_buy_summary)
def GET(self): import count_buys web.header("Content-Type", "text/html; charset=utf-8") query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING'])) db = utils.get_mongo_database() stats = count_buys.DeckBuyStats() utils.read_object_from_db(stats, db.buys, '') player_buy_summary = None if 'player' in query_dict: targ_name = NormName(query_dict['player']) games = map(game.Game, list(db.games.find({'players': targ_name}))) player_buy_summary = count_buys.DeckBuyStats() match_name = lambda g, name: NormName(name) == targ_name count_buys.accum_buy_stats(games, player_buy_summary, match_name) count_buys.add_effectiveness(player_buy_summary, stats) render = web.template.render('', globals={'round': round}) return render.buy_template(stats, player_buy_summary)