def main(parsed_args): """ Scan and update buy data""" start = time.time() db = utils.get_mongo_database() games = db.games output_db = db overall_stats = DeckBuyStats() scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db) buy_collection = output_db[BUYS_COL_NAME] if not parsed_args.incremental: log.warning('resetting scanner and db') scanner.reset() buy_collection.drop() start_size = scanner.get_num_games() log.info("Starting run: %s", scanner.status_msg()) do_scan(scanner, games, overall_stats, parsed_args.max_games) log.info("Ending run: %s", scanner.status_msg()) end_size = scanner.get_num_games() if parsed_args.incremental: existing_overall_data = DeckBuyStats() utils.read_object_from_db(existing_overall_data, buy_collection, '') overall_stats.merge(existing_overall_data) def deck_freq(data_set): return data_set[dominioncards.Estate].available.frequency() log.info('existing %s decks', deck_freq(existing_overall_data)) log.info('after merge %s decks', deck_freq(overall_stats)) utils.write_object_to_db(overall_stats, buy_collection, '') scanner.save()
def update_db(self, mongo_db_inst): for event_type_name, stats_dict in self.event_stats.iteritems(): mongo_collection = mongo_db_inst[event_type_name] for full_key, gain_stats_obj in sorted(stats_dict.iteritems()): existing_raw_obj = mongo_collection.find_one({'_id': full_key}) if existing_raw_obj: existing_stat = SmallGainStat() existing_stat.from_primitive_object( existing_raw_obj['vals']) gain_stats_obj.merge(existing_stat) key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()} utils.write_object_to_db(key_wrap_obj, mongo_collection, full_key)
def main(args): """ Update analysis statistics. By default, do so incrementally, unless --noincremental argument is given.""" commit_after = 25000 database = utils.get_mongo_database() games = database.games output_collection_name = 'analysis' output_collection = database[output_collection_name] game_analysis = GamesAnalysis() scanner = incremental_scanner.IncrementalScanner(output_collection_name, database) if args.incremental: utils.read_object_from_db(game_analysis, output_collection, '') else: log.warning('resetting scanner and db') scanner.reset() output_file_name = 'static/output/all_games_card_stats.js' if not os.path.exists('static/output'): os.makedirs('static/output') log.info("Starting run: %s", scanner.status_msg()) for idx, raw_game in enumerate( utils.progress_meter(scanner.scan(games, {}))): try: game_analysis.analyze_game(Game(raw_game)) if args.max_games >= 0 and idx >= args.max_games: log.info("Reached max_games of %d", args.max_games) break if idx % commit_after == 0 and idx > 0: start = time.time() game_analysis.max_game_id = scanner.get_max_game_id() game_analysis.num_games = scanner.get_num_games() utils.write_object_to_db(game_analysis, output_collection, '') scanner.save() log.info("Committed calculations to the DB in %5.2fs", time.time() - start) except int, exception: log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game) raise
def update_db(self, mongo_db_inst): for event_type_name, stats_dict in self.event_stats.iteritems(): mongo_collection = mongo_db_inst[event_type_name] for full_key, gain_stats_obj in sorted(stats_dict.iteritems()): existing_raw_obj = mongo_collection.find_one( {'_id': full_key}) if existing_raw_obj: existing_stat = SmallGainStat() existing_stat.from_primitive_object( existing_raw_obj['vals']) gain_stats_obj.merge(existing_stat) key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()} utils.write_object_to_db(key_wrap_obj, mongo_collection, full_key)
def main(args): """ Update analysis statistics. By default, do so incrementally, unless --noincremental argument is given.""" commit_after = 25000 database = utils.get_mongo_database() games = database.games output_collection_name = 'analysis' output_collection = database[output_collection_name] game_analysis = GamesAnalysis() scanner = incremental_scanner.IncrementalScanner(output_collection_name, database) if args.incremental: utils.read_object_from_db(game_analysis, output_collection, '') else: log.warning('resetting scanner and db') scanner.reset() output_file_name = 'static/output/all_games_card_stats.js' if not os.path.exists('static/output'): os.makedirs('static/output') log.info("Starting run: %s", scanner.status_msg()) for idx, raw_game in enumerate(utils.progress_meter(scanner.scan(games, {}))): try: game_analysis.analyze_game(Game(raw_game)) if args.max_games >= 0 and idx >= args.max_games: log.info("Reached max_games of %d", args.max_games) break if idx % commit_after == 0 and idx > 0: start = time.time() game_analysis.max_game_id = scanner.get_max_game_id() game_analysis.num_games = scanner.get_num_games() utils.write_object_to_db(game_analysis, output_collection, '') scanner.save() log.info("Committed calculations to the DB in %5.2fs", time.time() - start) except int, exception: log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game) raise
def main(): """ Scan and update buy data""" start = time.time() con = pymongo.Connection() games = con.test.games output_db = con.test parser = utils.incremental_parser() parser.add_argument('--max_games', default=-1, type=int) args = parser.parse_args() overall_stats = DeckBuyStats() scanner = incremental_scanner.IncrementalScanner('buys', output_db) buy_collection = output_db['buys'] if not args.incremental: print 'resetting scanner and db' scanner.Reset() buy_collection.drop() start_size = scanner.NumGames() print scanner.StatusMsg() do_scan(scanner, games, overall_stats, args.max_games) print scanner.StatusMsg() end_size = scanner.NumGames() if args.incremental: existing_overall_data = DeckBuyStats() utils.read_object_from_db(existing_overall_data, buy_collection, '') overall_stats.Merge(existing_overall_data) def deck_freq(data_set): return data_set['Estate'].available.Frequency() print 'existing', deck_freq(existing_overall_data), 'decks' print 'after merge', deck_freq(overall_stats), 'decks' utils.write_object_to_db(overall_stats, buy_collection, '') scanner.Save() time_diff = time.time() - start games_diff = end_size - start_size print 'took', time_diff, 'seconds for', games_diff, 'games for a rate' \ ' of', games_diff / time_diff, 'games/sec'
def update_db(self, mongo_db_inst): for event_type_name, stats_dict in self.event_stats.iteritems(): log.debug('Updating database for event type %s, %d stats', event_type_name, len(stats_dict)) mongo_collection = mongo_db_inst[event_type_name] inserts = 0 updates = 0 for full_key, gain_stats_obj in sorted(stats_dict.iteritems()): existing_raw_obj = mongo_collection.find_one({'_id': full_key}) if existing_raw_obj: updates += 1 existing_stat = SmallGainStat() existing_stat.from_primitive_object( existing_raw_obj['vals']) gain_stats_obj.merge(existing_stat) else: inserts += 1 key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()} utils.write_object_to_db(key_wrap_obj, mongo_collection, full_key) log.debug('Database update results for %s: %d inserts, %d updates', event_type_name, inserts, updates)
def update_db(self, mongo_db_inst): for event_type_name, stats_dict in self.event_stats.iteritems(): log.debug('Updating database for event type %s, %d stats', event_type_name, len(stats_dict)) mongo_collection = mongo_db_inst[event_type_name] inserts = 0 updates = 0 for full_key, gain_stats_obj in sorted(stats_dict.iteritems()): existing_raw_obj = mongo_collection.find_one( {'_id': full_key}) if existing_raw_obj: updates += 1 existing_stat = SmallGainStat() existing_stat.from_primitive_object( existing_raw_obj['vals']) gain_stats_obj.merge(existing_stat) else: inserts += 1 key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()} utils.write_object_to_db(key_wrap_obj, mongo_collection, full_key) log.debug('Database update results for %s: %d inserts, %d updates', event_type_name, inserts, updates)
print scanner.StatusMsg() for idx, raw_game in enumerate(scanner.Scan(games, {})): try: if idx % 1000 == 0: print idx game_analysis.analyze_game(Game(raw_game)) if idx == args.max_games: break except int, exception: print Game(raw_game).IsotropicUrl() print exception print raw_game raise game_analysis.max_game_id = scanner.MaxGameId() game_analysis.num_games = scanner.NumGames() utils.write_object_to_db(game_analysis, output_collection, '') output_file = open(output_file_name, 'w') output_file.write('var all_card_data = ') json.dump(game_analysis.ToPrimitiveObject(), output_file) print scanner.StatusMsg() scanner.Save() if __name__ == '__main__': main()
def save(self): if not self.incremental: self.collection.drop() for key, tracker in self.trackers.iteritems(): utils.write_object_to_db(tracker, self.collection, key)
def save(self): for key, val in self.skill_infos.iteritems(): utils.write_object_to_db(val, self.coll, key)