def run_trueskill_openings():
con = get_mongo_connection()
db = con.test
games = db.games
collection = db.trueskill_openings
player_collection = db.trueskill_players
# player_collection.remove()
# collection.remove()
setup_openings_collection(collection)
# setup_openings_collection(player_collection)
opening_skill_table = DbBackedSkillTable(collection)
# player_skill_table = DbBackedSkillTable(player_collection)
args = utils.incremental_max_parser().parse_args()
scanner = incremental_scanner.IncrementalScanner('trueskill', db)
if not args.incremental:
scanner.reset()
collection.drop()
for ind, game in enumerate(
progress_meter(scanner.scan(db.games, {}), 100)):
if len(game['decks']) >= 2 and len(game['decks'][1]['turns']) >= 5:
update_skills_for_game(game, opening_skill_table)
if ind == args.max_games:
break
#player_skill_table.save()
opening_skill_table.save()
scanner.save()
print scanner.status_msg()
def main():
""" Update analysis statistics. By default, do so incrementally, unless
--noincremental argument is given."""
parser = utils.incremental_max_parser()
parser.add_argument('--output_collection_name', default='analysis')
args = parser.parse_args()
conn = pymongo.Connection()
database = conn.test
games = database.games
output_collection_name = args.output_collection_name
output_collection = database[output_collection_name]
game_analysis = GamesAnalysis()
scanner = incremental_scanner.IncrementalScanner(output_collection_name,
database)
if args.incremental:
utils.read_object_from_db(game_analysis, output_collection, '')
else:
scanner.Reset()
output_file_name = 'static/output/all_games_card_stats.js'
if not os.path.exists('static/output'):
os.makedirs('static/output')
print scanner.StatusMsg()
for idx, raw_game in enumerate(scanner.Scan(games, {})):
try:
if idx % 1000 == 0:
print idx
game_analysis.analyze_game(Game(raw_game))
if idx == args.max_games:
break
except int, exception:
print Game(raw_game).IsotropicUrl()
print exception
print raw_game
raise
def main():
""" Scan and update buy data"""
start = time.time()
con = pymongo.Connection()
games = con.test.games
output_db = con.test
parser = utils.incremental_max_parser()
args = parser.parse_args()
overall_stats = DeckBuyStats()
scanner = incremental_scanner.IncrementalScanner('buys', output_db)
buy_collection = output_db['buys']
if not args.incremental:
print 'resetting scanner and db'
scanner.reset()
buy_collection.drop()
start_size = scanner.get_num_games()
print scanner.status_msg()
do_scan(scanner, games, overall_stats, args.max_games)
print scanner.status_msg()
end_size = scanner.get_num_games()
if args.incremental:
existing_overall_data = DeckBuyStats()
utils.read_object_from_db(existing_overall_data, buy_collection, '')
overall_stats.merge(existing_overall_data)
def deck_freq(data_set):
return data_set['Estate'].available.frequency()
print 'existing', deck_freq(existing_overall_data), 'decks'
print 'after merge', deck_freq(overall_stats), 'decks'
utils.write_object_to_db(overall_stats, buy_collection, '')
scanner.save()
time_diff = time.time() - start
games_diff = end_size - start_size
print('took', time_diff, 'seconds for', games_diff, 'games for a rate of',
games_diff / time_diff, 'games/sec')
def main():
parser = utils.incremental_max_parser()
args = parser.parse_args()
conn = pymongo.Connection()
database = conn.test
games = database.games
collection = database.optimal_card_ratios
db_tracker = None
scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios',
database)
if not args.incremental:
scanner.reset()
print scanner.status_msg()
total_checked = 0
for game in scanner.scan(games, {}):
if not db_tracker:
db_tracker = DBCardRatioTrackerManager(collection,
args.incremental)
total_checked += 1
result = process_game(Game(game))
for final_ratio_dict, progressive_ratio_dict, win_points in result:
db_tracker.integrate_results('final', final_ratio_dict, win_points)
db_tracker.integrate_results('progressive', progressive_ratio_dict,
win_points)
if total_checked % 1000 == 0:
print total_checked
if args.max_games >= 0 and total_checked >= args.max_games:
break
print scanner.status_msg()
if db_tracker:
db_tracker.save()
scanner.save()
def main():
""" Scan and update buy data"""
start = time.time()
con = pymongo.Connection()
games = con.test.games
output_db = con.test
parser = utils.incremental_max_parser()
args = parser.parse_args()
overall_stats = DeckBuyStats()
scanner = incremental_scanner.IncrementalScanner('buys', output_db)
buy_collection = output_db['buys']
if not args.incremental:
print 'resetting scanner and db'
scanner.reset()
buy_collection.drop()
start_size = scanner.get_num_games()
print scanner.status_msg()
do_scan(scanner, games, overall_stats, args.max_games)
print scanner.status_msg()
end_size = scanner.get_num_games()
if args.incremental:
existing_overall_data = DeckBuyStats()
utils.read_object_from_db(existing_overall_data, buy_collection, '')
overall_stats.merge(existing_overall_data)
def deck_freq(data_set):
return data_set['Estate'].available.frequency()
print 'existing', deck_freq(existing_overall_data), 'decks'
print 'after merge', deck_freq(overall_stats), 'decks'
utils.write_object_to_db(overall_stats, buy_collection, '')
scanner.save()
time_diff = time.time() - start
games_diff = end_size - start_size
print ('took', time_diff, 'seconds for', games_diff, 'games for a rate of',
games_diff / time_diff, 'games/sec')
def main():
args = utils.incremental_max_parser().parse_args()
c = pymongo.Connection()
db = c.test
scanner = incremental_scanner.IncrementalScanner('analyze2', db)
if not args.incremental:
scanner.reset()
for collection_name, _ in event_detectors:
db[collection_name].drop()
print scanner.status_msg()
games_stream = analysis_util.games_stream(scanner, c.test.games)
accumulator = EventAccumulator()
accumulate_card_stats(games_stream, accumulator, args.max_games)
print 'saving to database'
accumulator.update_db(db)
scanner.save()
print scanner.status_msg()
def main():
args = utils.incremental_max_parser().parse_args()
c = pymongo.Connection()
db = c.test
scanner = incremental_scanner.IncrementalScanner('analyze2', db)
if not args.incremental:
scanner.reset()
for collection_name, _ in event_detectors:
db[collection_name].drop()
print scanner.status_msg()
games_stream = analysis_util.games_stream(scanner, c.test.games)
accumulator = EventAccumulator()
accumulate_card_stats(games_stream, accumulator, args.max_games)
print 'saving to database'
accumulator.update_db(db)
scanner.save()
print scanner.status_msg()
def main():
parser = utils.incremental_max_parser()
args = parser.parse_args()
conn = pymongo.Connection()
database = conn.test
games = database.games
collection = database.optimal_card_ratios
db_tracker = None
scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios', database)
if not args.incremental:
scanner.reset()
print scanner.status_msg()
total_checked = 0
for game in scanner.scan(games, {}):
if not db_tracker:
db_tracker = DBCardRatioTrackerManager(collection, args.incremental)
total_checked += 1
result = process_game(Game(game))
for final_ratio_dict, progressive_ratio_dict, win_points in result:
db_tracker.integrate_results('final', final_ratio_dict, win_points)
db_tracker.integrate_results('progressive', progressive_ratio_dict, win_points)
if total_checked % 1000 == 0:
print total_checked
if args.max_games >= 0 and total_checked >= args.max_games:
break
print scanner.status_msg()
if db_tracker:
db_tracker.save()
scanner.save()
def main():
c = pymongo.Connection()
games_collection = c.test.games
output_collection = c.test.goals
total_checked = 0
checker_output = collections.defaultdict(list)
parser = utils.incremental_max_parser()
args = parser.parse_args()
scanner = incremental_scanner.IncrementalScanner('goals', c.test)
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('attainers.player')
output_collection.ensure_index('goal')
print 'starting with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
for goal_name, output in all_goals(game_val).items():
for attainer in output:
attainer['player'] = name_merger.norm_name(
attainer['player'])
checker_output[goal_name].append(
(game_val.isotropic_url(), output))
mongo_val = {'_id': game_val.get_id(),
'goal': goal_name,
'attainers': output}
output_collection.save(mongo_val)
print 'ending with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
scanner.save()
print_totals(checker_output, total_checked)
start = time.time()
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
scanner.save()
log.info("Committed calculations to the DB in %5.2fs",
time.time() - start)
except exception:
log.exception('Exception occurred for %s in raw game %s',
Game(raw_game).isotropic_url(), raw_game)
raise
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
output_file = open(output_file_name, 'w')
output_file.write('var all_card_data = ')
json.dump(game_analysis.to_primitive_object(), output_file)
log.info("Ending run: %s", scanner.status_msg())
scanner.save()
if __name__ == '__main__':
parser = utils.incremental_max_parser()
args = parser.parse_args()
dominionstats.utils.log.initialize_logging(args.debug)
main(args)
if idx % commit_after == 0 and idx > 0:
start = time.time()
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
scanner.save()
log.info("Committed calculations to the DB in %5.2fs", time.time() - start)
except int, exception:
log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game)
raise
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
output_file = open(output_file_name, 'w')
output_file.write('var all_card_data = ')
json.dump(game_analysis.to_primitive_object(), output_file)
log.info("Ending run: %s", scanner.status_msg())
scanner.save()
if __name__ == '__main__':
parser = utils.incremental_max_parser()
args = parser.parse_args()
dominionstats.utils.log.initialize_logging(args.debug)
main(args)
def main():
c = pymongo.Connection()
games_collection = c.test.games
output_collection = c.test.goals
stats_collection = c.test.goal_stats
total_checked = 0
checker_output = collections.defaultdict(int)
parser = utils.incremental_max_parser()
parser.add_argument(
'--goals',
metavar='goal_name',
nargs='+',
help=('If set, check only the goals specified for all of ' +
'the games that have already been scanned'))
args = parser.parse_args()
if args.goals:
valid_goals = True
for goal_name in args.goals:
if goal_name not in goal_check_funcs:
valid_goals = False
print "Unrecognized goal name '%s'" % goal_name
if not valid_goals:
exit(-1)
goals_to_check = args.goals
for goal_name in args.goals:
stats_collection.save({'_id': goal_name, 'total': 0})
scanner = incremental_scanner.IncrementalScanner('subgoals', c.test)
scanner.reset()
main_scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = main_scanner.get_max_game_id()
else:
goals_to_check = None
scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = None
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('goals.player')
print 'starting with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = output_collection.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict(dict)
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
name = name_merger.norm_name(goal['player'])
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
output_collection.save(mongo_val)
if last and game_id == last:
break
if args.max_games >= 0 and total_checked >= args.max_games:
break
print 'ending with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
scanner.save()
print_totals(checker_output, total_checked)
for goal_name, count in checker_output.items():
stats = stats_collection.find_one({'_id': goal_name})
if stats is None:
stats = {'_id': goal_name, 'total': 0}
stats['total'] += count
stats_collection.save(stats)
def main():
c = pymongo.Connection()
games_collection = c.test.games
output_collection = c.test.goals
stats_collection = c.test.goal_stats
total_checked = 0
checker_output = collections.defaultdict(int)
parser = utils.incremental_max_parser()
parser.add_argument(
'--goals', metavar='goal_name', nargs='+',
help=('If set, check only the goals specified for all of ' +
'the games that have already been scanned'))
args = parser.parse_args()
if args.goals:
valid_goals = True
for goal_name in args.goals:
if goal_name not in goal_check_funcs:
valid_goals = False
print "Unrecognized goal name '%s'" % goal_name
if not valid_goals:
exit(-1)
goals_to_check = args.goals
for goal_name in args.goals:
stats_collection.save( {'_id': goal_name, 'total': 0} )
scanner = incremental_scanner.IncrementalScanner('subgoals', c.test)
scanner.reset()
main_scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = main_scanner.get_max_game_id()
else:
goals_to_check = None
scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = None
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('goals.player')
print 'starting with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = output_collection.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict( dict )
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
name = name_merger.norm_name(goal['player'])
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
output_collection.save(mongo_val)
if last and game_id == last:
break
if args.max_games >= 0 and total_checked >= args.max_games:
break
print 'ending with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
scanner.save()
print_totals(checker_output, total_checked)
for goal_name, count in checker_output.items():
stats = stats_collection.find_one( {'_id': goal_name} )
if stats is None:
stats = {'_id': goal_name, 'total': 0}
stats['total'] += count
stats_collection.save( stats )
