def run_trueskill_openings():
con = get_mongo_connection()
db = con.test
games = db.games
collection = db.trueskill_openings
player_collection = db.trueskill_players
# player_collection.remove()
# collection.remove()
setup_openings_collection(collection)
# setup_openings_collection(player_collection)
opening_skill_table = DbBackedSkillTable(collection)
# player_skill_table = DbBackedSkillTable(player_collection)
args = utils.incremental_max_parser().parse_args()
scanner = incremental_scanner.IncrementalScanner('trueskill', db)
if not args.incremental:
scanner.reset()
collection.drop()
for ind, game in enumerate(
progress_meter(scanner.scan(db.games, {}), 100)):
if len(game['decks']) >= 2 and len(game['decks'][1]['turns']) >= 5:
update_skills_for_game(game, opening_skill_table)
if ind == args.max_games:
break
#player_skill_table.save()
opening_skill_table.save()
scanner.save()
print scanner.status_msg()
def main(args):
""" Update analysis statistics. By default, do so incrementally, unless
--noincremental argument is given."""
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
output_collection_name = 'analysis'
output_collection = database[output_collection_name]
game_analysis = GamesAnalysis()
scanner = incremental_scanner.IncrementalScanner(output_collection_name,
database)
if args.incremental:
utils.read_object_from_db(game_analysis, output_collection, '')
else:
log.warning('resetting scanner and db')
scanner.reset()
output_file_name = 'static/output/all_games_card_stats.js'
if not os.path.exists('static/output'):
os.makedirs('static/output')
log.info("Starting run: %s", scanner.status_msg())
for idx, raw_game in enumerate(
utils.progress_meter(scanner.scan(games, {}))):
try:
game_analysis.analyze_game(Game(raw_game))
if args.max_games >= 0 and idx >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if idx % commit_after == 0 and idx > 0:
start = time.time()
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
scanner.save()
log.info("Committed calculations to the DB in %5.2fs",
time.time() - start)
except int, exception:
log.exception('Exception occurred for %s in raw game %s',
Game(raw_game).isotropic_url(), raw_game)
raise
def main(args):
""" Update analysis statistics. By default, do so incrementally, unless
--noincremental argument is given."""
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
output_collection_name = 'analysis'
output_collection = database[output_collection_name]
game_analysis = GamesAnalysis()
scanner = incremental_scanner.IncrementalScanner(output_collection_name,
database)
if args.incremental:
utils.read_object_from_db(game_analysis, output_collection, '')
else:
log.warning('resetting scanner and db')
scanner.reset()
output_file_name = 'static/output/all_games_card_stats.js'
if not os.path.exists('static/output'):
os.makedirs('static/output')
log.info("Starting run: %s", scanner.status_msg())
for idx, raw_game in enumerate(utils.progress_meter(scanner.scan(games, {}))):
try:
game_analysis.analyze_game(Game(raw_game))
if args.max_games >= 0 and idx >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if idx % commit_after == 0 and idx > 0:
start = time.time()
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
scanner.save()
log.info("Committed calculations to the DB in %5.2fs", time.time() - start)
except int, exception:
log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game)
raise
def main(args):
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
collection = database.optimal_card_ratios
db_tracker = None
scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios',
database)
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
log.info("Starting run: %s", scanner.status_msg())
for ind, game in enumerate(utils.progress_meter(scanner.scan(games, {}))):
if not db_tracker:
log.debug("Initializing db tracker manager")
db_tracker = DBCardRatioTrackerManager(collection,
args.incremental)
log.debug("DB tracker manager initialized")
result = process_game(Game(game))
for final_ratio_dict, progressive_ratio_dict, win_points in result:
db_tracker.integrate_results('final', final_ratio_dict, win_points)
db_tracker.integrate_results('progressive', progressive_ratio_dict,
win_points)
if args.max_games >= 0 and ind >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if ind % commit_after == 0 and ind > 0:
start = time.time()
db_tracker.save()
scanner.save()
log.info("Committed calculations to the DB in %5.2fs",
time.time() - start)
log.info("Ending run: %s", scanner.status_msg())
if db_tracker:
db_tracker.save()
scanner.save()
def main(args):
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
collection = database.optimal_card_ratios
db_tracker = None
scanner = incremental_scanner.IncrementalScanner('optimal_card_ratios', database)
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
log.info("Starting run: %s", scanner.status_msg())
for ind, game in enumerate(
utils.progress_meter(scanner.scan(games, {}))):
if not db_tracker:
log.debug("Initializing db tracker manager")
db_tracker = DBCardRatioTrackerManager(collection, args.incremental)
log.debug("DB tracker manager initialized")
result = process_game(Game(game))
for final_ratio_dict, progressive_ratio_dict, win_points in result:
db_tracker.integrate_results('final', final_ratio_dict, win_points)
db_tracker.integrate_results('progressive', progressive_ratio_dict, win_points)
if args.max_games >= 0 and ind >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if ind % commit_after == 0 and ind > 0:
start = time.time()
db_tracker.save()
scanner.save()
log.info("Committed calculations to the DB in %5.2fs", time.time() - start)
log.info("Ending run: %s", scanner.status_msg())
if db_tracker:
db_tracker.save()
scanner.save()
def main():
db = utils.get_mongo_database()
game_summaries = []
output_fn = open('margin.txt', 'w')
for idx, raw_game in enumerate(
utils.progress_meter(db.games.find({}), 1000)):
g = game.Game(raw_game)
if g.dubious_quality():
continue
if len(g.get_player_decks()) != 2:
continue
try:
joined_names = ','.join(
p.Name().encode('utf-8')
for p in g.get_player_decks(sort_by_turn_order=True))
output_fn.write('%f:%s:%s\n' %
(first_player_margin(g), joined_names, ','.join(
g.get_supply())))
except UnicodeDecodeError:
pass
def main():
db = utils.get_mongo_database()
game_summaries = []
output_fn = open('margin.txt', 'w')
for idx, raw_game in enumerate(
utils.progress_meter(db.games.find({}), 1000)):
g = game.Game(raw_game)
if g.dubious_quality():
continue
if len(g.get_player_decks()) != 2:
continue
try:
joined_names = ','.join(
p.Name().encode('utf-8') for p in
g.get_player_decks(sort_by_turn_order=True))
output_fn.write('%f:%s:%s\n' % (
first_player_margin(g),
joined_names,
','.join(g.get_supply())))
except UnicodeDecodeError:
pass
def run_trueskill_openings(args, db, log, commit_after=25000):
games = db.games
collection = db.trueskill_openings
player_collection = db.trueskill_players
# player_collection.remove()
# collection.remove()
setup_openings_collection(collection)
# setup_openings_collection(player_collection)
opening_skill_table = DbBackedSkillTable(collection)
# player_skill_table = DbBackedSkillTable(player_collection)
scanner = incremental_scanner.IncrementalScanner('trueskill', db)
log.info("Starting run: %s", scanner.status_msg())
if not args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
collection.drop()
for ind, game in enumerate(
utils.progress_meter(scanner.scan(db.games, {}))):
if ( len(game[DECKS]) >= 2 and len(game[DECKS][1][TURNS]) >= 5 and (RATING_SYSTEM not in game or (RATING_SYSTEM in game and 'adventure' not in game[RATING_SYSTEM] and 'unknown' not in game[RATING_SYSTEM]))):
update_skills_for_game(game, opening_skill_table)
if ind == args.max_games:
break
if ind % commit_after == 0 and ind > 0:
start = time.time()
#player_skill_table.save()
opening_skill_table.save()
scanner.save()
log.info("Committed calculations to the DB in %5.2fs", time.time() - start)
#player_skill_table.save()
opening_skill_table.save()
scanner.save()
log.info("Ending run: %s", scanner.status_msg())
def main():
c = utils.get_mongo_connection()
force_classification = True
prefix = 'data/test_small_'
# limit = 10000
r_output_file = open(prefix + 'r_format.data', 'w')
weka_output_file = open(prefix + 'games.arff', 'w')
librf_output_file = open(prefix + 'librf_games.csv', 'w')
librf_labels_file = open(prefix + 'librf_games_labels.txt', 'w')
libsvm_output_file = open(prefix + 'libsvm_games.txt', 'w')
write_r_header(r_output_file)
write_weka_header(weka_output_file, force_classification)
for raw_game in utils.progress_meter(
c.test.games.find({
'_id': {
'$gt': 'game-2010-10'
}
}).limit(20000), 100):
g = game.Game(raw_game)
if g.dubious_quality() or len(g.get_player_decks()) != 2:
continue
if force_classification and g.get_player_decks()[0].WinPoints() == 1.0:
print 'skipping tie'
saved_turn_ind = random.randint(0, len(g.get_turns()) - 1)
for ind, game_state in enumerate(g.game_state_iterator()):
# if ind == saved_turn_ind:
encoded_state = state_to_features(g, game_state)
if force_classification:
encoded_state[-1] = int(encoded_state[-1] / 2)
#output_state(encoded_state, r_output_file, ' ')
#output_state(encoded_state, weka_output_file, ',')
#output_state(encoded_state[:-1], librf_output_file, ',')
#librf_labels_file.write('%d\n' % encoded_state[-1])
output_libsvm_state(encoded_state, libsvm_output_file)
def main():
c = utils.get_mongo_connection()
force_classification = True
prefix = 'data/test_small_'
# limit = 10000
r_output_file = open(prefix + 'r_format.data', 'w')
weka_output_file = open(prefix + 'games.arff', 'w')
librf_output_file = open(prefix + 'librf_games.csv', 'w')
librf_labels_file = open(prefix + 'librf_games_labels.txt', 'w')
libsvm_output_file = open(prefix + 'libsvm_games.txt', 'w')
write_r_header(r_output_file)
write_weka_header(weka_output_file, force_classification)
for raw_game in utils.progress_meter(
c.test.games.find(
{'_id': {'$gt': 'game-2010-10'} }
).limit(20000), 100):
g = game.Game(raw_game)
if g.dubious_quality() or len(g.get_player_decks()) != 2:
continue
if force_classification and g.get_player_decks()[0].WinPoints() == 1.0:
print 'skipping tie'
saved_turn_ind = random.randint(0, len(g.get_turns()) - 1)
for ind, game_state in enumerate(g.game_state_iterator()):
# if ind == saved_turn_ind:
encoded_state = state_to_features(g, game_state)
if force_classification:
encoded_state[-1] = int(encoded_state[-1] / 2)
#output_state(encoded_state, r_output_file, ' ')
#output_state(encoded_state, weka_output_file, ',')
#output_state(encoded_state[:-1], librf_output_file, ',')
#librf_labels_file.write('%d\n' % encoded_state[-1])
output_libsvm_state(encoded_state, libsvm_output_file)
def main():
c = pymongo.Connection()
games_collection = c.test.games
output_collection = c.test.goals
total_checked = 0
checker_output = collections.defaultdict(list)
parser = utils.incremental_max_parser()
args = parser.parse_args()
scanner = incremental_scanner.IncrementalScanner('goals', c.test)
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('attainers.player')
output_collection.ensure_index('goal')
print 'starting with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
for goal_name, output in all_goals(game_val).items():
for attainer in output:
attainer['player'] = name_merger.norm_name(
attainer['player'])
checker_output[goal_name].append(
(game_val.isotropic_url(), output))
mongo_val = {'_id': game_val.get_id(),
'goal': goal_name,
'attainers': output}
output_collection.save(mongo_val)
print 'ending with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
scanner.save()
print_totals(checker_output, total_checked)
def main():
c = utils.get_mongo_connection()
force_classification = True
prefix = "data/test_huge_"
# limit = 10000
r_output_file = open(prefix + "r_format.data", "w")
weka_output_file = open(prefix + "games.arff", "w")
vw_output_file = open(prefix + "games.vw.txt", "w")
librf_output_file = open(prefix + "librf_games.csv", "w")
librf_labels_file = open(prefix + "librf_games_labels.txt", "w")
libsvm_output_file = open(prefix + "libsvm_games.txt", "w")
write_r_header(r_output_file)
write_weka_header(weka_output_file, force_classification)
for raw_game in utils.progress_meter(c.test.games.find()):
g = game.Game(raw_game)
if g.dubious_quality() or len(g.get_player_decks()) != 2:
continue
if force_classification and g.get_player_decks()[0].WinPoints() == 1.0:
print "skipping tie"
saved_turn_ind = random.randint(0, len(g.get_turns()) - 1)
for ind, game_state in enumerate(g.game_state_iterator()):
# if ind == saved_turn_ind:
encoded_state = state_to_features(g, game_state)
if force_classification:
encoded_state[-1] = int(encoded_state[-1] / 2)
# output_state(encoded_state, r_output_file, ' ')
# output_state(encoded_state, weka_output_file, ',')
# output_state(encoded_state[:-1], librf_output_file, ',')
# librf_labels_file.write('%d\n' % encoded_state[-1])
output_libsvm_state(encoded_state, libsvm_output_file)
output_vw_state(encoded_state, vw_output_file)
def main():
c = pymongo.Connection()
games_collection = c.test.games
output_collection = c.test.goals
stats_collection = c.test.goal_stats
total_checked = 0
checker_output = collections.defaultdict(int)
parser = utils.incremental_max_parser()
parser.add_argument(
'--goals',
metavar='goal_name',
nargs='+',
help=('If set, check only the goals specified for all of ' +
'the games that have already been scanned'))
args = parser.parse_args()
if args.goals:
valid_goals = True
for goal_name in args.goals:
if goal_name not in goal_check_funcs:
valid_goals = False
print "Unrecognized goal name '%s'" % goal_name
if not valid_goals:
exit(-1)
goals_to_check = args.goals
for goal_name in args.goals:
stats_collection.save({'_id': goal_name, 'total': 0})
scanner = incremental_scanner.IncrementalScanner('subgoals', c.test)
scanner.reset()
main_scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = main_scanner.get_max_game_id()
else:
goals_to_check = None
scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = None
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('goals.player')
print 'starting with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = output_collection.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict(dict)
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
name = name_merger.norm_name(goal['player'])
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
output_collection.save(mongo_val)
if last and game_id == last:
break
if args.max_games >= 0 and total_checked >= args.max_games:
break
print 'ending with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
scanner.save()
print_totals(checker_output, total_checked)
for goal_name, count in checker_output.items():
stats = stats_collection.find_one({'_id': goal_name})
if stats is None:
stats = {'_id': goal_name, 'total': 0}
stats['total'] += count
stats_collection.save(stats)
def main():
c = pymongo.Connection()
games_collection = c.test.games
output_collection = c.test.goals
stats_collection = c.test.goal_stats
total_checked = 0
checker_output = collections.defaultdict(int)
parser = utils.incremental_max_parser()
parser.add_argument(
'--goals', metavar='goal_name', nargs='+',
help=('If set, check only the goals specified for all of ' +
'the games that have already been scanned'))
args = parser.parse_args()
if args.goals:
valid_goals = True
for goal_name in args.goals:
if goal_name not in goal_check_funcs:
valid_goals = False
print "Unrecognized goal name '%s'" % goal_name
if not valid_goals:
exit(-1)
goals_to_check = args.goals
for goal_name in args.goals:
stats_collection.save( {'_id': goal_name, 'total': 0} )
scanner = incremental_scanner.IncrementalScanner('subgoals', c.test)
scanner.reset()
main_scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = main_scanner.get_max_game_id()
else:
goals_to_check = None
scanner = incremental_scanner.IncrementalScanner('goals', c.test)
last = None
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('goals.player')
print 'starting with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = output_collection.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict( dict )
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
name = name_merger.norm_name(goal['player'])
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
output_collection.save(mongo_val)
if last and game_id == last:
break
if args.max_games >= 0 and total_checked >= args.max_games:
break
print 'ending with id', scanner.get_max_game_id(), 'and num games', \
scanner.get_num_games()
scanner.save()
print_totals(checker_output, total_checked)
for goal_name, count in checker_output.items():
stats = stats_collection.find_one( {'_id': goal_name} )
if stats is None:
stats = {'_id': goal_name, 'total': 0}
stats['total'] += count
stats_collection.save( stats )
def main(args):
db = utils.get_mongo_database()
games_collection = db.games
output_collection = db.goals
total_checked = 0
checker_output = collections.defaultdict(int)
if args.goals:
valid_goals = True
for goal_name in args.goals:
if goal_name not in goal_check_funcs:
valid_goals = False
log.error("Unrecognized goal name '%s'", goal_name)
if not valid_goals:
exit(-1)
goals_to_check = args.goals
scanner = incremental_scanner.IncrementalScanner('subgoals', db)
scanner.reset()
main_scanner = incremental_scanner.IncrementalScanner('goals', db)
last = main_scanner.get_max_game_id()
else:
goals_to_check = None
scanner = incremental_scanner.IncrementalScanner('goals', db)
last = None
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('goals.player')
log.info("Starting run: %s", scanner.status_msg())
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = output_collection.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict( dict )
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
output_collection.save(mongo_val)
if last and game_id == last:
break
if args.max_games >= 0 and total_checked >= args.max_games:
break
log.info("Ending run: %s", scanner.status_msg())
scanner.save()
print_totals(checker_output, total_checked)
def main(args):
db = utils.get_mongo_database()
games_collection = db.games
output_collection = db.goals
total_checked = 0
checker_output = collections.defaultdict(int)
if args.goals:
valid_goals = True
for goal_name in args.goals:
if goal_name not in goal_check_funcs:
valid_goals = False
log.error("Unrecognized goal name '%s'", goal_name)
if not valid_goals:
exit(-1)
goals_to_check = args.goals
scanner = incremental_scanner.IncrementalScanner('subgoals', db)
scanner.reset()
main_scanner = incremental_scanner.IncrementalScanner('goals', db)
last = main_scanner.get_max_game_id()
else:
goals_to_check = None
scanner = incremental_scanner.IncrementalScanner('goals', db)
last = None
if not args.incremental:
scanner.reset()
output_collection.remove()
output_collection.ensure_index('goals.player')
log.info("Starting run: %s", scanner.status_msg())
for g in utils.progress_meter(scanner.scan(games_collection, {})):
total_checked += 1
game_val = game.Game(g)
# Get existing goal set (if exists)
game_id = game_val.get_id()
mongo_val = output_collection.find_one({'_id': game_id})
if mongo_val is None:
mongo_val = collections.defaultdict(dict)
mongo_val['_id'] = game_id
mongo_val['goals'] = []
# If rechecking, delete old values
if goals_to_check is not None:
goals = mongo_val['goals']
for ind in range(len(goals) - 1, -1, -1):
goal = goals[ind]
if goal['goal_name'] in goals_to_check:
del goals[ind]
# Get new values
goals = check_goals(game_val, goals_to_check)
# Write new values
for goal in goals:
goal_name = goal['goal_name']
mongo_val['goals'].append(goal)
checker_output[goal_name] += 1
mongo_val = dict(mongo_val)
output_collection.save(mongo_val)
if last and game_id == last:
break
if args.max_games >= 0 and total_checked >= args.max_games:
break
log.info("Ending run: %s", scanner.status_msg())
scanner.save()
print_totals(checker_output, total_checked)
def games_stream():
for raw_game in utils.progress_meter(scanner.scan(games_col, {}),
1000):
yield game.Game(raw_game)
def games_stream():
for raw_game in utils.progress_meter(
scanner.scan(games_col, {}), 1000):
yield game.Game(raw_game)