def main(parsed_args):
""" Scan and update buy data"""
start = time.time()
db = utils.get_mongo_database()
games = db.games
output_db = db
overall_stats = DeckBuyStats()
scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db)
buy_collection = output_db[BUYS_COL_NAME]
if not parsed_args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
buy_collection.drop()
start_size = scanner.get_num_games()
log.info("Starting run: %s", scanner.status_msg())
do_scan(scanner, games, overall_stats, parsed_args.max_games)
log.info("Ending run: %s", scanner.status_msg())
end_size = scanner.get_num_games()
if parsed_args.incremental:
existing_overall_data = DeckBuyStats()
utils.read_object_from_db(existing_overall_data, buy_collection, '')
overall_stats.merge(existing_overall_data)
def deck_freq(data_set):
return data_set[dominioncards.Estate].available.frequency()
log.info('existing %s decks', deck_freq(existing_overall_data))
log.info('after merge %s decks', deck_freq(overall_stats))
utils.write_object_to_db(overall_stats, buy_collection, '')
scanner.save()
def main(args):
""" Update analysis statistics. By default, do so incrementally, unless
--noincremental argument is given."""
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
output_collection_name = 'analysis'
output_collection = database[output_collection_name]
game_analysis = GamesAnalysis()
scanner = incremental_scanner.IncrementalScanner(output_collection_name,
database)
if args.incremental:
utils.read_object_from_db(game_analysis, output_collection, '')
else:
log.warning('resetting scanner and db')
scanner.reset()
output_file_name = 'static/output/all_games_card_stats.js'
if not os.path.exists('static/output'):
os.makedirs('static/output')
log.info("Starting run: %s", scanner.status_msg())
for idx, raw_game in enumerate(
utils.progress_meter(scanner.scan(games, {}))):
try:
game_analysis.analyze_game(Game(raw_game))
if args.max_games >= 0 and idx >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if idx % commit_after == 0 and idx > 0:
start = time.time()
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
scanner.save()
log.info("Committed calculations to the DB in %5.2fs",
time.time() - start)
except int, exception:
log.exception('Exception occurred for %s in raw game %s',
Game(raw_game).isotropic_url(), raw_game)
raise
def main(args):
""" Update analysis statistics. By default, do so incrementally, unless
--noincremental argument is given."""
commit_after = 25000
database = utils.get_mongo_database()
games = database.games
output_collection_name = 'analysis'
output_collection = database[output_collection_name]
game_analysis = GamesAnalysis()
scanner = incremental_scanner.IncrementalScanner(output_collection_name,
database)
if args.incremental:
utils.read_object_from_db(game_analysis, output_collection, '')
else:
log.warning('resetting scanner and db')
scanner.reset()
output_file_name = 'static/output/all_games_card_stats.js'
if not os.path.exists('static/output'):
os.makedirs('static/output')
log.info("Starting run: %s", scanner.status_msg())
for idx, raw_game in enumerate(utils.progress_meter(scanner.scan(games, {}))):
try:
game_analysis.analyze_game(Game(raw_game))
if args.max_games >= 0 and idx >= args.max_games:
log.info("Reached max_games of %d", args.max_games)
break
if idx % commit_after == 0 and idx > 0:
start = time.time()
game_analysis.max_game_id = scanner.get_max_game_id()
game_analysis.num_games = scanner.get_num_games()
utils.write_object_to_db(game_analysis, output_collection, '')
scanner.save()
log.info("Committed calculations to the DB in %5.2fs", time.time() - start)
except int, exception:
log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game)
raise
def main():
""" Scan and update buy data"""
start = time.time()
con = pymongo.Connection()
games = con.test.games
output_db = con.test
parser = utils.incremental_parser()
parser.add_argument('--max_games', default=-1, type=int)
args = parser.parse_args()
overall_stats = DeckBuyStats()
scanner = incremental_scanner.IncrementalScanner('buys', output_db)
buy_collection = output_db['buys']
if not args.incremental:
print 'resetting scanner and db'
scanner.Reset()
buy_collection.drop()
start_size = scanner.NumGames()
print scanner.StatusMsg()
do_scan(scanner, games, overall_stats, args.max_games)
print scanner.StatusMsg()
end_size = scanner.NumGames()
if args.incremental:
existing_overall_data = DeckBuyStats()
utils.read_object_from_db(existing_overall_data, buy_collection, '')
overall_stats.Merge(existing_overall_data)
def deck_freq(data_set):
return data_set['Estate'].available.Frequency()
print 'existing', deck_freq(existing_overall_data), 'decks'
print 'after merge', deck_freq(overall_stats), 'decks'
utils.write_object_to_db(overall_stats, buy_collection, '')
scanner.Save()
time_diff = time.time() - start
games_diff = end_size - start_size
print 'took', time_diff, 'seconds for', games_diff, 'games for a rate' \
' of', games_diff / time_diff, 'games/sec'
def main():
""" Update analysis statistics. By default, do so incrementally, unless
--noincremental argument is given."""
parser = utils.incremental_max_parser()
parser.add_argument('--output_collection_name', default='analysis')
args = parser.parse_args()
conn = pymongo.Connection()
database = conn.test
games = database.games
output_collection_name = args.output_collection_name
output_collection = database[output_collection_name]
game_analysis = GamesAnalysis()
scanner = incremental_scanner.IncrementalScanner(output_collection_name,
database)
if args.incremental:
utils.read_object_from_db(game_analysis, output_collection, '')
else:
scanner.Reset()
output_file_name = 'static/output/all_games_card_stats.js'
if not os.path.exists('static/output'):
os.makedirs('static/output')
print scanner.StatusMsg()
for idx, raw_game in enumerate(scanner.Scan(games, {})):
try:
if idx % 1000 == 0:
print idx
game_analysis.analyze_game(Game(raw_game))
if idx == args.max_games:
break
except int, exception:
print Game(raw_game).IsotropicUrl()
print exception
print raw_game
raise
def main():
""" Scan and update buy data"""
start = time.time()
con = pymongo.Connection()
games = con.test.games
output_db = con.test
parser = utils.incremental_parser()
parser.add_argument('--max_games', default=-1, type=int)
args = parser.parse_args()
overall_stats = DeckBuyStats()
scanner = incremental_scanner.IncrementalScanner('buys', output_db)
buy_collection = output_db['buys']
if not args.incremental:
print 'resetting scanner and db'
scanner.Reset()
buy_collection.drop()
start_size = scanner.NumGames()
print scanner.StatusMsg()
do_scan(scanner, games, overall_stats, args.max_games)
print scanner.StatusMsg()
end_size = scanner.NumGames()
if args.incremental:
existing_overall_data = DeckBuyStats()
utils.read_object_from_db(existing_overall_data, buy_collection, '')
overall_stats.Merge(existing_overall_data)
def deck_freq(data_set):
return data_set['Estate'].available.Frequency()
print 'existing', deck_freq(existing_overall_data), 'decks'
print 'after merge', deck_freq(overall_stats), 'decks'
utils.write_object_to_db(overall_stats, buy_collection, '')
scanner.Save()
time_diff = time.time() - start
games_diff = end_size - start_size
print 'took', time_diff, 'seconds for', games_diff, 'games for a rate' \
' of', games_diff / time_diff, 'games/sec'
def GET(self):
import count_buys
web.header("Content-Type", "text/html; charset=utf-8")
query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING']))
db = utils.get_mongo_database()
stats = count_buys.DeckBuyStats()
utils.read_object_from_db(stats, db.buys, '')
player_buy_summary = None
if 'player' in query_dict:
targ_name = norm_name(query_dict['player'])
games = map(game.Game, list(db.games.find({'players': targ_name})))
player_buy_summary = count_buys.DeckBuyStats()
match_name = lambda g, name: norm_name(name) == targ_name
count_buys.accum_buy_stats(games, player_buy_summary, match_name)
count_buys.add_effectiveness(player_buy_summary, stats)
render = web.template.render('', globals={'round': round})
return render.buy_template(stats, player_buy_summary)
def GET(self):
import count_buys
web.header("Content-Type", "text/html; charset=utf-8")
query_dict = dict(urlparse.parse_qsl(web.ctx.env['QUERY_STRING']))
db = utils.get_mongo_database()
stats = count_buys.DeckBuyStats()
utils.read_object_from_db(stats, db.buys, '')
player_buy_summary = None
if 'player' in query_dict:
targ_name = NormName(query_dict['player'])
games = map(game.Game, list(db.games.find({'players': targ_name})))
player_buy_summary = count_buys.DeckBuyStats()
match_name = lambda g, name: NormName(name) == targ_name
count_buys.accum_buy_stats(games, player_buy_summary, match_name)
count_buys.add_effectiveness(player_buy_summary, stats)
render = web.template.render('', globals={'round': round})
return render.buy_template(stats, player_buy_summary)
def main(parsed_args):
""" Scan and update buy data"""
start = time.time()
db = utils.get_mongo_database()
games = db.games
output_db = db
overall_stats = DeckBuyStats()
scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db)
buy_collection = output_db[BUYS_COL_NAME]
if not parsed_args.incremental:
log.warning('resetting scanner and db')
scanner.reset()
buy_collection.drop()
start_size = scanner.get_num_games()
log.info("Starting run: %s", scanner.status_msg())
do_scan(scanner, games, overall_stats, parsed_args.max_games)
log.info("Ending run: %s", scanner.status_msg())
end_size = scanner.get_num_games()
if parsed_args.incremental:
existing_overall_data = DeckBuyStats()
utils.read_object_from_db(existing_overall_data, buy_collection, '')
overall_stats.merge(existing_overall_data)
def deck_freq(data_set):
return data_set[dominioncards.Estate].available.frequency()
log.info('existing %s decks', deck_freq(existing_overall_data))
log.info('after merge %s decks', deck_freq(overall_stats))
utils.write_object_to_db(overall_stats, buy_collection, '')
scanner.save()
