Example #1
0
def main(parsed_args):
    """ Scan and update buy data"""
    start = time.time()
    db = utils.get_mongo_database()
    games = db.games
    output_db = db

    overall_stats = DeckBuyStats()

    scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db)
    buy_collection = output_db[BUYS_COL_NAME]

    if not parsed_args.incremental:
        log.warning('resetting scanner and db')
        scanner.reset()
        buy_collection.drop()

    start_size = scanner.get_num_games()
    log.info("Starting run: %s", scanner.status_msg())
    do_scan(scanner, games, overall_stats, parsed_args.max_games)
    log.info("Ending run: %s", scanner.status_msg())
    end_size = scanner.get_num_games()

    if parsed_args.incremental:
        existing_overall_data = DeckBuyStats()
        utils.read_object_from_db(existing_overall_data, buy_collection, '')
        overall_stats.merge(existing_overall_data)
        def deck_freq(data_set):
            return data_set[dominioncards.Estate].available.frequency()
        log.info('existing %s decks', deck_freq(existing_overall_data))
        log.info('after merge %s decks', deck_freq(overall_stats))

    utils.write_object_to_db(overall_stats, buy_collection, '')

    scanner.save()
Example #2
0
 def update_db(self, mongo_db_inst):
     for event_type_name, stats_dict in self.event_stats.iteritems():
         mongo_collection = mongo_db_inst[event_type_name]
         for full_key, gain_stats_obj in sorted(stats_dict.iteritems()):
             existing_raw_obj = mongo_collection.find_one({'_id': full_key})
             if existing_raw_obj:
                 existing_stat = SmallGainStat()
                 existing_stat.from_primitive_object(
                     existing_raw_obj['vals'])
                 gain_stats_obj.merge(existing_stat)
             key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()}
             utils.write_object_to_db(key_wrap_obj, mongo_collection,
                                      full_key)
Example #3
0
def main(args):
    """ Update analysis statistics.  By default, do so incrementally, unless
    --noincremental argument is given."""

    commit_after = 25000

    database = utils.get_mongo_database()
    games = database.games

    output_collection_name = 'analysis'
    output_collection = database[output_collection_name]
    game_analysis = GamesAnalysis()

    scanner = incremental_scanner.IncrementalScanner(output_collection_name,
                                                     database)

    if args.incremental:
        utils.read_object_from_db(game_analysis, output_collection, '')
    else:
        log.warning('resetting scanner and db')
        scanner.reset()

    output_file_name = 'static/output/all_games_card_stats.js'

    if not os.path.exists('static/output'):
        os.makedirs('static/output')

    log.info("Starting run: %s", scanner.status_msg())

    for idx, raw_game in enumerate(
            utils.progress_meter(scanner.scan(games, {}))):
        try:
            game_analysis.analyze_game(Game(raw_game))

            if args.max_games >= 0 and idx >= args.max_games:
                log.info("Reached max_games of %d", args.max_games)
                break

            if idx % commit_after == 0 and idx > 0:
                start = time.time()
                game_analysis.max_game_id = scanner.get_max_game_id()
                game_analysis.num_games = scanner.get_num_games()
                utils.write_object_to_db(game_analysis, output_collection, '')
                scanner.save()
                log.info("Committed calculations to the DB in %5.2fs",
                         time.time() - start)

        except int, exception:
            log.exception('Exception occurred for %s in raw game %s',
                          Game(raw_game).isotropic_url(), raw_game)
            raise
Example #4
0
 def update_db(self, mongo_db_inst):
     for event_type_name, stats_dict in self.event_stats.iteritems():
         mongo_collection = mongo_db_inst[event_type_name]
         for full_key, gain_stats_obj in sorted(stats_dict.iteritems()):
             existing_raw_obj = mongo_collection.find_one(
                 {'_id': full_key})
             if existing_raw_obj:
                 existing_stat = SmallGainStat()
                 existing_stat.from_primitive_object(
                     existing_raw_obj['vals'])
                 gain_stats_obj.merge(existing_stat)
             key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()}
             utils.write_object_to_db(key_wrap_obj, mongo_collection,
                                      full_key)
Example #5
0
def main(args):
    """ Update analysis statistics.  By default, do so incrementally, unless
    --noincremental argument is given."""

    commit_after = 25000

    database = utils.get_mongo_database()
    games = database.games

    output_collection_name = 'analysis'
    output_collection = database[output_collection_name]
    game_analysis = GamesAnalysis()

    scanner = incremental_scanner.IncrementalScanner(output_collection_name,
                                                     database)
 
    if args.incremental:
        utils.read_object_from_db(game_analysis, output_collection, '')
    else:
        log.warning('resetting scanner and db')
        scanner.reset()

    output_file_name = 'static/output/all_games_card_stats.js'

    if not os.path.exists('static/output'):
        os.makedirs('static/output')

    log.info("Starting run: %s", scanner.status_msg())

    for idx, raw_game in enumerate(utils.progress_meter(scanner.scan(games, {}))):
        try:
            game_analysis.analyze_game(Game(raw_game))

            if args.max_games >= 0 and idx >= args.max_games:
                log.info("Reached max_games of %d", args.max_games)
                break

            if idx % commit_after == 0 and idx > 0:
                start = time.time()
                game_analysis.max_game_id = scanner.get_max_game_id()
                game_analysis.num_games = scanner.get_num_games()
                utils.write_object_to_db(game_analysis, output_collection, '')
                scanner.save()
                log.info("Committed calculations to the DB in %5.2fs", time.time() - start)

        except int, exception:
            log.exception('Exception occurred for %s in raw game %s', Game(raw_game).isotropic_url(), raw_game)
            raise 
Example #6
0
def main():
    """ Scan and update buy data"""
    start = time.time()
    con = pymongo.Connection()
    games = con.test.games
    output_db = con.test

    parser = utils.incremental_parser()
    parser.add_argument('--max_games', default=-1, type=int)
    args = parser.parse_args()

    overall_stats = DeckBuyStats()

    scanner = incremental_scanner.IncrementalScanner('buys', output_db)
    buy_collection = output_db['buys']

    if not args.incremental:
        print 'resetting scanner and db'
        scanner.Reset()
        buy_collection.drop()

    start_size = scanner.NumGames()
    print scanner.StatusMsg()
    do_scan(scanner, games, overall_stats, args.max_games)
    print scanner.StatusMsg()
    end_size = scanner.NumGames()

    if args.incremental:
        existing_overall_data = DeckBuyStats()
        utils.read_object_from_db(existing_overall_data, buy_collection, '')
        overall_stats.Merge(existing_overall_data)

        def deck_freq(data_set):
            return data_set['Estate'].available.Frequency()

        print 'existing', deck_freq(existing_overall_data), 'decks'
        print 'after merge', deck_freq(overall_stats), 'decks'

    utils.write_object_to_db(overall_stats, buy_collection, '')

    scanner.Save()
    time_diff = time.time() - start
    games_diff = end_size - start_size
    print 'took', time_diff, 'seconds for', games_diff, 'games for a rate' \
        ' of', games_diff / time_diff, 'games/sec'
Example #7
0
def main():
    """ Scan and update buy data"""
    start = time.time()
    con = pymongo.Connection()
    games = con.test.games
    output_db = con.test

    parser = utils.incremental_parser()
    parser.add_argument('--max_games', default=-1, type=int)
    args = parser.parse_args()

    overall_stats = DeckBuyStats()

    scanner = incremental_scanner.IncrementalScanner('buys', output_db)
    buy_collection = output_db['buys']

    if not args.incremental:
        print 'resetting scanner and db'
        scanner.Reset()
        buy_collection.drop()

    start_size = scanner.NumGames()
    print scanner.StatusMsg()
    do_scan(scanner, games, overall_stats, args.max_games)
    print scanner.StatusMsg()
    end_size = scanner.NumGames()

    if args.incremental:
        existing_overall_data = DeckBuyStats()
        utils.read_object_from_db(existing_overall_data, buy_collection, '')
        overall_stats.Merge(existing_overall_data)
        def deck_freq(data_set):
            return data_set['Estate'].available.Frequency()
        print 'existing', deck_freq(existing_overall_data), 'decks'
        print 'after merge', deck_freq(overall_stats), 'decks'

    utils.write_object_to_db(overall_stats, buy_collection, '')

    scanner.Save()
    time_diff = time.time() - start
    games_diff = end_size - start_size
    print 'took', time_diff, 'seconds for', games_diff, 'games for a rate' \
        ' of', games_diff / time_diff, 'games/sec'
Example #8
0
 def update_db(self, mongo_db_inst):
     for event_type_name, stats_dict in self.event_stats.iteritems():
         log.debug('Updating database for event type %s, %d stats',
                   event_type_name, len(stats_dict))
         mongo_collection = mongo_db_inst[event_type_name]
         inserts = 0
         updates = 0
         for full_key, gain_stats_obj in sorted(stats_dict.iteritems()):
             existing_raw_obj = mongo_collection.find_one({'_id': full_key})
             if existing_raw_obj:
                 updates += 1
                 existing_stat = SmallGainStat()
                 existing_stat.from_primitive_object(
                     existing_raw_obj['vals'])
                 gain_stats_obj.merge(existing_stat)
             else:
                 inserts += 1
             key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()}
             utils.write_object_to_db(key_wrap_obj, mongo_collection,
                                      full_key)
         log.debug('Database update results for %s: %d inserts, %d updates',
                   event_type_name, inserts, updates)
Example #9
0
 def update_db(self, mongo_db_inst):
     for event_type_name, stats_dict in self.event_stats.iteritems():
         log.debug('Updating database for event type %s, %d stats',
                   event_type_name, len(stats_dict))
         mongo_collection = mongo_db_inst[event_type_name]
         inserts = 0
         updates = 0
         for full_key, gain_stats_obj in sorted(stats_dict.iteritems()):
             existing_raw_obj = mongo_collection.find_one(
                 {'_id': full_key})
             if existing_raw_obj:
                 updates += 1
                 existing_stat = SmallGainStat()
                 existing_stat.from_primitive_object(
                     existing_raw_obj['vals'])
                 gain_stats_obj.merge(existing_stat)
             else:
                 inserts += 1
             key_wrap_obj = {'vals': gain_stats_obj.to_primitive_object()}
             utils.write_object_to_db(key_wrap_obj, mongo_collection,
                                      full_key)
         log.debug('Database update results for %s: %d inserts, %d updates',
                   event_type_name, inserts, updates)
Example #10
0
def main(parsed_args):
    """ Scan and update buy data"""
    start = time.time()
    db = utils.get_mongo_database()
    games = db.games
    output_db = db

    overall_stats = DeckBuyStats()

    scanner = incremental_scanner.IncrementalScanner(BUYS_COL_NAME, output_db)
    buy_collection = output_db[BUYS_COL_NAME]

    if not parsed_args.incremental:
        log.warning('resetting scanner and db')
        scanner.reset()
        buy_collection.drop()

    start_size = scanner.get_num_games()
    log.info("Starting run: %s", scanner.status_msg())
    do_scan(scanner, games, overall_stats, parsed_args.max_games)
    log.info("Ending run: %s", scanner.status_msg())
    end_size = scanner.get_num_games()

    if parsed_args.incremental:
        existing_overall_data = DeckBuyStats()
        utils.read_object_from_db(existing_overall_data, buy_collection, '')
        overall_stats.merge(existing_overall_data)

        def deck_freq(data_set):
            return data_set[dominioncards.Estate].available.frequency()

        log.info('existing %s decks', deck_freq(existing_overall_data))
        log.info('after merge %s decks', deck_freq(overall_stats))

    utils.write_object_to_db(overall_stats, buy_collection, '')

    scanner.save()
Example #11
0
    print scanner.StatusMsg()

    for idx, raw_game in enumerate(scanner.Scan(games, {})):
        try:
            if idx % 1000 == 0:
                print idx
            game_analysis.analyze_game(Game(raw_game))

            if idx == args.max_games:
                break
        except int, exception:
            print Game(raw_game).IsotropicUrl()
            print exception
            print raw_game
            raise 

    game_analysis.max_game_id = scanner.MaxGameId()
    game_analysis.num_games = scanner.NumGames()
    utils.write_object_to_db(game_analysis, output_collection, '')

    output_file = open(output_file_name, 'w')
    output_file.write('var all_card_data = ')

    json.dump(game_analysis.ToPrimitiveObject(), output_file)
    print scanner.StatusMsg()
    scanner.Save()

if __name__ == '__main__':
    main() 
 def save(self):
     if not self.incremental:
         self.collection.drop()
     for key, tracker in self.trackers.iteritems():
         utils.write_object_to_db(tracker, self.collection, key)
Example #13
0
 def save(self):
     for key, val in self.skill_infos.iteritems():
         utils.write_object_to_db(val, self.coll, key)
 def save(self):
     if not self.incremental:
         self.collection.drop()
     for key, tracker in self.trackers.iteritems():
         utils.write_object_to_db(tracker, self.collection, key)
Example #15
0
 def save(self):
     for key, val in self.skill_infos.iteritems():
         utils.write_object_to_db(val, self.coll, key)