Ejemplo n.º 1
0
def collect_gamedata(game_list):
    """ Get game data from BGG.

    Description:
        Get game data in chunks of 50 games per api call

    inputs:
        game_list (list): List of ids

    returns:
        games (list): List of game objects """

    bgg = BGGClient(retries=6, retry_delay=4)

    chunksize = 50
    if len(game_list) < chunksize:
        games = bgg.game_list(game_list)
        return games

    games = []
    id_chunks = [
        game_list[i:i + chunksize] for i in range(0, len(game_list), chunksize)
    ]
    for i in id_chunks:
        games = games + bgg.game_list(i)
    return games
Ejemplo n.º 2
0
def get_games(username):
    """ Get games and game collection using bgg API2 
    returns: list of games and collection object """

    bgg = BGGClient(timeout=120, requests_per_minute=20)
    print("Getting collection from BGG..")
    collection = bgg.collection(username,
                                exclude_subtype='boardgameexpansion',
                                own=True,
                                wishlist=None)
    ids = [x.id for x in collection.items]
    game_list = []

    # get games from BGG
    try:
        print("Getting games from BGG..")
        game_list = bgg.game_list(ids)
        if not game_list:
            print("Error: empy list returned.")
    except:
        print("An Error occured..")
        raise TimeoutError
    else:
        print("Done.")
    return game_list, collection
Ejemplo n.º 3
0
class Downloader():
    def __init__(self, project_name, cache_bgg):
        if cache_bgg:
            self.client = BGGClient(cache=CacheBackendSqlite(
                path=f"{project_name}-cache.sqlite",
                ttl=60 * 60 * 24,
            ))
        else:
            self.client = BGGClient()

    def collection(self, user_name, extra_params):
        collection = []

        if isinstance(extra_params, list):
            for params in extra_params:
                collection += self.client.collection(
                    user_name=user_name,
                    **params,
                )
        else:
            collection = list(
                self.client.collection(
                    user_name=user_name,
                    **extra_params,
                ))

        games_data = self.client.game_list(
            [game_in_collection.id for game_in_collection in collection])

        games = list(filter(lambda x: not x.expansion, games_data))
        expansions = list(filter(lambda x: x.expansion, games_data))

        game_id_to_expansion = {game.id: [] for game in games}
        for expansion_data in expansions:
            for expands_game in expansion_data.expands:
                if expands_game.id in game_id_to_expansion:
                    game_id_to_expansion[expands_game.id].append(
                        expansion_data)

        game_id_to_tags = {game.id: [] for game in games}
        for stats_data in collection:
            if stats_data.id in game_id_to_tags:
                for tag in [
                        'preordered', 'prevowned', 'want', 'wanttobuy',
                        'wanttoplay', 'fortrade', 'wishlist'
                ]:
                    if int(getattr(stats_data, tag)):
                        game_id_to_tags[stats_data.id].append(tag)

        return [
            BoardGame(
                game_data,
                tags=game_id_to_tags[game_data.id],
                expansions=[
                    BoardGame(expansion_data)
                    for expansion_data in game_id_to_expansion[game_data.id]
                ]) for game_data in games
        ]
Ejemplo n.º 4
0
 def get_games_from_bgg(self, bgg: BGGClient, game_ids) -> List[BoardGame]:
     uncached_games = []
     found_cache_games = self.__get_games_from_cache(game_ids)
     cached_ids = self.__extract_ids_from_games(found_cache_games)
     game_list_not_found = [id for id in game_ids if id not in cached_ids]
     if game_list_not_found:
         uncached_games = bgg.game_list(game_list_not_found)
         for game in uncached_games:
             self.game_cache.save(game)
     return found_cache_games + uncached_games
Ejemplo n.º 5
0
class Downloader():
    def __init__(self, cache_bgg):
        project_name = SETTINGS["project"]["name"]
        if cache_bgg:
            self.client = BGGClient(cache=CacheBackendSqlite(
                path=f"{SETTINGS['project']['name']}-cache.sqlite",
                ttl=60 * 60 * 24,
            ))
        else:
            self.client = BGGClient()

    def collection(self, user_name):
        collection = self.client.collection(
            user_name=user_name, **SETTINGS["boardgamegeek"]["extra_params"])

        games_data = self.client.game_list(
            [game_in_collection.id for game_in_collection in collection.items])

        return [BoardGame(game_data) for game_data in games_data]
Ejemplo n.º 6
0
class Downloader():
    def __init__(self):
        project_name = SETTINGS["project"]["name"]
        self.client = BGGClient(cache=CacheBackendSqlite(
            path=f"{project_name}-cache.sqlite", ttl=60 * 60 * 24))

    def collection(self, user_name):
        collection = self.client.collection(
            user_name=user_name,
            exclude_subtype=u'boardgameexpansion',
            **SETTINGS["boardgamegeek"]["extra_params"])

        game_data = self.client.game_list(
            [game_in_collection.id for game_in_collection in collection.items])

        return [self.game_data_to_boardgame(game) for game in game_data]

    def _num_players_is_recommended(self, num, votes):
        return int(votes['best_rating']) + int(
            votes['recommended_rating']) > int(votes['not_recommended_rating'])

    def _facet_for_num_player(self, num, num_with_maybe_plus, votes):
        is_best = int(votes['best_rating']) > 10 and int(
            votes['best_rating']) > int(votes['recommended_rating'])
        best_or_recommended = "Best" if is_best else "Recommended"

        return {
            "level1":
            num,
            "level2":
            f"{num} > " + best_or_recommended + f" with {num_with_maybe_plus}",
        }

    def game_data_to_boardgame(self, game):
        num_players = []
        for num, votes in game.suggested_players['results'].items():
            if not self._num_players_is_recommended(num, votes):
                continue

            if "+" not in num:
                num_players.append(self._facet_for_num_player(num, num, votes))
            else:
                for i in range(int(num.replace("+", "")) + 1, 11):
                    num_players.append(
                        self._facet_for_num_player(i, num, votes))

        playing_time_mapping = {
            30: '< 30min',
            60: '30min - 1h',
            120: '1-2h',
            180: '2-3h',
            240: '3-4h',
        }
        for playing_time_max, playing_time in playing_time_mapping.items():
            if playing_time_max > int(game.playing_time):
                break
        else:
            playing_time = '> 4h'

        weight_mapping = {
            0: "Light",
            1: "Light",
            2: "Light Medium",
            3: "Medium",
            4: "Medium Heavy",
            5: "Heavy",
        }
        weight = weight_mapping[math.ceil(game.rating_average_weight)]

        return BoardGame(
            id=game.id,
            name=game.name,
            description=game.description,
            image=game.thumbnail,
            categories=[cat for cat in game.categories],
            mechanics=[mec for mec in game.mechanics],
            players=num_players,
            weight=weight,
            playing_time=playing_time,
        )
Ejemplo n.º 7
0
                num_player not in numfit[1] and
                round(game.rating_average_weight) == weight):
            yield i, game.name

bgg = BGGClient()
USERNAME = eval(input("BGG Username: "******"How many players: "))
playtime = eval(input("Maximum playtime in minutes: "))
weight = eval(input("Weight (1 = light, 5 = heavy): "))

match = search_match(game_list)
for num in range(5):
    print('Match: Game No. %d: %s' % (next(match)))
Ejemplo n.º 8
0
            writer.writerow(line)


bgg = BGGClient()

collection = bgg.collection('Oniwa',
                            exclude_subtype='boardgameexpansion',
                            own=True)
foo = []

game_id = []

for item in collection:
    game_id.append(item.id)

games = bgg.game_list(game_id)

for personal_game, db_game in zip(collection, games):
    game = BoardGame.BoardGame()
    game.collection_to_game(personal_game, db_game)
    foo.append(game)

# TODO: Change Average rank and personal rank to rating and add a personal rank field
csv_data = [[
    'Name', 'BGG Rank', 'Average Rank', 'Personal Rank', 'Weight',
    'Number Plays', 'Category', 'Mechanics', 'Min Players', 'Max Players',
    'Suggested Players', 'Year Published', 'Purchase Date', 'Months Owned'
]]

for item in foo:
    csv_data.append(item.csv())
Ejemplo n.º 9
0
if args.force:
    print("Forcing cache refresh")
    effective_cache_ttl = 0

cache = CacheBackendSqlite(path=".cache.bgg", ttl=effective_cache_ttl)

bgg1 = BGGClientLegacy(cache=cache)
list = bgg1.geeklist(args.list_id, comments=True)
print(f"[{list.id}] {list.name}\n{list.description}")

bgg2 = BGGClient(cache=cache)
games_id_list = [
    item.object.id for item in list
    if item.object.type == 'thing' and item.object.subtype == 'boardgame'
]
games = bgg2.game_list(games_id_list)
games_dict = {game.id: game for game in games}

collection = bgg2.collection(user_name=args.username)
collection_dict = {colgame.id: colgame for colgame in collection}


def text_formatter(game, effective_name, effective_imageid):
    print(f"  [{game.id}]"
          f" img:{effective_imageid} {effective_name}", end='')
    print(f" #{game.bgg_rank}", end='')
    print(f" year:{game.year}", end='')
    print(f" players:[{game.min_players}-{game.max_players}]", end='')
    print(f" age:>={game.min_age}yr", end='')
    print(f" time:{game.playing_time}'", end='')
    print(f" weight:{game.rating_average_weight:.1f}/5", end='')
def batch_api_call(ids: np.array,
                   batch_size: int = 100,
                   requests_per_minute: int = 100) -> list:
    """Fetches games data in batches

    First fetches data one batch at a time. Then fetches one more batch with remainder_ids
    i.e. ids which were left over after the last full batch was formed

    Args:
        ids (`numpy.array`): the game_ids to fetch data for
        batch_size (`int`): the size of the batches (Recommend to not exceed 100)
        requests_per_minute (`int`): limit the number of requests of our API client

    Returns:
        games (`list`): List of BoardGame objects. See boardgamegeek package documentation for details:
        https://lcosmin.github.io/boardgamegeek/modules.html#boardgamegeek.objects.games.BoardGame
    """

    # Instantiate API client
    bgg = BGGClient(requests_per_minute=requests_per_minute)
    logger.info(
        f'Instantiated API client with  limit of {requests_per_minute} requests per minute.'
    )

    # Form batches
    number_of_ids = len(ids)
    remainder = number_of_ids % batch_size
    ids_no_remainder = ids[:number_of_ids -
                           remainder]  # Making sure ids are divisible by batch_size
    batches = ids_no_remainder.reshape(int(len(ids_no_remainder) / batch_size),
                                       batch_size)
    batches = list(batches)  # Converting from np.ndarray to regular list
    batches = [lst.tolist()
               for lst in batches]  # converting each batch to regular list

    # Fetch batches via API
    batch_number = 0
    batches_successful = 0
    batches_failed = 0
    games = []
    logger.info(
        f'Beginning batch calls to BoardGameGeek API for {batch_size} games per batch.'
    )
    logger.info(
        f"This will take approximately 15 minutes. Thank you for your patience."
    )
    for batch in batches:
        logger.debug(
            f"Fetching data for batch number {batch_number} / {len(batches)}")
        try:
            game_batch = bgg.game_list(batch)
            batches_successful += 1
            logger.debug(
                f"Successfully fetched games in batch number {batch_number}")
            if batches_successful % 10 == 0:
                logger.info(
                    f'Successfully fetched games for {batches_successful} batches'
                )
            games.extend(game_batch)
        except BGGApiError:
            batches_failed += 1
            logger.debug(
                f"Failed to fetch games data for batch number {batch_number}")

        batch_number += 1

    logger.info(f"Successful Batches: {batches_successful} ")
    logger.info(f"Failed Batches: {batches_failed} ")

    # Fetch remaining games, which didn't fit in a batch
    remainder_ids = list(ids[len(ids_no_remainder):])
    remainder_ids_successful = 0
    remainder_ids_failed = 0
    try:
        games.extend(bgg.game_list(remainder_ids))
        logger.debug("Successfully fetched games with remainder ids")
        remainder_ids_successful = len(remainder_ids)
    except BGGApiError:
        logger.debug("Failed to fetch games with remainder ids")
        remainder_ids_failed = len(remainder_ids)

    logger.info(
        f"Total games successfully fetched: {batch_size*batches_successful+remainder_ids_successful}"
    )
    logger.info(
        f"Total games failed to fetch: {batch_size * batches_failed+ remainder_ids_failed}"
    )

    return games
Ejemplo n.º 11
0
class BGGManager():
    ''' The BGGManager class is a wrapper around the boardgamegeek2 API. Init
        with an ids file, a names file, and a details file. The instance can
        optionally download from boardgamegeek, or loaded from disk.'''
    def __init__(self, ids_file, names_file, details_file):
        '''ids_file - name of a file with/for BGG ids
           names_file - name of a file with/for BGG game names
           details_file - name of a file with/for BGG game details'''
        self._ids_file = ids_file
        self._ids = None
        self._names_file = names_file
        self._names = None
        self._details_file = details_file
        self._bgg = BGGClient()

    @staticmethod
    def grouper(n, iterable, fillvalue=None):
        '''Turns a flat list into a set of groups of length n. BGGManager uses
           this to group the downloaded ids into batches for retrieval from
           boargamegeek's batch API.
           Example: grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx
        '''
        args = [iter(iterable)] * n
        return zip_longest(fillvalue=fillvalue, *args)

    @staticmethod
    def boardgame_to_dict(bg):
        ''' Turns the structure returned by boardgamegeek2 into a dict, suitable
            for uploading to Elasticsearch.
            TODO: there are some deeper substructures - stats and ranks that
                  would be nice to have as well. They don't bear directly, so I
                  have ignored them.

            Initially I was trying to use class introspection to implement this
            but not all of the below are actually attributes of the class. This
            method is messier but actually works.'''
        return {
            "accessory": bg.accessory,
            "alternative_names": bg.alternative_names,
            "artists": bg.artists,
            "bgg_rank": bg.bgg_rank,
            "boardgame_rank": bg.boardgame_rank,
            "categories": bg.categories,
            "comments": bg.comments,
            "description": bg.description,
            "designers": bg.designers,
            "expansion": bg.expansion,
            "families": bg.families,
            "id": bg.id,
            "image": bg.image,
            "implementations": bg.implementations,
            "max_players": bg.max_players,
            "max_playing_time": bg.max_playing_time,
            "mechanics": bg.mechanics,
            "min_age": bg.min_age,
            "min_players": bg.min_players,
            "min_playing_time": bg.min_playing_time,
            "name": bg.name,
            "playing_time": bg.playing_time,
            "publishers": bg.publishers,
            "rating_average": bg.rating_average,
            "rating_average_weight": bg.rating_average_weight,
            "rating_bayes_average": bg.rating_bayes_average,
            "rating_median": bg.rating_median,
            "rating_num_weights": bg.rating_num_weights,
            "rating_stddev": bg.rating_stddev,
            "thumbnail": bg.thumbnail,
            "users_commented": bg.users_commented,
            "users_owned": bg.users_owned,
            "users_rated": bg.users_rated,
            "versions": bg.versions,
            "videos": bg.videos,
            "year": bg.year
        }

    def _download_and_save_games(self):
        ''' Use the API to pull games from boardgamegeek. There isn't a good way
            to search generically, the API requires at least 1 character for the
            wildcard. So I took the approach of wildcarding each letter of the
            alphabet. This misses some games, even some top games, but gets most
            everything.
            Note, since the API is searching across words, there are duplicate
            names and ids downloaded. This is handled with a set() structure to
            hold names and ids.
            Pickles ids and names that it downloads to the names file and ids
            file.'''
        self._ids = set()
        self._names = set()
        small_letters = map(chr, range(ord('a'), ord('z') + 1))
        for letter in small_letters:
            things = self._bgg.search('{}*'.format(letter))
            for thing in things:
                self._ids.add(thing.id)
                self._names.add(thing.name)
        with open(self._ids_file, 'wb') as ids_file:
            pickle.dump(self._ids, ids_file)
        with open(self._names_file, 'wb') as names_file:
            pickle.dump(self._names, names_file)

    def _load_saved_games(self):
        '''Loads the pickled game names and ids'''
        with open(self._names_file, 'rb') as names_file:
            self._names = pickle.load(names_file)

        with open(self._ids_file, 'rb') as ids_file:
            self._ids = pickle.load(ids_file)

    def load_game_names_and_ids(self, download=False):
        ''' Entry point for loading game names and ids. Set download=True to use
            the Boardgame Geek APIs to download them fresh, or False to load
            pickled data from a prior run. '''
        if download:
            self._download_and_save_games()
        else:
            self._load_saved_games()

        print('Got {} ids and {} names'.format(len(self._ids),
                                               len(self._names)))

    def _download_and_save_game_details(self):
        ''' Downloads game details from the BoardgameGeek API. This takes the
            ids and splits them into chunks of 100 so that it can call the BGG
            batch API for retrieval.
            After retrieving the game details, this reformats them as a dict for
            easier transmission to ES.
            I made the decision to store the text JSON of the details instead of
            pickling or storing in some other binary format. This facilitates
            quickly grepping the source data as well as processing to generate
            test sets.
            A bunch of the parameters (e.g. group size) might be better specified
            on the command line. For simplicity, they're hard-coded here.'''
        n = 0
        chunk_size = 100
        with open(self._details_file, 'w') as details_file:
            for chunk in self.grouper(chunk_size, self._ids, fillvalue=None):
                games = self._bgg.game_list(game_id_list=list(chunk))
                n += chunk_size
                print('Downloaded {} games.'.format(n))
                for g in games:
                    try:
                        d = self.boardgame_to_dict(g)
                        if d:
                            json.dump(d, details_file)
                            details_file.write('\n')
                    except Exception as e:
                        print(
                            'Exception getting details. Skipping "{}".'.format(
                                g.name))

    def _load_saved_game_details(self):
        ''' Loads the game details from the JSON file where they are stored. '''
        self._details = list()
        with open(self._details_file, 'r') as f_in:
            for line in f_in:
                dic = json.loads(line.lstrip().rstrip())
                self._details.append(dic)
        print("Loaded {} game records".format(len(self._details)))

    def load_game_details(self, download=False):
        ''' Entry point for downloading or loading the game details. Specify
            download=True to pull from BGG or download=False to load a previous
            data set. '''
        if download:
            if not self._ids or not self._names:
                # TODO: Better exception
                raise Exception(
                    'Can\'t download game details without loading names and ids first!'
                )
            self._download_and_save_game_details()
        else:
            self._load_saved_game_details()

    def game_details(self):
        ''' Iterator over the game details. Details must be loaded first. '''
        if not self._details:
            raise ValueError(
                'Trying to send iterate game details, but none loaded')
        for detail in self._details:
            yield detail