def retrieve(self, request, pk=None):
"""Handle GET requests for single game
Returns:
Response -- JSON serialized game instance
"""
try:
game = Game.objects.get(pk=pk)
bgg = BGGClient()
player = Player.objects.get(user=game.player.user)
BGGObj = bgg.game(game_id=str(game.game))
game1={}
game1['name'] = game.name
game1["api_id"] = game.game
game1["api_game_name"]= BGGObj.name
playerObj = PlayerSerializer(player, context={'request': request})
game1['player'] = playerObj.data
game1['host_descrip'] = game.host_descrip
game1['max_players'] = BGGObj.max_players
game1['min_players'] = BGGObj.min_players
game1['category_ids'] = game.category_ids
game1['categories'] = []
for category in BGGObj.categories:
game1['categories'].append(category)
game1['image'] = BGGObj.image
game1['thumb_nail'] = BGGObj.thumbnail
return Response(game1)
except Exception as ex:
return HttpResponseServerError(ex)
def get_games(username):
""" Get games and game collection using bgg API2
returns: list of games and collection object """
bgg = BGGClient(timeout=120, requests_per_minute=20)
print("Getting collection from BGG..")
collection = bgg.collection(username,
exclude_subtype='boardgameexpansion',
own=True,
wishlist=None)
ids = [x.id for x in collection.items]
game_list = []
# get games from BGG
try:
print("Getting games from BGG..")
game_list = bgg.game_list(ids)
if not game_list:
print("Error: empy list returned.")
except:
print("An Error occured..")
raise TimeoutError
else:
print("Done.")
return game_list, collection
def new_game():
client = BGGClient()
not_found = True
while not_found:
game_search = input_prompt(
message=GAME_MESSAGE,
validation_function=lambda x: True
if x != "" else "Enter something!",
)
if game_search.lower() == "r":
return
games = client.search(game_search)
game_names = [game.name for game in games]
if not game_names:
print("No games found")
continue
game_question_list = game_names + [GAME_CANCEL_OPTION]
answer = list_prompt(message=GAME_CHOICE_MESSAGE,
items=game_question_list)
if answer == GAME_CANCEL_OPTION:
continue
if confirmation_prompt(
message=f"Are you sure you would like to add {answer}?"):
not_found = False
add_game(games[game_names.index(answer)])
def collect_gamedata(game_list):
""" Get game data from BGG.
Description:
Get game data in chunks of 50 games per api call
inputs:
game_list (list): List of ids
returns:
games (list): List of game objects """
bgg = BGGClient(retries=6, retry_delay=4)
chunksize = 50
if len(game_list) < chunksize:
games = bgg.game_list(game_list)
return games
games = []
id_chunks = [
game_list[i:i + chunksize] for i in range(0, len(game_list), chunksize)
]
for i in id_chunks:
games = games + bgg.game_list(i)
return games
def get_external_game(bgg_id):
bgg = BGGClient()
max_count = 10
while max_count:
try:
return bgg.game(game_id=bgg_id)
except bgg_exceptions.BGGValueError:
print("[ERROR] Invalid parameters")
raise
except bgg_exceptions.BGGApiRetryError:
print("[ERROR] Retry after delay, retrying...")
BoardGameGeek._retry(max_count)
except bgg_exceptions.BGGApiError:
print("[ERROR] API response invalid or not parsed")
BoardGameGeek._retry(max_count)
except bgg_exceptions.BGGApiTimeoutError:
print("[ERROR] Timeout")
BoardGameGeek._retry(max_count)
except Exception as err:
print("[ERROR] Exception caught getting external game: " +
str(err))
BoardGameGeek._retry(max_count)
raise Exception
def search_boardgame(boardgame, game_type=None):
"""
Uses the boardgamemeek XML API to search for games
Currently supports boardgames and TT RPGS
"""
if game_type is None:
search_type = None
game_type = 'boardgame'
elif game_type == 'RPG':
game_type = 'rpgitem'
search_type = [BGGRestrictSearchResultsTo.RPG]
bgg = BGGClient()
res = bgg.search(boardgame, search_type=search_type)
# this is terrible
if len(res) == 0:
return "I couldn't find any game similar to the title %s" % boardgame
print([x.data() for x in res[0:10]])
games = [x for x in res if boardgame.lower() == x.name.lower()]
resp = ""
if len(games) == 0:
resp += ("Couldn't find game {} exactly, but I found {} which " +
"looks close\n").format(boardgame, len(res))
games = [x for x in res if boardgame.lower() in x.name.lower()]
if len(games) == 0:
resp += "Well, not exactly close, "
resp += "but here's the oldest one on the list\n"
games = res
g = sorted(games, key=lambda x: x.year)[0]
url_format = "https://www.boardgamegeek.com/{game_tp}/{id}/{name}".format(
game_tp=game_type, id=g.id, name=g.name.lower().replace(' ', '-'))
return resp + url_format
def __init__(self, project_name, cache_bgg):
if cache_bgg:
self.client = BGGClient(cache=CacheBackendSqlite(
path=f"{project_name}-cache.sqlite",
ttl=60 * 60 * 24,
))
else:
self.client = BGGClient()
def category_ids(self):
bgg = BGGClient()
id_list = []
api_game = bgg.game(game_id=str(self.game))
for category in api_game.categories:
cat = Category.objects.get(name=category)
id_list.append(cat.id)
return id_list
def get_guild_user_list(guild_id, bgg=None):
"""Fetch the member list for a BGG Guild"""
if bgg is None:
bgg = BGGClient()
print 'Fetching guild user list'
guild = bgg.guild(guild_id)
return list(guild.members)
class Downloader():
def __init__(self, project_name, cache_bgg):
if cache_bgg:
self.client = BGGClient(cache=CacheBackendSqlite(
path=f"{project_name}-cache.sqlite",
ttl=60 * 60 * 24,
))
else:
self.client = BGGClient()
def collection(self, user_name, extra_params):
collection = []
if isinstance(extra_params, list):
for params in extra_params:
collection += self.client.collection(
user_name=user_name,
**params,
)
else:
collection = list(
self.client.collection(
user_name=user_name,
**extra_params,
))
games_data = self.client.game_list(
[game_in_collection.id for game_in_collection in collection])
games = list(filter(lambda x: not x.expansion, games_data))
expansions = list(filter(lambda x: x.expansion, games_data))
game_id_to_expansion = {game.id: [] for game in games}
for expansion_data in expansions:
for expands_game in expansion_data.expands:
if expands_game.id in game_id_to_expansion:
game_id_to_expansion[expands_game.id].append(
expansion_data)
game_id_to_tags = {game.id: [] for game in games}
for stats_data in collection:
if stats_data.id in game_id_to_tags:
for tag in [
'preordered', 'prevowned', 'want', 'wanttobuy',
'wanttoplay', 'fortrade', 'wishlist'
]:
if int(getattr(stats_data, tag)):
game_id_to_tags[stats_data.id].append(tag)
return [
BoardGame(
game_data,
tags=game_id_to_tags[game_data.id],
expansions=[
BoardGame(expansion_data)
for expansion_data in game_id_to_expansion[game_data.id]
]) for game_data in games
]
def is_full(self):
gameObj = Game.objects.get(pk=self.game_id)
bgg= BGGClient()
game = bgg.game(game_id=str(gameObj.game))
if len(self.player_list) >= int(game.max_players):
return True
else:
return False
def __init__(self, cache_bgg):
project_name = SETTINGS["project"]["name"]
if cache_bgg:
self.client = BGGClient(cache=CacheBackendSqlite(
path=f"{SETTINGS['project']['name']}-cache.sqlite",
ttl=60 * 60 * 24,
))
else:
self.client = BGGClient()
def need_players(self):
gameObj = Game.objects.get(pk=self.game_id)
bgg= BGGClient()
game = bgg.game(game_id=str(gameObj.game))
if len(self.player_list) < int(game.min_players):
players_needed = int(game.min_players)- len(self.player_list)
return players_needed
else:
return 0
def __init__(self, ids_file, names_file, details_file):
'''ids_file - name of a file with/for BGG ids
names_file - name of a file with/for BGG game names
details_file - name of a file with/for BGG game details'''
self._ids_file = ids_file
self._ids = None
self._names_file = names_file
self._names = None
self._details_file = details_file
self._bgg = BGGClient()
def get_collection(username: str) -> BGGClient.collection:
bgg = BGGClient()
for n in range(0, 8):
try:
return bgg.collection(username)
except Exception as error:
logging.warning(
str(error) + ' error received, trying again in ' + str(3**n) +
' seconds')
time.sleep(3**n)
logging.error('too many errors, gg')
def get_user(username: str) -> User:
logging.info(f'fetching data from user {username} from bgg')
bgg = BGGClient()
for n in range(0, 8):
try:
user_data = bgg.user(username)
logging.info('user data received')
return user_data
except Exception as error:
logging.warning(str(error) + ' error received, trying again in ' + str(3**n) + ' seconds')
time.sleep(3**n)
logging.error('too many errors, gg')
def get_user_ratings(username, bgg=None):
"""Returns a dict: gameid -> rating"""
if bgg is None:
bgg = BGGClient()
collection = bgg.collection(username)
user_ratings = dict()
for item in collection:
if item.rating:
user_ratings[item.id] = item.rating
return user_ratings
def get_all_ratings(members, bgg=None):
"""Get the ratings for all users in the list members.
Returns: A dict (gameid, game name) -> list of ratings
"""
if bgg is None:
bgg = BGGClient()
all_member_ratings = dict()
print 'Retrieving user ratings...'
work_queue = Queue()
for member in members:
work_queue.put(member)
while not work_queue.empty():
print work_queue.qsize(), 'members to process'
member = work_queue.get()
print 'Fetching data for ', member
try:
user_ratings = get_user_ratings(member, bgg=bgg)
except Exception:
work_queue.put(member)
continue
all_member_ratings[member] = user_ratings
print 'Ratings retrieved for all users.'
return all_member_ratings
def get_game_info(game_id, bgg=None):
"""Fetch the BGG info for game having game_id"""
if bgg is None:
bgg = BGGClient()
print 'Fetching info for game', str(game_id)
game = None
while game is None:
try:
game = bgg.game(game_id=game_id)
except Exception:
print 'Trying to fetch again...'
continue
return game
def create_client():
return BGGClient(cache=CacheBackendSqlite(
path=BoardGameFactory.cache_location,
ttl=BoardGameFactory.item_cache_duration),
timeout=60,
retry_delay=10,
retries=6)
def create(bgg_id):
client = BGGClient()
game = BoardGameGeek.get_external_game(bgg_id)
bgg_game = BggGame(bggid=game.id)
bgg_game = BoardGameGeek.convert_external_game(external=game,
game=bgg_game)
bgg_game.save()
return bgg_game
class Downloader():
def __init__(self, cache_bgg):
project_name = SETTINGS["project"]["name"]
if cache_bgg:
self.client = BGGClient(cache=CacheBackendSqlite(
path=f"{SETTINGS['project']['name']}-cache.sqlite",
ttl=60 * 60 * 24,
))
else:
self.client = BGGClient()
def collection(self, user_name):
collection = self.client.collection(
user_name=user_name, **SETTINGS["boardgamegeek"]["extra_params"])
games_data = self.client.game_list(
[game_in_collection.id for game_in_collection in collection.items])
return [BoardGame(game_data) for game_data in games_data]
def get_games_from_bgg(self, bgg: BGGClient, game_ids) -> List[BoardGame]:
uncached_games = []
found_cache_games = self.__get_games_from_cache(game_ids)
cached_ids = self.__extract_ids_from_games(found_cache_games)
game_list_not_found = [id for id in game_ids if id not in cached_ids]
if game_list_not_found:
uncached_games = bgg.game_list(game_list_not_found)
for game in uncached_games:
self.game_cache.save(game)
return found_cache_games + uncached_games
def get(self, request):
form = SearchForm(request.GET)
if form.is_valid():
bgg = BGGClient()
query = form.cleaned_data["query"]
query = bgg.search(query)
query = [q.data() for q in query]
keys = ['id', 'name', 'type', 'yearpublished']
query_items = [[(key, data[key]) for key in keys]
for data in query]
return render(request, "results.html", {
"query_items": query_items,
"keys": keys,
})
else:
return render(request, "main.html", {"form": form})
def user(request, username, owned, rating):
bgg = BGGClient(retries=10, retry_delay=10)
coll = bgg.collection(username)
running_score = defaultdict(float)
n = 0
for game in coll:
game_obj, game_created = BoardGame.objects.get_or_create(
title=game.name, bgg_id=game.id)
coll_obj, created = BGGCollection.objects.update_or_create(
boardgame=game_obj,
rating=game.rating,
owned=bool(game.owned),
comment=game.comment,
)
if game_created: # Newly added board games would not have genre and mechanism
continue
if bool(owned) and not bool(game.owned):
continue
if game.rating is not None:
if rating > game.rating:
continue
combined_score = _recommend_boardgames(game_obj)
n += 1
for k, v in combined_score.items():
running_score[k] += v
running_score = {k: v / n for k, v in running_score.items()}
running_score = sorted(running_score.items(), key=lambda x: -x[1])
return render(
request, 'genome/user.html', {
'username': username,
'recs': running_score,
'form': form,
'user_form': user_form,
})
def open_file_find_games(path, out=None):
lines = None
with open(path, 'r') as file:
lines = [l.strip() for l in file.readlines()]
click.echo('Read {} lines'.format(len(lines)))
client = BGGClient()
games = search_games(lines, client)
click.echo('Found {} games'.format(len(games)))
if out:
with open(out, 'w', newline='') as file:
fieldnames = list(printable_game_dict(games[0].data()).keys())
csvfile = csv.DictWriter(file,
delimiter=',',
fieldnames=fieldnames)
csvfile.writeheader()
for game in games:
printable = printable_game_dict(game.data())
logger.debug(printable)
csvfile.writerow(printable)
def main(user, member_data_file):
bgg = BGGClient()
with open(member_data_file, 'r') as data_file:
member_data = yaml.load(data_file)
user_data = member_data[user]
del member_data[user]
user_collection_size = len(user_data)
member_scores = list()
for user, ratings in member_data.iteritems():
score = 0
games_in_common = 0
for game, rating in user_data.iteritems():
if game in ratings:
diff = (rating - ratings[game])**2
score += diff
games_in_common += 1
member_scores.append({
'user': user,
'score': score,
'common': games_in_common
})
member_scores = filter(lambda x: x['common'] >= 0.5 * user_collection_size,
member_scores)
member_scores.sort(key=lambda x: x['score'])
filename = user + '_followers.yml'
with open(filename, 'w') as fo:
yaml.dump(member_scores, fo)
for i in range(5):
member = member_scores[i]
print member['user'], member['score'], member['common']
import re
import time
from bs4 import BeautifulSoup
from boardgamegeek import BGGClient
import scrapy
from scrapy.spiders import CrawlSpider
from scrapy.item import Item, Field
# from selenium.common.exceptions import NoSuchElementException
# pip install boardgamegeek2 to install boardgameGeek
#I have used selenium to scrape using XPATH
from selenium import webdriver
gameurl = 'https://boardgamegeek.com/boardgame/'
bgg = BGGClient()
#based on the platform download phantomjs and give the path to the the executable
browser = webdriver.PhantomJS()
class Game(Item):
id = Field()
title = Field()
geek_rate = Field()
avg_rate = Field()
num_votes = Field()
type = Field()
yearpublished = Field()
minplayers = Field()
maxplayers = Field()
playingtime = Field()
minplaytime = Field()
def main(users=None, raw_data=None, generate_report=False, prune=None):
bgg = BGGClient()
# if not users and not raw_data: get users, get user ratings, process ratings
# if users and not raw_data: load users, get user ratings, process ratings
# if raw data: load users, load user ratings, process ratings
now_str = str(datetime.datetime.now())
filename_tag = '_'.join(now_str[:10].split())
if raw_data is None:
# load members from file or query for current list
if users is None:
members = get_guild_user_list(PUNCHING_CARDBOARD, bgg=bgg)
of = open('members_' + filename_tag + '.txt', 'w')
for member in members:
of.write(member + '\n')
else:
members = load_members_from_file(users)
guild_size = len(members)
print 'Members list loaded: %d guild members' % guild_size
member_ratings = get_all_ratings(members, bgg=bgg)
guild_ratings = collapse_ratings(member_ratings)
print 'Processing results...'
print '%d games rated' % len(guild_ratings)
top_games = list()
for game_id, ratings in guild_ratings.iteritems():
num_ratings = len(ratings)
avg_rating = round(mean(ratings), 3)
sd_ratings = round(stdev(ratings), 3)
top_games.append((game_id, num_ratings, avg_rating, sd_ratings))
# Sort the list
top_games.sort(key=lambda x: x[2], reverse=True)
# Write out the raw data to this point
current_time_str = str(datetime.datetime.now())
rating_data = dict()
rating_data[SUMMARY] = { GUILD_MEMBER_COUNT: guild_size,
TOTAL_GAMES: len(guild_ratings),
TIME: current_time_str
}
rating_data[MEMBERS] = members
rating_data[SORTED_GAMES] = top_games
with open('guild_data_' + filename_tag + '.json', 'w') as raw_data_file:
json.dump(rating_data, raw_data_file)
with open('member_data_' + filename_tag + '.yml', 'w') as raw_data_file:
yaml.dump(member_ratings, raw_data_file)
elif raw_data is not None:
rating_data = json.load(open(raw_data, 'r'))
# Either path we now have rating_data
top_games = rating_data[SORTED_GAMES]
member_count = rating_data[SUMMARY][GUILD_MEMBER_COUNT]
# If we want to prune the games
if prune is not None:
pruned_games = list()
with open(prune, 'r') as f:
reader = csv.reader(f)
for row in reader:
gameid = int(row[0])
matches = [x for x in top_games if x[0] == gameid]
if len(matches) == 1:
match = matches[0]
matched_game = (row[1], match[0], match[1], match[2], match[3])
elif len(matches) == 0:
matched_game = (row[1], gameid, 0, 0, 0)
else:
print 'ERROR'
return
pruned_games.append(matched_game)
pruned_games.sort(key=lambda x: x[3], reverse=True)
for idx, game in enumerate(pruned_games):
label = str(idx + 1) + '.'
print label, game[0], game[2], game[3], game[4]
return
else:
top_games = filter(lambda x: x[1] >= 0.1 * member_count, top_games)
# Get the top 50
top50 = list()
top_games.sort(key=lambda x: x[2], reverse=True)
count_of_printed = 0
for game in top_games:
game_info = get_game_info(game[0], bgg)
if not game_info.expansion:
count_of_printed += 1
top50.append((game_info.name, game[0], game[1], game[2], game[3]))
if count_of_printed > 49:
break
# Get the bottom 10
bottom10 = list()
top_games.sort(key=lambda x: x[2])
count_of_printed = 0
for game in top_games:
game_info = get_game_info(game[0], bgg)
if not game_info.expansion:
count_of_printed += 1
bottom10.append((game_info.name, game[0], game[1], game[2], game[3]))
if count_of_printed > 9:
break
# Get the most variable
variable10 = list()
top_games.sort(key=lambda x: x[3], reverse=True)
count_of_printed = 0
for game in top_games:
game_info = get_game_info(game[0], bgg)
if not game_info.expansion:
count_of_printed += 1
variable10.append((game_info.name, game[0], game[1], game[2], game[3]))
if count_of_printed > 9:
break
# Get the most rated
most10 = list()
top_games.sort(key=lambda x: x[1], reverse=True)
count_of_printed = 0
for game in top_games:
game_info = get_game_info(game[0], bgg)
if not game_info.expansion:
count_of_printed += 1
most10.append((game_info.name, game[0], game[1], game[2], game[3]))
if count_of_printed > 9:
break
fi = open('lists_' + filename_tag + '.json', 'w')
lists_dict = dict()
lists_dict['top50'] = top50
lists_dict['bottom10'] = bottom10
lists_dict['variable10'] = variable10
lists_dict['most10'] = most10
json.dump(lists_dict, fi)
print 'Finished'
from boardgamegeek import BGGClient
bgg = BGGClient()
my_col = bgg.collection('DarkyLondon', exclude_subtype='boardgameexpansion')
games = my_col.items
def get_game_dict():
'''
Returns a dictionary mapping games to ids.
'''
game_dict = {}
for g in games:
if g.id not in game_dict:
game_dict[g.id] = g.name
return game_dict
###Anzahl und Liste der gespielten Liste
from boardgamegeek import BGGClient
bgg = BGGClient()
plays = bgg.plays(name="schrobe")
print("Anzahl gespielter Spiele: %d" % (len(plays)))
l_games_played = []
for session in plays._plays:
l_games_played.append(session.game_name)
###Top 100 Liste, webscraped
from urllib2 import urlopen
from bs4 import BeautifulSoup
quote_page = 'https://www.boardgamegeek.com/browse/boardgame'
page = urlopen(quote_page)
soup = BeautifulSoup(page, 'html.parser')
#print(soup)
games_top_100 = []
for i in range(1,100):
games_top_100.append(soup.find('div', attrs={'id': 'results_objectname'+str(i)}).get_text())
print(games_top_100)