def merge_articles():
"""
Merge cached articles into the working set
"""
load_working_set()
load_registry('Article', 'article_id')
for article_cached in tqdm(TC['Article.article_id'],
'[MERGE] Merging Articles',
bar_format=PROGRESS_FORMAT):
if article_cached.steam_data is None and article_cached.newsapi_data is None:
continue
steam_data = article_cached.steam_data
newsapi_data = article_cached.newsapi_data
title = steam_data[
'title'] if steam_data is not None else newsapi_data['title']
article = WS.build_article(article_cached.article_id, title,
condition(title))
steam.build_article(article, steam_data)
newsapi.build_article(article, newsapi_data)
related_game = WS.games.get(article_cached.game_id)
if related_game is not None:
xappend(related_game.articles, article)
for developer in related_game.developers:
xappend(developer.articles, article)
def link_articles():
"""
Compute Game-Article links.
Complexity: O(N^2)
"""
load_working_set()
games = {condition_heavy(game.c_name): game for game in WS.games.values()}
for article in tqdm(WS.articles.values(), '[LINK] Linking Articles'):
content = condition_heavy(article.introduction)
for name in games.keys():
if name in content:
# Link the models
xappend(article.games, games[name])
def link_videos():
"""
Compute Game-Video links.
Complexity: O(N^2)
"""
load_working_set()
videos = {
condition_heavy(video.name + video.description): video
for video in WS.videos.values()
}
for game in tqdm(WS.games.values(), '[LINK] Linking Videos'):
name = condition_heavy(game.c_name)
for text in videos.keys():
if name in text:
# Link the models
xappend(game.videos, videos[text])
def link_developers():
"""
Compute Game-Developer links according to IGDB ID for IGDB games
"""
load_working_set()
load_registry('Developer', 'igdb_id')
for developer in tqdm(WS.developers.values(), '[LINK] Linking Developers',
bar_format=PROGRESS_FORMAT):
dev_json = TC['Developer.igdb_id'].get(developer.igdb_id).igdb_data
for igdb_id in chain(dev_json.get('published', []), dev_json.get('developed', [])):
game = WS.games_igdb.get(igdb_id)
if game is not None:
# Set the primary developer to the first one
if game.developer is None:
game.developer = developer.name
# Link the models
xappend(developer.games, game)
def merge_tweets():
"""
Merge cached tweets into the working set
"""
load_working_set()
load_registry('Tweet', 'tweet_id')
for tweet_cached in tqdm(TC['Tweet.tweet_id'],
'[MERGE] Merging Tweets',
bar_format=PROGRESS_FORMAT):
if tweet_cached.twitter_data is None:
continue
tweet_data = tweet_cached.twitter_data
tweet = WS.build_tweet(tweet_cached.tweet_id,
tweet_data['user']['name'], tweet_data['text'])
twitter.build_tweet(tweet, tweet_data)
related_game = WS.games.get(tweet_cached.game_id)
if related_game is not None and len(related_game.tweets) < 75:
xappend(related_game.tweets, tweet)
unload_registry('Tweet', 'tweet_id')
def merge_videos():
"""
Merge cached videos into the working set
"""
load_working_set()
load_registry('Video', 'video_id')
for video_cached in tqdm(TC['Video.video_id'],
'[MERGE] Merging Videos',
bar_format=PROGRESS_FORMAT):
if video_cached.youtube_data is None:
continue
youtube_data = video_cached.youtube_data
video = WS.build_video(video_cached.video_id,
youtube_data['snippet']['title'])
google.build_video(video, video_cached.youtube_data)
related_game = WS.games.get(video_cached.game_id)
if related_game is not None:
xappend(related_game.videos, video)
unload_registry('Video', 'video_id')
def test_xappend(self):
video = Video()
game = Game()
self.assertFalse(video in game.videos)
xappend(game.videos, video)
self.assertTrue(video in game.videos)
dev = Developer()
article = Article()
xappend(dev.articles, article)
self.assertTrue(article in dev.articles)
self.assertTrue(dev in article.developers)
len1 = len(dev.articles)
len2 = len(article.developers)
xappend(dev.articles, article)
self.assertTrue(article in dev.articles)
self.assertTrue(dev in article.developers)
self.assertEqual(len1, len(dev.articles))
self.assertEqual(len2, len(article.developers))
def build_game(game, game_json):
"""
Build a Game object from the raw data
"""
if game is None or game_json is None:
return
# IGDB ID
if game.igdb_id is None:
game.igdb_id = int(game_json['id'])
# IGDB link
if game.igdb_link is None and 'url' in game_json:
game.igdb_link = game_json['url']
# Title
if game.name is None:
game.name = game_json['name']
game.c_name = condition(game.name)
# Genre
for numeric_genre in game_json.get('genres', []):
xappend(game.genres, WS.genres[numeric_genre])
# Platform
for numeric_platform in game_json.get('platforms', []):
xappend(game.platforms, WS.platforms[numeric_platform])
# Summary
if game.summary is None and 'summary' in game_json:
game.summary = game_json['summary']
# Steam ID
if game.steam_id is None and 'external' in game_json \
and 'steam' in game_json['external']:
game.steam_id = int(game_json['external']['steam'])
# Release date
if game.release is None and 'first_release_date' in game_json:
game.release = date.fromtimestamp(
game_json['first_release_date'] // 1000)
# Screenshots
if game.steam_id is None or game.screenshots is None:
for screenshot in game_json.get('screenshots', []):
if game.screenshots is None:
game.screenshots = []
xappend(game.screenshots, {
'url': screenshot['url'][2:].replace("t_thumb", "t_original")})
# Cover
if game.cover is None and 'cover' in game_json:
game.cover = game_json['cover']['url'][2:].replace(
"t_thumb", "t_original")
# ESRB rating
if game.esrb is None and 'esrb' in game_json:
game.esrb = game_json['esrb']['rating']
# Website
if game.website is None and 'websites' in game_json:
for site_json in game_json['websites']:
if 'category' in site_json and site_json['category'] == 1:
game.website = site_json['url']
break