def merge_articles():
"""
Merge cached articles into the working set
"""
load_working_set()
load_registry('Article', 'article_id')
for article_cached in tqdm(TC['Article.article_id'],
'[MERGE] Merging Articles',
bar_format=PROGRESS_FORMAT):
if article_cached.steam_data is None and article_cached.newsapi_data is None:
continue
steam_data = article_cached.steam_data
newsapi_data = article_cached.newsapi_data
title = steam_data[
'title'] if steam_data is not None else newsapi_data['title']
article = WS.build_article(article_cached.article_id, title,
condition(title))
steam.build_article(article, steam_data)
newsapi.build_article(article, newsapi_data)
related_game = WS.games.get(article_cached.game_id)
if related_game is not None:
xappend(related_game.articles, article)
for developer in related_game.developers:
xappend(developer.articles, article)
def compute_all():
"""
Compute the VINDEX for all games in the working set
"""
load_working_set()
precompute(WS.games.values())
for game in tqdm(WS.games.values(), '[VINDEX] Computing game vindicies',
bar_format=PROGRESS_FORMAT):
compute(game)
def benckmark():
"""
Compute and print the VINDEX for a few games for easy tweaking
"""
load_working_set()
precompute(WS.games.values())
# Compute benchmark games
for appid in [578080, 570, 359550, 271590, 552520, 477160, 50300]:
game = WS.games_steam[appid]
compute(game)
print("Computed VINDEX: %d for game: %s" % (game.vindex, game.name))
def gather_tweets():
"""
Search for tweets related to games and download them to the cache
"""
load_working_set()
load_registry('Tweet', 'game_id')
generic_gather(rq_tweets, TC['Tweet.game_id'],
'[GATHER] Downloading Tweets', [
game for game in WS.games.values()
if not TC['Tweet.game_id'].exists(game.game_id)
])
def gather_videos():
"""
Download videos from YouTube by game
"""
load_working_set()
load_registry('Video', 'game_id')
generic_gather(rq_videos, TC['Video.game_id'],
'[GATHER] Downloading Videos', [
game for game in WS.games.values()
if not TC['Video.game_id'].exists(game.game_id)
])
def gather_articles():
"""
Search for articles related to games and download them to the cache
"""
load_working_set()
load_registry('Article', 'game_id')
generic_gather(rq_articles, TC['Article.game_id'],
'[GATHER] Downloading Articles', [
game for game in WS.games.values()
if not TC['Article.game_id'].exists(game.game_id)
])
def link_articles():
"""
Compute Game-Article links.
Complexity: O(N^2)
"""
load_working_set()
games = {condition_heavy(game.c_name): game for game in WS.games.values()}
for article in tqdm(WS.articles.values(), '[LINK] Linking Articles'):
content = condition_heavy(article.introduction)
for name in games.keys():
if name in content:
# Link the models
xappend(article.games, games[name])
def link_tweets():
"""
Compute Game-Tweet links.
Complexity: O(N^2)
"""
load_working_set()
tweets = {
condition_heavy(tweet.content): tweet
for tweet in WS.tweets.values()
}
for game in tqdm(WS.games.values(), '[LINK] Linking Tweets'):
name = condition_heavy(game.c_name)
for text in tweets.keys():
if name in text:
# Link the models
xappend(game.tweets, tweets[text])
def clean_tweets():
"""
Remove unwanted tweets from the registry
"""
load_working_set()
load_registry('Tweet', 'tweet_id')
removals = []
for tweet_cached in tqdm(TC['Tweet.tweet_id'],
'[CLEAN] Scanning Tweets',
bar_format=PROGRESS_FORMAT):
game = WS.games.get(tweet_cached.game_id)
if not twitter.validate_tweet(tweet_cached.twitter_data) or not \
twitter.relevant_tweet(game, tweet_cached.twitter_data):
removals.append(tweet_cached)
if input("Delete %d low quality tweets? " % len(removals)) == 'y':
for tweet_cached in removals:
db.session.delete(tweet_cached)
db.session.commit()
def merge_developers():
"""
Merge cached developers into the working set
"""
load_working_set()
load_registry('Developer', 'igdb_id')
for developer_cached in tqdm(TC['Developer.igdb_id'],
'[MERGE] Merging Developers',
bar_format=PROGRESS_FORMAT):
if developer_cached.igdb_data is None:
continue
igdb_data = developer_cached.igdb_data
developer = WS.build_developer(developer_cached.developer_id,
developer_cached.igdb_id,
igdb_data['name'],
condition_developer(igdb_data['name']))
igdb.build_developer(developer, igdb_data)
def link_videos():
"""
Compute Game-Video links.
Complexity: O(N^2)
"""
load_working_set()
videos = {
condition_heavy(video.name + video.description): video
for video in WS.videos.values()
}
for game in tqdm(WS.games.values(), '[LINK] Linking Videos'):
name = condition_heavy(game.c_name)
for text in videos.keys():
if name in text:
# Link the models
xappend(game.videos, videos[text])
def link_developers():
"""
Compute Game-Developer links according to IGDB ID for IGDB games
"""
load_working_set()
load_registry('Developer', 'igdb_id')
for developer in tqdm(WS.developers.values(), '[LINK] Linking Developers',
bar_format=PROGRESS_FORMAT):
dev_json = TC['Developer.igdb_id'].get(developer.igdb_id).igdb_data
for igdb_id in chain(dev_json.get('published', []), dev_json.get('developed', [])):
game = WS.games_igdb.get(igdb_id)
if game is not None:
# Set the primary developer to the first one
if game.developer is None:
game.developer = developer.name
# Link the models
xappend(developer.games, game)
def merge_games():
"""
Merge cached games into the working set
"""
load_working_set()
load_registry('Game', 'game_id')
for game_cached in tqdm(TC['Game.game_id'],
'[MERGE] Merging Games',
bar_format=PROGRESS_FORMAT):
if game_cached.steam_data is None and game_cached.igdb_data is None:
continue
steam_data = game_cached.steam_data
igdb_data = game_cached.igdb_data
name = steam_data['name'] if steam_data is not None else igdb_data[
'name']
game = WS.build_game(game_cached.game_id, game_cached.steam_id,
game_cached.igdb_id, name, condition(name))
steam.build_game(game, steam_data)
igdb.build_game(game, igdb_data)
def merge_tweets():
"""
Merge cached tweets into the working set
"""
load_working_set()
load_registry('Tweet', 'tweet_id')
for tweet_cached in tqdm(TC['Tweet.tweet_id'],
'[MERGE] Merging Tweets',
bar_format=PROGRESS_FORMAT):
if tweet_cached.twitter_data is None:
continue
tweet_data = tweet_cached.twitter_data
tweet = WS.build_tweet(tweet_cached.tweet_id,
tweet_data['user']['name'], tweet_data['text'])
twitter.build_tweet(tweet, tweet_data)
related_game = WS.games.get(tweet_cached.game_id)
if related_game is not None and len(related_game.tweets) < 75:
xappend(related_game.tweets, tweet)
unload_registry('Tweet', 'tweet_id')
def merge_videos():
"""
Merge cached videos into the working set
"""
load_working_set()
load_registry('Video', 'video_id')
for video_cached in tqdm(TC['Video.video_id'],
'[MERGE] Merging Videos',
bar_format=PROGRESS_FORMAT):
if video_cached.youtube_data is None:
continue
youtube_data = video_cached.youtube_data
video = WS.build_video(video_cached.video_id,
youtube_data['snippet']['title'])
google.build_video(video, video_cached.youtube_data)
related_game = WS.games.get(video_cached.game_id)
if related_game is not None:
xappend(related_game.videos, video)
unload_registry('Video', 'video_id')
