def update_all_db_users_data():
#Load list of all groups & users from DB
groups = get_groups_with_users()
all_users = MySqlConnector.get_all_users()
#Generate users list from all groups
all_group_users = set()
for group, group_data in groups.items():
all_group_users.update(group_data.group_users)
group_users = MySqlConnector.get_users_by_names(all_group_users)
#Go over list, check if any changed
changed_users = []
deleted_users = [
user for user in all_users if user.user_name not in all_group_users
]
for user in group_users.values():
new_user = GroupUser(user.user_name)
user_data_fetched = SteamGiftsScrapingUtils.update_user_additional_data(
new_user)
if new_user.steam_id:
user_data_fetched &= SteamScrapingUtils.update_user_additional_data(
new_user)
if not user_data_fetched:
deleted_users.append(user)
elif not user.equals(new_user):
changed_users.append(new_user)
# Save changed users to the DB
if changed_users:
MySqlConnector.update_existing_users(changed_users)
# Delete from DB users no longer on SteamGifts
if deleted_users:
MySqlConnector.delete_users(deleted_users)
def update_game_data(game):
if game.game_link.startswith('http:'):
# Converting all HTTP links into HTTPS
game.game_link = game.game_link.replace('http', 'https')
game_link = game.game_link
game_name = game.game_name
try:
steam_score = 0
num_of_reviews = 0
if game_link.startswith(SteamConsts.STEAM_GAME_LINK):
try:
steam_score, num_of_reviews = SteamScrapingUtils.get_game_additional_data(game_name, game_link)
except:
LogUtils.log_error('Error extracting Steam data for game: ' + game_name + ' at link: ' + game_link)
try:
steam_score, num_of_reviews = SteamDBScrapingUtils.get_game_additional_data(game_name, game_link)
except:
LogUtils.log_error('Error extracting SteamDB data for game: ' + game_name + ' at link: ' + game_link)
game.steam_score = steam_score
game.num_of_reviews = num_of_reviews
if (not steam_score and not num_of_reviews) or (steam_score == 0 and num_of_reviews == 0):
LogUtils.log_error('Unable to extract Steam Score & Number of reviews for: ' + game_name)
elif game_link.startswith(SteamConsts.STEAM_PACKAGE_LINK):
chosen_score = -1
chosen_num_of_reviews = -1
package_games = SteamScrapingUtils.get_games_from_package(game_name, game_link)
i = 0
for package_url in package_games:
tmp_game_name = game_name + ' - package #' + str(i)
try:
steam_score, num_of_reviews = SteamScrapingUtils.get_game_additional_data(tmp_game_name, package_url)
except:
try:
steam_score, num_of_reviews = SteamDBScrapingUtils.get_game_additional_data(tmp_game_name, package_url)
except:
LogUtils.log_warning('Unable to extract Steam Score & Number of reviews for package: ' + package_url)
if num_of_reviews > chosen_num_of_reviews:
chosen_score = steam_score
chosen_num_of_reviews = num_of_reviews
i += 1
game.steam_score = chosen_score
game.num_of_reviews = chosen_num_of_reviews
else:
LogUtils.log_error('Don\'t know how to handle game: ' + game_name + ' at ' + game_link)
except Exception as e:
LogUtils.log_error('Cannot add additional data for game: ' + game_name + ' ERROR: ' + str(e))
traceback.print_exc()
def update_group_data(group_webpage, cookies, group, force_full_run=False, start_date=None, end_date=None):
group_users = SteamGiftsScrapingUtils.get_group_users(group_webpage)
if not group_users:
LogUtils.log_error("group_users is empty")
return dict()
existing_users = MySqlConnector.check_existing_users(group_users.keys())
for group_user in group_users.values():
if group_user.user_name not in existing_users:
try:
SteamGiftsScrapingUtils.update_user_additional_data(group_user)
SteamScrapingUtils.update_user_additional_data(group_user)
except Exception as e:
LogUtils.log_error('Cannot add additional data for user: '******' ERROR: ' + str(e))
traceback.print_exc()
group_giveaways, ignored_giveaways, games, reached_threshold = SteamGiftsScrapingUtils.get_group_giveaways(group_webpage, cookies, group.group_giveaways, force_full_run=force_full_run, start_date=start_date, end_date=end_date)
if not reached_threshold:
remove_deleted_giveaways(cookies, group, group_giveaways, ignored_giveaways)
MySqlConnector.save_group(group_webpage, Group(group_users, group_giveaways, group_webpage=group_webpage, cookies=cookies, group_name=group.group_name), existing_users, group)
return games
def load_user(group_user, user_name):
if not group_user:
group_user = GroupUser(user_name)
SteamGiftsScrapingUtils.update_user_additional_data(group_user)
SteamScrapingUtils.update_user_additional_data(group_user)
return group_user