def test():
# from BusinessLogic.Utils import WebUtils
# group = MySqlConnector.load_group('https://www.steamgifts.com/group/6HSPr/qgg-group')
# print WebUtils.get_page_content('https://www.steamgifts.com/giveaway/JtzUN/broken-sword-5-the-serpents-curse', cookies=group.cookies)
# group = add_new_group(group_webpage, '')
# MySqlConnector.save_group(group_webpage, group)
# group = MySqlConnector.load_group(group_webpage)
# SteamScrapingUtils.get_steam_user_name_from_steam_id('76561198018110309')
# for group_user in group.group_users.values():
# print '\nUser: '******'User ' + group_user.user_name + ' has negative global gifts ratio'
# game = GameData('Conarium', 'https://store.steampowered.com/app/313780/Conarium/', 20)
# update_game_data(game)
# MySqlConnector.update_existing_games([game])
#
# MySqlConnector.get_game_data('Conarium')
# try:
# SteamScrapingUtils.get_game_additional_data(game.game_name, game.game_link)
# except:
# user = GroupUser('a404381120')
# SteamGiftsScrapingUtils.update_user_additional_data(user)
# MySqlConnector.update_existing_users([user])
# MySqlConnector.get_users_by_names(['a404381120'])
# pass
# free_games = BarterVGScrapingUtils.get_free_games_list()
SteamGiftsScrapingUtils.get_group_giveaways(
"https://www.steamgifts.com/group/6HSPr/qgg-group",
"__gads=ID=5dc87d4015cff31b:T=1486315976:S=ALNI_MbsjCesUHx_EnIbaUSsE9BG0dIYng;_ga=GA1.2.1062554528.1469041573; _gid=GA1.2.723800780.1538165878;PHPSESSID=fpp6kcu05kte5pp62daalkctsq403uutdo49tt9p2e5kj8n0n449i3majs8ig0f8cdg2011tch8mm43ltf8brfpu52tgpilau2cik12;_gat=1",
dict())
pass
def user_check_rules(user_name,
check_nonactivated=False,
check_multiple_wins=False,
check_real_cv_ratio=False,
check_steamgifts_ratio=False,
check_level=False,
min_level=0,
check_steamrep=False):
nonactivated = False
multiple_wins = False
real_cv_ratio = False
steamgifts_ratio = None
level = None
steamrep = None
if check_nonactivated:
LogUtils.log_info('Checking user ' + user_name +
' for non-activated games')
if SGToolsScrapingUtils.check_nonactivated(user_name):
nonactivated = True
if check_multiple_wins:
LogUtils.log_info('Checking user ' + user_name + ' for multiple wins')
if SGToolsScrapingUtils.check_multiple_wins(user_name):
multiple_wins = True
if check_real_cv_ratio:
LogUtils.log_info('Checking user ' + user_name + ' for Real CV ratio')
if SGToolsScrapingUtils.check_real_cv_RATIO(user_name):
real_cv_ratio = True
if check_steamgifts_ratio or (check_level and min_level > 0):
global_won, global_sent, user_level = SteamGiftsScrapingUtils.get_user_contribution_data(
user_name)
if check_steamgifts_ratio:
LogUtils.log_info('Checking user ' + user_name +
' for SteamGifts Global ratio')
if global_won > global_sent:
steamgifts_ratio = (global_won, global_sent)
if check_level and min_level > 0:
LogUtils.log_info('Checking user ' + user_name +
' for minimal level')
if user_level < float(min_level):
level = min_level
if check_steamrep:
LogUtils.log_info('Checking user ' + user_name + ' in SteamRep')
user_steam_id = SteamGiftsScrapingUtils.get_user_steam_id(user_name)
if user_steam_id and not SteamRepScrapingUtils.check_user_not_public_or_banned(
user_steam_id):
steamrep = SteamRepConsts.get_steamrep_link(user_steam_id)
return nonactivated, multiple_wins, real_cv_ratio, steamgifts_ratio, level, steamrep
def update_all_db_users_data():
#Load list of all groups & users from DB
groups = get_groups_with_users()
all_users = MySqlConnector.get_all_users()
#Generate users list from all groups
all_group_users = set()
for group, group_data in groups.items():
all_group_users.update(group_data.group_users)
group_users = MySqlConnector.get_users_by_names(all_group_users)
#Go over list, check if any changed
changed_users = []
deleted_users = [
user for user in all_users if user.user_name not in all_group_users
]
for user in group_users.values():
new_user = GroupUser(user.user_name)
user_data_fetched = SteamGiftsScrapingUtils.update_user_additional_data(
new_user)
if new_user.steam_id:
user_data_fetched &= SteamScrapingUtils.update_user_additional_data(
new_user)
if not user_data_fetched:
deleted_users.append(user)
elif not user.equals(new_user):
changed_users.append(new_user)
# Save changed users to the DB
if changed_users:
MySqlConnector.update_existing_users(changed_users)
# Delete from DB users no longer on SteamGifts
if deleted_users:
MySqlConnector.delete_users(deleted_users)
def add_new_group(group_webpage, cookies, start_date=None):
group_name = SteamGiftsScrapingUtils.get_group_name(group_webpage)
if not cookies:
cookies = common_cookies
games = update_group_data(group_webpage,
cookies,
Group(group_name=group_name,
group_webpage=group_webpage,
cookies=cookies),
force_full_run=True,
start_date=start_date)
update_games_data(games)
def update_group_data(group_webpage, cookies, group, force_full_run=False, start_date=None, end_date=None):
group_users = SteamGiftsScrapingUtils.get_group_users(group_webpage)
if not group_users:
LogUtils.log_error("group_users is empty")
return dict()
existing_users = MySqlConnector.check_existing_users(group_users.keys())
for group_user in group_users.values():
if group_user.user_name not in existing_users:
try:
SteamGiftsScrapingUtils.update_user_additional_data(group_user)
SteamScrapingUtils.update_user_additional_data(group_user)
except Exception as e:
LogUtils.log_error('Cannot add additional data for user: '******' ERROR: ' + str(e))
traceback.print_exc()
group_giveaways, ignored_giveaways, games, reached_threshold = SteamGiftsScrapingUtils.get_group_giveaways(group_webpage, cookies, group.group_giveaways, force_full_run=force_full_run, start_date=start_date, end_date=end_date)
if not reached_threshold:
remove_deleted_giveaways(cookies, group, group_giveaways, ignored_giveaways)
MySqlConnector.save_group(group_webpage, Group(group_users, group_giveaways, group_webpage=group_webpage, cookies=cookies, group_name=group.group_name), existing_users, group)
return games
def remove_deleted_giveaways(cookies, group, updated_group_giveaways,
ignored_group_giveaways):
# If any existing GA is missing from newly parsed data - remove it from group giveaways.
giveaways_sorted_by_end_time = sorted(filter(
lambda x: x.end_time, updated_group_giveaways.values()),
key=lambda x: x.end_time)
if not giveaways_sorted_by_end_time:
return
earliest_giveaway_end_time = giveaways_sorted_by_end_time[0].end_time
for giveaway in sorted(filter(lambda x: x.end_time,
group.group_giveaways.values()),
key=lambda x: x.end_time,
reverse=True):
if giveaway.end_time < earliest_giveaway_end_time:
break
if giveaway.link not in updated_group_giveaways and giveaway.link not in ignored_group_giveaways and not giveaway.has_winners(
) and SteamGiftsScrapingUtils.is_giveaway_deleted(
giveaway.link, cookies):
LogUtils.log_info('Removing deleted giveaway: ' + giveaway.link)
group.group_giveaways.pop(giveaway.link, None)
def lazy_add_group(group_webpage, cookies):
if not cookies:
cookies = ''
group_name = SteamGiftsScrapingUtils.get_group_name(group_webpage, cookies)
MySqlConnector.save_empty_group(group_name, group_webpage, cookies)
return group_name
def load_user(group_user, user_name):
if not group_user:
group_user = GroupUser(user_name)
SteamGiftsScrapingUtils.update_user_additional_data(group_user)
SteamScrapingUtils.update_user_additional_data(group_user)
return group_user
def check_user_first_giveaway(group_webpage,
users,
addition_date,
days_to_create_ga=0,
min_ga_time=0,
min_entries=1,
min_value=0.0,
min_num_of_reviews=0,
min_score=0,
alt_min_value=0.0,
alt_min_num_of_reviews=0,
alt_min_score=0,
alt2_min_game_value=0,
alt2_min_steam_num_of_reviews=0,
alt2_min_steam_score=0,
check_entered_giveaways=False):
group = MySqlConnector.load_group(group_webpage,
load_users_data=False,
limit_by_time=True,
ends_after_str=addition_date)
if not group:
return None
users_list = users.split(',')
user_added_time = datetime.datetime.strptime(addition_date, '%Y-%m-%d')
user_end_time = dict()
user_first_giveaway = dict()
succesfully_ended = dict()
user_no_giveaway = set()
for group_giveaway in group.group_giveaways.values():
game_name = group_giveaway.game_name
user_name = group_giveaway.creator
if (user_name in users_list and len(group_giveaway.groups) == 1 and
(days_to_create_ga == 0 or group_giveaway.start_time <=
user_added_time + datetime.timedelta(days=days_to_create_ga + 1))
and (min_ga_time == 0 or
(min_ga_time > 0
and group_giveaway.end_time - group_giveaway.start_time
>= datetime.timedelta(days=min_ga_time - 1)))):
game_data = MySqlConnector.get_game_data(game_name)
check_game_data(game_data, game_name)
if game_is_according_to_requirements(
game_data, min_value, min_num_of_reviews, min_score,
alt_min_value, alt_min_num_of_reviews, alt_min_score,
alt2_min_game_value, alt2_min_steam_num_of_reviews,
alt2_min_steam_score):
if user_name not in user_end_time or group_giveaway.end_time < user_end_time[
user_name]:
user_end_time[user_name] = group_giveaway.end_time
if user_name not in user_first_giveaway:
user_first_giveaway[user_name] = set()
user_first_giveaway[user_name].add((group_giveaway, game_data))
if datetime.datetime.now(
) > group_giveaway.end_time and group_giveaway.has_winners(
) and len(group_giveaway.entries) > min_entries:
if user_name not in succesfully_ended:
succesfully_ended[user_name] = set()
succesfully_ended[user_name].add(group_giveaway.link)
for user in users_list:
if user not in user_end_time.keys():
user_no_giveaway.add(user)
partial_group_webpage = group_webpage.split(
SteamGiftsConsts.STEAMGIFTS_LINK)[1]
user_entered_giveaway = dict()
for group_giveaway in group.group_giveaways.values():
if check_entered_giveaways and user_added_time < group_giveaway.end_time:
for user in users_list:
if user in group_giveaway.entries \
and group_giveaway.entries[user].entry_time >= user_added_time \
and (user not in user_end_time or group_giveaway.entries[user].entry_time < user_end_time[user])\
and (len(group_giveaway.groups) == 1 or not SteamGiftsScrapingUtils.is_user_in_group(user, filter(lambda x: x != partial_group_webpage, group_giveaway.groups))):
#TODO: Add "Whitelist detected" warning
if user not in user_entered_giveaway:
user_entered_giveaway[user] = set()
user_entered_giveaway[user].add(group_giveaway)
time_to_create_over = False
if days_to_create_ga > 0 and datetime.date.today() > user_added_time.date(
) + datetime.timedelta(days=days_to_create_ga + 1):
time_to_create_over = True
return user_first_giveaway, succesfully_ended, user_no_giveaway, user_entered_giveaway, time_to_create_over