def check_giveaways_valid(group_webpage, start_date=None, min_days=0, min_entries=1,
min_value=0.0, min_num_of_reviews=0, min_score=0,
alt_min_value=0.0, alt_min_num_of_reviews=0, alt_min_score=0,
alt2_min_value=0.0, alt2_min_num_of_reviews=0, alt2_min_score=0, free_group_only=False):
group = MySqlConnector.load_group(group_webpage, limit_by_time=start_date, starts_after_str=start_date)
if not group:
return None
free_games_list = None
if free_group_only:
free_games_list = BarterVGScrapingUtils.get_free_games_list()
users = group.group_users.keys()
invalid_giveaways = dict()
free_games = dict()
games = dict()
for group_giveaway in group.group_giveaways.values():
creator = group_giveaway.creator
if creator in users:
game_name = group_giveaway.game_name
game_data = MySqlConnector.get_game_data(game_name)
check_game_data(game_data, game_name)
if not game_is_according_to_requirements(game_data, min_value, min_num_of_reviews, min_score, alt_min_value, alt_min_num_of_reviews, alt_min_score, alt2_min_value, alt2_min_num_of_reviews, alt2_min_score)\
or (group_giveaway.end_time - group_giveaway.start_time).days < min_days\
or (datetime.datetime.now() > group_giveaway.end_time and len(group_giveaway.entries) < min_entries):
if creator not in invalid_giveaways:
invalid_giveaways[creator] = set()
invalid_giveaways[creator].add(group_giveaway)
games[game_name] = game_data
if free_group_only and group_giveaway.game_name in free_games_list and len(group_giveaway.groups) > 1:
if creator not in free_games:
free_games[creator] = set()
free_games[creator].add(group_giveaway)
games[game_name] = game_data
return invalid_giveaways, free_games, games
def group_users_check_rules(group_webpage, check_nonactivated=False, check_multiple_wins=False, check_real_cv_ratio=False, check_steamgifts_ratio=False, check_level=False, min_level=0, check_steamrep=False):
group = MySqlConnector.load_group(group_webpage, load_giveaway_data=False)
if not group:
return None
result=dict()
for user_name in group.group_users.keys():
nonactivated, multiple_wins, real_cv_ratio, steamgifts_ratio, level, steamrep = user_check_rules(user_name, check_nonactivated, check_multiple_wins, check_real_cv_ratio, check_steamgifts_ratio, check_level, min_level, check_steamrep)
result[user_name] = (nonactivated, multiple_wins, real_cv_ratio, steamgifts_ratio, level, steamrep)
return result
def update_existing_group(group_webpage, start_date=None, end_date=None, force_full_run=False, update_games=False):
group = MySqlConnector.load_group(group_webpage, fetch_not_started_giveaways=True)
if not group:
return None
cookies = common_cookies
if group.cookies:
cookies = group.cookies
games = update_group_data(group_webpage, cookies, group, start_date=start_date, end_date=end_date, force_full_run=force_full_run)
if update_games:
update_games_data(games, update_value=True)
def get_group_by_year_month(group_webpage, year_month):
split_date = year_month.split('-')
year = int(split_date[0])
month = int(split_date[1])
month_start = year_month + '-01'
if month == 12:
month_end = str(year + 1) + '-01-01'
else:
month_end = str(year) + '-' + get_next_month(month) + '-01'
group = MySqlConnector.load_group(group_webpage, limit_by_time=True, ends_after_str=month_start, ends_before_str=month_end)
return group
def get_game_giveaways(group_webpage, game_name, start_time):
group = MySqlConnector.load_group(group_webpage, limit_by_time=start_time, starts_after_str=start_time)
if not group:
return None
group_users = group.group_users.keys()
all_game_giveaways = dict()
for group_giveaway in group.group_giveaways.values():
group_game_name = group_giveaway.game_name.decode('utf-8')
if group_game_name.lower() in game_name.lower() or game_name.lower() in group_game_name.lower():
giveaway_entries = group_giveaway.entries.values()
all_game_giveaways[group_giveaway] = len([entry for entry in giveaway_entries if entry.user_name in group_users])
return all_game_giveaways
def get_user_all_giveways(group_webpage, user, start_time):
group = MySqlConnector.load_group(group_webpage, load_users_data=False, limit_by_time=start_time, starts_after_str=start_time)
if not group:
return None
created_giveaways=[]
entered_giveaways=[]
games=dict()
for group_giveaway in group.group_giveaways.values():
if not start_time or start_time <= group_giveaway.end_time.strftime('%Y-%m-%d %H:%M:%S'):
game_name = group_giveaway.game_name
if group_giveaway.creator == user:
created_giveaways.append(group_giveaway)
games[game_name] = MySqlConnector.get_game_data(game_name)
elif user in group_giveaway.entries.keys():
entered_giveaways.append(group_giveaway)
games[game_name] = MySqlConnector.get_game_data(game_name)
return created_giveaways, entered_giveaways, games
def check_user_first_giveaway(group_webpage,
users,
addition_date,
days_to_create_ga=0,
min_ga_time=0,
min_entries=1,
min_value=0.0,
min_num_of_reviews=0,
min_score=0,
alt_min_value=0.0,
alt_min_num_of_reviews=0,
alt_min_score=0,
alt2_min_game_value=0,
alt2_min_steam_num_of_reviews=0,
alt2_min_steam_score=0,
check_entered_giveaways=False):
group = MySqlConnector.load_group(group_webpage,
load_users_data=False,
limit_by_time=True,
ends_after_str=addition_date)
if not group:
return None
users_list = users.split(',')
user_added_time = datetime.datetime.strptime(addition_date, '%Y-%m-%d')
user_end_time = dict()
user_first_giveaway = dict()
succesfully_ended = dict()
user_no_giveaway = set()
for group_giveaway in group.group_giveaways.values():
game_name = group_giveaway.game_name
user_name = group_giveaway.creator
if (user_name in users_list and len(group_giveaway.groups) == 1 and
(days_to_create_ga == 0 or group_giveaway.start_time <=
user_added_time + datetime.timedelta(days=days_to_create_ga + 1))
and (min_ga_time == 0 or
(min_ga_time > 0
and group_giveaway.end_time - group_giveaway.start_time
>= datetime.timedelta(days=min_ga_time - 1)))):
game_data = MySqlConnector.get_game_data(game_name)
check_game_data(game_data, game_name)
if game_is_according_to_requirements(
game_data, min_value, min_num_of_reviews, min_score,
alt_min_value, alt_min_num_of_reviews, alt_min_score,
alt2_min_game_value, alt2_min_steam_num_of_reviews,
alt2_min_steam_score):
if user_name not in user_end_time or group_giveaway.end_time < user_end_time[
user_name]:
user_end_time[user_name] = group_giveaway.end_time
if user_name not in user_first_giveaway:
user_first_giveaway[user_name] = set()
user_first_giveaway[user_name].add((group_giveaway, game_data))
if datetime.datetime.now(
) > group_giveaway.end_time and group_giveaway.has_winners(
) and len(group_giveaway.entries) > min_entries:
if user_name not in succesfully_ended:
succesfully_ended[user_name] = set()
succesfully_ended[user_name].add(group_giveaway.link)
for user in users_list:
if user not in user_end_time.keys():
user_no_giveaway.add(user)
partial_group_webpage = group_webpage.split(
SteamGiftsConsts.STEAMGIFTS_LINK)[1]
user_entered_giveaway = dict()
for group_giveaway in group.group_giveaways.values():
if check_entered_giveaways and user_added_time < group_giveaway.end_time:
for user in users_list:
if user in group_giveaway.entries \
and group_giveaway.entries[user].entry_time >= user_added_time \
and (user not in user_end_time or group_giveaway.entries[user].entry_time < user_end_time[user])\
and (len(group_giveaway.groups) == 1 or not SteamGiftsScrapingUtils.is_user_in_group(user, filter(lambda x: x != partial_group_webpage, group_giveaway.groups))):
#TODO: Add "Whitelist detected" warning
if user not in user_entered_giveaway:
user_entered_giveaway[user] = set()
user_entered_giveaway[user].add(group_giveaway)
time_to_create_over = False
if days_to_create_ga > 0 and datetime.date.today() > user_added_time.date(
) + datetime.timedelta(days=days_to_create_ga + 1):
time_to_create_over = True
return user_first_giveaway, succesfully_ended, user_no_giveaway, user_entered_giveaway, time_to_create_over
def get_group_summary(group_webpage, start_time):
group = MySqlConnector.load_group(group_webpage,
limit_by_time=start_time,
starts_after_str=start_time)
if not group:
return None
all_group_users = group.group_users.keys()
users_created = dict()
users_entered = dict()
users_won = dict()
group_games_count = 0
group_games_value = 0.0
group_games_without_data = 0
group_games_total_score = 0.0
group_games_total_num_of_reviews = 0.0
group_total_entered = 0.0
group_total_won = 0.0
for group_giveaway in group.group_giveaways.values():
creator = group_giveaway.creator
# Go over all giveaways started after "addition_date"
if not start_time or start_time <= group_giveaway.end_time.strftime(
'%Y-%m-%d %H:%M:%S'):
group_games_count += 1
try:
game_data = MySqlConnector.get_game_data(
group_giveaway.game_name)
except Exception as e:
LogUtils.log_error(
u'Crashed while trying to load game data of ' +
group_giveaway.game_name.decode('utf-8') + u'. Reason: ' +
str(e))
traceback.print_exc()
continue
if not game_data:
LogUtils.log_error(u'Could not load game data: ' +
group_giveaway.game_name.decode('utf-8'))
score = -1
num_of_reviews = -1
value = -1
if game_data:
value = game_data.value
group_games_value += value
score = game_data.steam_score
num_of_reviews = game_data.num_of_reviews
game_data_available = score != -1 and num_of_reviews != -1
if game_data_available:
group_games_total_score += score
group_games_total_num_of_reviews += num_of_reviews
else:
group_games_without_data += 1
# Number of created GAs, Total Value, Number of GAs with data, Total Score, Total NumOfReviews
if creator in all_group_users:
if creator not in users_created:
users_created[creator] = [0, 0, 0, 0, 0]
users_created[creator][0] += 1
if value != -1:
users_created[creator][1] += value
if game_data_available:
users_created[creator][2] += 1
if score != -1:
users_created[creator][3] += score
if num_of_reviews != -1:
users_created[creator][4] += num_of_reviews
giveaway_entries = group_giveaway.entries.values()
num_of_giveaway_entries = len(giveaway_entries)
for entry in giveaway_entries:
user_name = entry.user_name
if user_name not in all_group_users:
continue
group_total_entered += 1
# Number of entered GAs, Number of Unique GAs, Total Value, Number of GAs with data, Total Score, Total NumOfReviews
if user_name not in users_entered:
users_entered[user_name] = [0, 0, 0, 0, 0, 0, 0]
users_entered[user_name][0] += 1
users_entered[user_name][1] += (1.0 / num_of_giveaway_entries)
if len(group_giveaway.groups) == 1:
users_entered[user_name][2] += 1
if value != -1:
users_entered[user_name][3] += value
if game_data_available:
users_entered[user_name][4] += 1
if score != -1:
users_entered[user_name][5] += score
if num_of_reviews != -1:
users_entered[user_name][6] += num_of_reviews
if entry.winner:
group_total_won += 1
# Number of won GAs, Total Value, Number of GAs with data, Total Score, Total NumOfReviews
if user_name not in users_won:
users_won[user_name] = [0, 0, 0, 0, 0]
users_won[user_name][0] += 1
if value != -1:
users_won[user_name][1] += value
if game_data_available:
users_won[user_name][2] += 1
if score != -1:
users_won[user_name][3] += score
if num_of_reviews != -1:
users_won[user_name][4] += num_of_reviews
group_average_game_value = group_games_value / group_games_count
group_average_game_score = group_games_total_score / (
group_games_count - group_games_without_data)
group_average_game_num_of_reviews = group_games_total_num_of_reviews / (
group_games_count - group_games_without_data)
# Total Giveaways Count, Total Games Value, Average games value, Average Game Score, Average Game NumOfReviews, Average number of entered per game, Average number of created per user, Average number of entrered per user, Average number of won per user
total_group_data = (group_games_count, group_games_value,
group_average_game_value, group_average_game_score,
group_average_game_num_of_reviews,
group_total_entered / group_games_count,
float(group_games_count) / len(all_group_users),
group_total_entered / len(all_group_users),
group_total_won / len(all_group_users))
# Number of created GAs, Total Value, Average Value, Average Score, Average NumOfReviews
# Number of entered GAs, Percentage of unique, Average Value, Average Score, Average Num Of Reviews
# If avialable: number of won GAs, Winning percentage, Average Value, Average Score, Average Num Of Reviews
users_data = dict()
for user in users_created.keys():
if user not in users_data:
users_data[user] = [(), (), ()]
# Number of created GAs, Total Value, Number of GAs with data, Total Score, Total NumOfReviews
user_data = users_created[user]
# Number of created GAs, Total Value, Average Value, Average Score, Average NumOfReviews
if user_data[2] > 0:
users_data[user][0] = (user_data[0], user_data[1],
float(user_data[1]) / user_data[0],
float(user_data[3]) / user_data[2],
float(user_data[4]) / user_data[2])
else:
users_data[user][0] = (user_data[0], user_data[1],
float(user_data[1]) / user_data[0], 0, 0)
for user in users_entered.keys():
if user not in users_data:
users_data[user] = [(), (), ()]
# Number of entered GAs, Total number of entries in GAs he entered, Number of Shared GAs, Total Value,
# Number of GAs with data, Total Score, Total NumOfReviews
user_data = users_entered[user]
# Number of entered GAs, Probability of winning, Percentage of unique, Total Value, Average Value, Average Score, Average Num Of Reviews
if user_data[3] > 0 and user_data[4] > 0:
users_data[user][1] = (user_data[0], user_data[1] * 100,
float(user_data[2]) / user_data[0] * 100,
user_data[3],
float(user_data[3]) / user_data[0],
float(user_data[5]) / user_data[4],
float(user_data[6]) / user_data[4])
else:
users_data[user][1] = (user_data[0], user_data[1] * 100,
float(user_data[2]) / user_data[0] * 100,
user_data[3],
float(user_data[3]) / user_data[0], 0, 0)
if user in users_won:
# Number of won GAs, Total Value, Number of GAs with data, Total Score, Total NumOfReviews
user_data = users_won[user]
# Number of won GAs, Winning percentage, Total value, Average Value, Average Score, Average Num Of Reviews
if user_data[2] > 0:
users_data[user][2] = (user_data[0], float(user_data[0]) /
users_data[user][1][0] * 100,
user_data[1],
float(user_data[1]) / user_data[0],
float(user_data[3]) / user_data[2],
float(user_data[4]) / user_data[2])
else:
users_data[user][2] = (user_data[0], float(user_data[0]) /
users_data[user][1][0] * 100,
user_data[1], float(user_data[1]) /
user_data[0], 0, 0)
return total_group_data, users_data