def get_user_contribution_data(user_name):
html_content = WebUtils.get_html_page(SteamGiftsConsts.get_user_link(user_name))
if html_content is None:
LogUtils.log_error('Cannot update additional data for user: '******'.//div[@class="sidebar__shortcut-inner-wrap"]/a/@href')
if not steam_user:
LogUtils.log_error('Cannot update non-existent user: '******'.//div[@class="featured__table__row"]')
for row_content in all_rows:
row_title = WebUtils.get_item_by_xpath(row_content, u'.//div[@class="featured__table__row__left"]/text()')
if row_title == u'Gifts Won':
global_won = StringUtils.normalize_int(WebUtils.get_item_by_xpath(row_content, u'.//div[@class="featured__table__row__right"]/span/span/a/text()'))
elif row_title == u'Gifts Sent':
global_sent = StringUtils.normalize_int(WebUtils.get_item_by_xpath(row_content, u'.//div[@class=" featured__table__row__right"]/span/span/a/text()'))
elif row_title == u'Contributor Level':
user_level_item = WebUtils.get_item_by_xpath(row_content, u'.//div[@class="featured__table__row__right"]/span/@data-ui-tooltip')
level = StringUtils.normalize_float(user_level_item.split('name" : "')[2].split('", "color')[0])
if global_won or global_sent or level:
return global_won, global_sent, level
return None
def check_real_cv_RATIO(user):
sent_html_content = WebUtils.get_html_page(
SGToolsConsts.SGTOOLS_CHECK_SENT_LINK + user)
sent_value = WebUtils.get_item_by_xpath(
sent_html_content,
u'.//div[@class="total"]/h1/text()').replace('$', '')
won_html_content = WebUtils.get_html_page(
SGToolsConsts.SGTOOLS_CHECK_WON_LINK + user)
won_value = WebUtils.get_item_by_xpath(
won_html_content,
u'.//div[@class="total"]/h1/text()').replace('$', '')
return StringUtils.normalize_float(
won_value) > StringUtils.normalize_float(sent_value)
def get_game_additional_data(game_name, game_link):
LogUtils.log_info('Processing game ' + game_name)
steam_score = 0
num_of_reviews = 0
html_content = WebUtils.get_html_page(game_link, "birthtime=-7199; lastagecheckage=1-January-1970; mature_content=1;")
base_game_link = WebUtils.get_item_by_xpath(html_content, u'.//div[@class="glance_details"]/a/@href')
if base_game_link is not None:
# If this is DLC - get additional data according to base game
html_content = WebUtils.get_html_page(base_game_link, "birthtime=-7199; lastagecheckage=1-January-1970; mature_content=1;")
steam_game_tooltip = WebUtils.get_items_by_xpath(html_content, u'.//div[@class="user_reviews_summary_row"]/@data-tooltip-html')[-1]
if steam_game_tooltip != 'Need more user reviews to generate a score' and steam_game_tooltip != 'No user reviews':
steam_score = StringUtils.normalize_int(steam_game_tooltip.split('%')[0])
num_of_reviews = StringUtils.normalize_int(steam_game_tooltip.split('of the')[1].split('user reviews')[0])
return steam_score, num_of_reviews
def process_entries(entries_content, giveaway_entries, winners):
entries_elements = WebUtils.get_items_by_xpath(entries_content, u'.//div[@class="table__row-inner-wrap"]')
for entry_element in entries_elements:
entry_user = WebUtils.get_item_by_xpath(entry_element, u'.//a[@class="table__column__heading"]/text()').encode('utf-8')
entry_timestamp = WebUtils.get_item_by_xpath(entry_element,u'.//div[@class="table__column--width-small text-center"]/span/@data-timestamp')
entry_time = datetime.utcfromtimestamp(StringUtils.normalize_float(entry_timestamp))
winner = False
if entry_user in winners:
winner = True
giveaway_entries[entry_user] = GiveawayEntry(entry_user, entry_time, winner=winner)
def save_empty_group(group_name, group_webpage, cookies):
connection = pymysql.connect(host=host, port=port, user=user, passwd=password, db=db_schema, charset='utf8')
cursor = connection.cursor()
cursor.execute('INSERT IGNORE INTO Groups (GroupID,Users,Giveaways,Name,Webpage,Cookies) '
'VALUES ("' + StringUtils.get_hashed_id(group_webpage) + '","[]","[]","' + group_name + '","' + group_webpage + '","' + to_str(cookies.replace('"','')) + '")')
connection.commit() # you need to call commit() method to save your changes to the database
cursor.close()
connection.close()
def get_group_giveaways(group_webpage,
cookies,
existing_giveaways=None,
force_full_run=False,
start_date=None,
end_date=None):
if existing_giveaways is None:
existing_giveaways = dict()
LogUtils.log_info('Starting to process giveaways for group ' +
group_webpage)
group_giveaways = dict()
games = dict()
reached_end = False
giveaways_changed = True
reached_ended_giveaways = False
page_index = 1
while not reached_end and (giveaways_changed or not reached_ended_giveaways
or force_full_run):
giveaways_changed = False
reached_ended_giveaways = False
giveaways_page_url = group_webpage + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
page_index)
html_content = WebUtils.get_html_page(giveaways_page_url)
if html_content is None:
LogUtils.log_error('Cannot process page: ' + giveaways_page_url)
break
current_page_num = WebUtils.get_item_by_xpath(
html_content, u'.//a[@class="is-selected"]/span/text()')
if current_page_num and current_page_num != str(page_index):
break
if current_page_num:
LogUtils.log_info('Processing giveaways page ' +
get_page_num_str(current_page_num))
giveaway_elements = WebUtils.get_items_by_xpath(
html_content, u'.//div[@class="giveaway__summary"]')
if end_date:
earliest_end_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(
WebUtils.get_items_by_xpath(
giveaway_elements[-1], u'.//span/@data-timestamp')[0]))
if earliest_end_time and earliest_end_time > datetime.strptime(
end_date, '%Y-%m-%d'):
page_index += 1
continue
for giveaway_elem in giveaway_elements:
giveaway_not_started_yet = False
for end_time_text in WebUtils.get_items_by_xpath(
giveaway_elem, u'.//div/text()'):
if end_time_text == ' Begins in ':
giveaway_not_started_yet = True
partial_giveaway_link = WebUtils.get_item_by_xpath(
giveaway_elem, u'.//a[@class="giveaway__heading__name"]/@href')
giveaway_link = SteamGiftsConsts.get_giveaway_link(
partial_giveaway_link)
LogUtils.log_info('Processing ' + giveaway_link)
game_name = WebUtils.get_item_by_xpath(
giveaway_elem,
u'.//a[@class="giveaway__heading__name"]/text()',
default='').encode('utf-8')
winners = WebUtils.get_items_by_xpath(
giveaway_elem,
u'.//div[@class="giveaway__column--positive"]/a/text()')
poster = WebUtils.get_item_by_xpath(
giveaway_elem, u'.//a[@class="giveaway__username"]/text()')
timestamps = WebUtils.get_items_by_xpath(
giveaway_elem, u'.//span/@data-timestamp')
creation_time = None
end_time = None
if len(timestamps) >= 2:
if giveaway_not_started_yet:
creation_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(timestamps[0]))
else:
end_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(timestamps[0]))
creation_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(timestamps[1]))
if winners:
reached_ended_giveaways = True
if len(winners) >= 3:
winners_content = WebUtils.get_html_page(
SteamGiftsConsts.get_giveaway_winners_link(
partial_giveaway_link),
cookies=cookies)
if winners_content is not None:
error_message = WebUtils.get_item_by_xpath(
winners_content,
u'.//div[@class="page__heading__breadcrumbs"]/text()')
if not error_message or error_message != 'Error':
current_winners_page_num = WebUtils.get_item_by_xpath(
winners_content,
u'.//a[@class="is-selected"]/span/text()')
LogUtils.log_info(
'Processing ' + giveaway_link + ' winners page ' +
get_page_num_str(current_winners_page_num))
winners.extend(
WebUtils.get_items_by_xpath(
winners_content,
u'.//p[@class="table__column__heading"]/a/text()'
))
if current_winners_page_num:
winners_page_index = 2
while True:
winners_url = SteamGiftsConsts.get_giveaway_winners_link(
partial_giveaway_link
) + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
winners_page_index)
winners_content = WebUtils.get_html_page(
winners_url, cookies=cookies)
if winners_content is None:
LogUtils.log_error(
'Cannot process page: ' + winners_url)
break
current_winners_page_num = WebUtils.get_item_by_xpath(
winners_content,
u'.//a[@class="is-selected"]/span/text()')
if current_winners_page_num and current_winners_page_num != str(
winners_page_index):
break
LogUtils.log_info(
'Processing ' + giveaway_link +
' winners page ' +
get_page_num_str(current_winners_page_num))
winners.extend(
WebUtils.get_items_by_xpath(
winners_content,
u'.//p[@class="table__column__heading"]/a/text()'
))
winners_page_index += 1
giveaway_entries = dict()
entries_content = None
if not giveaway_not_started_yet:
entries_content = WebUtils.get_html_page(
SteamGiftsConsts.get_giveaway_entries_link(
partial_giveaway_link),
cookies=cookies)
if entries_content is not None:
error_message = WebUtils.get_item_by_xpath(
entries_content,
u'.//div[@class="page__heading__breadcrumbs"]/text()')
if not error_message or error_message != 'Error':
current_entries_page_num = WebUtils.get_item_by_xpath(
entries_content,
u'.//a[@class="is-selected"]/span/text()')
LogUtils.log_info(
'Processing ' + giveaway_link + ' entries page ' +
get_page_num_str(current_entries_page_num))
process_entries(entries_content, giveaway_entries, winners)
if current_entries_page_num:
entries_page_index = 2
while True:
entries_url = SteamGiftsConsts.get_giveaway_entries_link(
partial_giveaway_link
) + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
entries_page_index)
entries_content = WebUtils.get_html_page(
entries_url, cookies=cookies)
if entries_content is None:
LogUtils.log_error('Cannot process page: ' +
entries_url)
break
current_entries_page_num = WebUtils.get_item_by_xpath(
entries_content,
u'.//a[@class="is-selected"]/span/text()')
if current_entries_page_num and current_entries_page_num != str(
entries_page_index):
break
LogUtils.log_info(
'Processing ' + giveaway_link +
' entries page ' +
get_page_num_str(current_entries_page_num))
process_entries(entries_content, giveaway_entries,
winners)
entries_page_index += 1
else:
LogUtils.log_warning('Unable to process entries for ' +
giveaway_link)
# We can't access the GA data, but we can still know who won
for entry_user in winners:
giveaway_entries[entry_user] = GiveawayEntry(
entry_user,
datetime.utcfromtimestamp(0),
winner=True)
giveaway_groups = []
giveaway_groups_content = WebUtils.get_html_page(
SteamGiftsConsts.get_giveaway_groups_link(
partial_giveaway_link),
cookies=cookies)
if giveaway_groups_content is not None:
giveaway_groups = WebUtils.get_items_by_xpath(
giveaway_groups_content,
u'.//a[@class="table__column__heading"]/@href')
else:
LogUtils.log_warning('Unable to process groups for ' +
giveaway_link)
group_giveaway = GroupGiveaway(giveaway_link, game_name, poster,
creation_time, end_time,
giveaway_entries, giveaway_groups)
steam_game_link = WebUtils.get_item_by_xpath(
giveaway_elem, u'.//a[@class="giveaway__icon"]/@href')
game_value = float(
WebUtils.get_items_by_xpath(
giveaway_elem,
u'.//span[@class="giveaway__heading__thin"]/text()')[-1]
[1:-2])
games[game_name] = GameData(game_name, steam_game_link, game_value)
if giveaway_link not in existing_giveaways or not group_giveaway.equals(
existing_giveaways[giveaway_link]):
giveaways_changed = True
if giveaway_link in existing_giveaways and existing_giveaways[
giveaway_link].start_time != group_giveaway.start_time:
group_giveaway.start_time = existing_giveaways[
giveaway_link].start_time
group_giveaways[giveaway_link] = group_giveaway
if start_date:
if end_time and end_time < datetime.strptime(
start_date, '%Y-%m-%d'):
reached_end = True
break
if not current_page_num or reached_end:
break
page_index += 1
LogUtils.log_info('Finished processing giveaways for group ' +
group_webpage)
return group_giveaways, games
def save_group(group_website,
group,
users_to_ignore,
existing_group_data=None):
start_time = time.time()
connection = pymysql.connect(host=host,
port=port,
user=user,
passwd=password,
db=db_schema,
charset='utf8')
cursor = connection.cursor()
# Insert Giveaways
giveaways_data = []
group_giveaways_data = []
for group_giveaway in group.group_giveaways.values():
giveaway_id = StringUtils.get_hashed_id(group_giveaway.link)
group_giveaways_data.append(
(giveaway_id, to_epoch(group_giveaway.start_time),
to_epoch(group_giveaway.end_time)))
entries_data = []
for entry in group_giveaway.entries.values():
entries_data.append(
(entry.user_name, to_epoch(entry.entry_time), entry.winner))
giveaways_data.append(
(giveaway_id, group_giveaway.link, group_giveaway.creator,
group_giveaway.game_name, json.dumps(entries_data),
json.dumps(group_giveaway.groups)))
if giveaways_data:
cursor.executemany("INSERT INTO Giveaways (GiveawayID,LinkURL,Creator,GameName,Entries,Groups) VALUES (%s,%s,%s,%s,%s,%s)"\
+ " ON DUPLICATE KEY UPDATE LinkURL=VALUES(LinkURL),Creator=VALUES(Creator),GameName=VALUES(GameName),Entries=VALUES(Entries),Groups=VALUES(Groups)", giveaways_data)
# Merge with existing group data (in case of update/merge)
if existing_group_data and existing_group_data.group_giveaways:
for existing_giveaway_data in existing_group_data.group_giveaways.values(
):
if existing_giveaway_data.link not in group.group_giveaways.keys():
giveaway_id = StringUtils.get_hashed_id(
existing_giveaway_data.link)
group_giveaways_data.append(
(giveaway_id, to_epoch(existing_giveaway_data.start_time),
to_epoch(existing_giveaway_data.end_time)))
# Insert Users
users_data = []
group_users_data = []
for group_user in group.group_users.values():
group_users_data.append(group_user.user_name)
if group_user.user_name not in users_to_ignore:
users_data.append((group_user.user_name, group_user.steam_id,
group_user.steam_user_name,
to_mysql_date(group_user.creation_time)))
if users_data:
cursor.executemany(
"INSERT INTO Users (UserName,SteamId,SteamUserName,CreationTime) VALUES (%s, %s, %s, %s)"
" ON DUPLICATE KEY UPDATE UserName=VALUES(UserName),SteamId=VALUES(SteamId),SteamUserName=VALUES(SteamUserName),CreationTime=VALUES(CreationTime)",
users_data)
# Insert Group
group_id_str = "\"" + StringUtils.get_hashed_id(group_website) + "\""
group_users_data_str = "\"" + json.dumps(group_users_data).replace(
'"', '\\"') + "\""
group_giveaways_data_str = "\"" + json.dumps(group_giveaways_data).replace(
'"', '\\"') + "\""
group_name_str = "\"" + to_str(group.group_name) + "\""
group_webpage_str = "\"" + to_str(group.group_webpage) + "\""
cookies_str = "\"" + to_str(group.cookies) + "\""
cursor.execute(
"INSERT INTO Groups (GroupID,Users,Giveaways,Name,Webpage,Cookies) VALUES ("
+ group_id_str + "," + group_users_data_str + "," +
group_giveaways_data_str + "," + group_name_str + "," +
group_webpage_str + "," + cookies_str +
") ON DUPLICATE KEY UPDATE Users=VALUES(Users),Giveaways=VALUES(Giveaways)"
)
connection.commit(
) # you need to call commit() method to save your changes to the database
cursor.close()
connection.close()
LogUtils.log_info('Save Group ' + group_website + ' took ' +
str(time.time() - start_time) + ' seconds')
def load_group(group_website,
load_users_data=True,
load_giveaway_data=True,
fetch_not_started_giveaways=False,
limit_by_time=False,
starts_after_str=None,
ends_before_str=None,
ends_after_str=None):
start_time = time.time()
connection = pymysql.connect(host=host,
port=port,
user=user,
passwd=password,
db=db_schema,
charset='utf8')
cursor = connection.cursor()
# Load Group
group_id = StringUtils.get_hashed_id(group_website)
cursor.execute(
'SELECT Users,Giveaways,Cookies FROM Groups WHERE GroupID="' +
group_id + '"')
data = cursor.fetchone()
group_users_data = json.loads(data[0])
group_giveaways_data = json.loads(data[1])
cookies = data[2]
# Load Users Data
group_users = dict()
if load_users_data and group_users_data:
for row in group_users_data:
# (group_user.user_name)
if isinstance(row, basestring):
user_name = row
else:
#Handling old data
user_name = row[0]
group_users[user_name] = GroupUser(user_name)
cursor.execute('SELECT * FROM Users WHERE UserName in (' +
parse_list(group_users.keys()) + ')')
data = cursor.fetchall()
for row in data:
# (group_user.user_name, group_user.steam_id, group_user.steam_user_name, group_user.creation_time)
user_name = row[0]
user_data = GroupUser(user_name,
steam_id=row[1],
steam_user_name=row[2],
creation_time=row[3])
group_users[user_name] = user_data
# Load Giveaways Data
group_giveaways = dict()
giveaways_by_id = dict()
if load_giveaway_data and group_giveaways_data:
for row in group_giveaways_data:
start_time_epoch = row[1]
end_time_epoch = row[2]
if not fetch_not_started_giveaways and not end_time_epoch:
continue
if limit_by_time and \
((starts_after_str and datetime.datetime.utcfromtimestamp(start_time_epoch) < datetime.datetime.strptime(starts_after_str, '%Y-%m-%d'))
or (ends_before_str and datetime.datetime.utcfromtimestamp(end_time_epoch) > datetime.datetime.strptime(ends_before_str, "%Y-%m-%d"))
or (ends_after_str and datetime.datetime.utcfromtimestamp(end_time_epoch) < datetime.datetime.strptime(ends_after_str, "%Y-%m-%d"))):
continue
# (giveaway_id, calendar.timegm(group_giveaway.start_time), calendar.timegm(group_giveaway.end_time))
giveaway_id = row[0]
giveaways_by_id[giveaway_id] = GroupGiveaway(
None,
start_time=from_epoch(start_time_epoch),
end_time=from_epoch(end_time_epoch))
cursor.execute('SELECT * FROM Giveaways WHERE GiveawayID in (' +
parse_list(giveaways_by_id.keys()) + ')')
data = cursor.fetchall()
for row in data:
# (giveaway_id, group_giveaway.link, group_giveaway.creator, group_giveaway.game_name, json.dumps(entries_data), json.dumps(group_giveaway.groups))
giveaway_link = str(row[1])
giveaway_id = StringUtils.get_hashed_id(giveaway_link)
group_giveaways[giveaway_link] = giveaways_by_id[giveaway_id]
group_giveaways[giveaway_link].link = giveaway_link
group_giveaways[giveaway_link].creator = str(row[2])
group_giveaways[giveaway_link].game_name = row[3].encode('utf-8')
group_giveaways[giveaway_link].entries = dict()
for ent_row in json.loads(row[4]):
# (entry.user_name, entry.entry_time, entry.winner)
user_name = str(ent_row[0])
group_giveaways[giveaway_link].entries[
user_name] = GiveawayEntry(user_name,
entry_time=from_epoch(
ent_row[1]),
winner=ent_row[2])
group_giveaways[giveaway_link].groups = json.loads(str(row[5]))
cursor.close()
connection.close()
LogUtils.log_info('Load Group ' + group_website + ' took ' +
str(time.time() - start_time) + ' seconds')
return Group(group_users, group_giveaways, cookies=cookies)