def get_user_contribution_data(user_name):
html_content = WebUtils.get_html_page(SteamGiftsConsts.get_user_link(user_name))
if html_content is None:
LogUtils.log_error('Cannot update additional data for user: '******'.//div[@class="sidebar__shortcut-inner-wrap"]/a/@href')
if not steam_user:
LogUtils.log_error('Cannot update non-existent user: '******'.//div[@class="featured__table__row"]')
for row_content in all_rows:
row_title = WebUtils.get_item_by_xpath(row_content, u'.//div[@class="featured__table__row__left"]/text()')
if row_title == u'Gifts Won':
global_won = StringUtils.normalize_int(WebUtils.get_item_by_xpath(row_content, u'.//div[@class="featured__table__row__right"]/span/span/a/text()'))
elif row_title == u'Gifts Sent':
global_sent = StringUtils.normalize_int(WebUtils.get_item_by_xpath(row_content, u'.//div[@class=" featured__table__row__right"]/span/span/a/text()'))
elif row_title == u'Contributor Level':
user_level_item = WebUtils.get_item_by_xpath(row_content, u'.//div[@class="featured__table__row__right"]/span/@data-ui-tooltip')
level = StringUtils.normalize_float(user_level_item.split('name" : "')[2].split('", "color')[0])
if global_won or global_sent or level:
return global_won, global_sent, level
return None
def get_group_users(group_webpage):
LogUtils.log_info('Processing users for group ' + group_webpage)
group_users = dict()
page_index = 1
while True:
LogUtils.log_info('Processing users page #' + str(page_index))
users_page_url = SteamGiftsConsts.get_steamgifts_users_page(
group_webpage) + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
page_index)
html_content = WebUtils.get_html_page(users_page_url)
if html_content is None:
LogUtils.log_error('Cannot process users page: ' + users_page_url)
break
current_page_num = WebUtils.get_item_by_xpath(
html_content, u'.//a[@class="is-selected"]/span/text()')
if current_page_num and current_page_num != str(page_index):
break
user_elements = WebUtils.get_items_by_xpath(
html_content, u'.//div[@class="table__row-outer-wrap"]')
for user_elem in user_elements:
user = WebUtils.get_item_by_xpath(
user_elem, u'.//a[@class="table__column__heading"]/text()')
group_users[user] = GroupUser(user)
if not current_page_num:
break
page_index += 1
LogUtils.log_info('Finished processing users for group ' + group_webpage)
return group_users
def update_user_additional_data(user):
LogUtils.log_info('Processing SteamGifts user ' + user.user_name)
html_content = WebUtils.get_html_page(SteamGiftsConsts.get_user_link(
user.user_name),
delay=delay_time)
if html_content is None:
LogUtils.log_error('Cannot update additional data for user: '******'.//div[@class="sidebar__shortcut-inner-wrap"]/a/@href')
if not steam_user:
LogUtils.log_error('Cannot update non-existent user: '******'.//div[@class="featured__table__row"]')
for row_content in user_menu_rows:
user_menu_type = WebUtils.get_item_by_xpath(
row_content, u'.//div[@class="featured__table__row__left"]/text()')
if user_menu_type == 'Registered':
data_timestamp = float(
WebUtils.get_item_by_xpath(
row_content,
u'.//div[@class="featured__table__row__right"]/span/@data-timestamp'
))
user.creation_time = datetime.fromtimestamp(data_timestamp)
break
return True
def get_games_from_package(package_name, package_link):
LogUtils.log_info('Processing package ' + package_name)
html_content = WebUtils.get_html_page(
package_link,
"birthtime=-7199; lastagecheckage=1-January-1970; mature_content=1;")
games = WebUtils.get_items_by_xpath(
html_content, u'.//a[@class="tab_item_overlay"]/@href')
return games
def is_giveaway_deleted(giveaway_link, cookies):
LogUtils.log_info('Checking if giveaway was deleted: ' + giveaway_link)
giveaway_content = WebUtils.get_html_page(giveaway_link, cookies=cookies, delay=delay_time)
if giveaway_content is not None:
error_messages = WebUtils.get_items_by_xpath(giveaway_content,u'.//div[@class="table__column--width-fill"]/text()')
if error_messages and len(error_messages) >= 4 and error_messages[0].startswith('Deleted'):
return True
return False
def process_entries(entries_content, giveaway_entries, winners):
entries_elements = WebUtils.get_items_by_xpath(entries_content, u'.//div[@class="table__row-inner-wrap"]')
for entry_element in entries_elements:
entry_user = WebUtils.get_item_by_xpath(entry_element, u'.//a[@class="table__column__heading"]/text()').encode('utf-8')
entry_timestamp = WebUtils.get_item_by_xpath(entry_element,u'.//div[@class="table__column--width-small text-center"]/span/@data-timestamp')
entry_time = datetime.utcfromtimestamp(StringUtils.normalize_float(entry_timestamp))
winner = False
if entry_user in winners:
winner = True
giveaway_entries[entry_user] = GiveawayEntry(entry_user, entry_time, winner=winner)
def verify_after_n_giveaways(steam_after_n_thread_link, giveaways, group_users, max_pages=0):
after_n_giveaways = dict()
steam_id_to_user = get_steam_id_to_user_dict(group_users.values())
page_index = 1
while page_index < max_pages or max_pages == 0:
page_content = WebUtils.get_page_content(steam_after_n_thread_link + SteamConsts.STEAM_SEARCH_QUERY + str(page_index))
partial_page = page_content[page_content.find('_pageend') + 10:]
page_end = partial_page[:partial_page.find('</span>')]
partial_page = page_content[page_content.find('_pagetotal') + 12:]
page_total = partial_page[:partial_page.find('</span>')]
html_content = html.fromstring(page_content)
gifter_elemens = WebUtils.get_items_by_xpath(html_content, u'.//div[@class="commentthread_comment responsive_body_text "]')
for gifter_elem in gifter_elemens:
steam_user = WebUtils.get_item_by_xpath(gifter_elem, u'.//a[@class="hoverunderline commentthread_author_link "]/@href')
if not steam_user:
continue # this means this is the creator of the thread
if SteamConsts.STEAM_PROFILE_LINK in steam_user:
steam_id = steam_user.split(SteamConsts.STEAM_PROFILE_LINK)[1]
else:
steam_id = SteamFinderScrapingUtils.get_steam_id(steam_user)
# Check if steam_id is currently in SG group
if steam_id in steam_id_to_user:
user = steam_id_to_user[steam_id]
giveaway_links = WebUtils.get_items_by_xpath(gifter_elem, u'.//div[@class="commentthread_comment_text"]/a/@href')
for giveaway_link in giveaway_links:
giveaway_link = str(giveaway_link).replace(SteamConsts.STEAM_FILTER_LINK, '')
if (giveaway_link in giveaways.keys() and giveaways[giveaway_link].creator == user and giveaway_link
and giveaway_link.startswith(SteamGiftsConsts.STEAMGIFTS_GIVEAWAY_LINK)):
if user not in after_n_giveaways:
after_n_giveaways[user] = set()
after_n_giveaways[user].add(giveaway_link)
if page_end == page_total:
break
page_index += 1
return after_n_giveaways
def get_game_additional_data(game_name, game_link):
LogUtils.log_info('Processing game ' + game_name)
steam_score = 0
num_of_reviews = 0
html_content = WebUtils.get_html_page(game_link, "birthtime=-7199; lastagecheckage=1-January-1970; mature_content=1;")
base_game_link = WebUtils.get_item_by_xpath(html_content, u'.//div[@class="glance_details"]/a/@href')
if base_game_link is not None:
# If this is DLC - get additional data according to base game
html_content = WebUtils.get_html_page(base_game_link, "birthtime=-7199; lastagecheckage=1-January-1970; mature_content=1;")
steam_game_tooltip = WebUtils.get_items_by_xpath(html_content, u'.//div[@class="user_reviews_summary_row"]/@data-tooltip-html')[-1]
if steam_game_tooltip != 'Need more user reviews to generate a score' and steam_game_tooltip != 'No user reviews':
steam_score = StringUtils.normalize_int(steam_game_tooltip.split('%')[0])
num_of_reviews = StringUtils.normalize_int(steam_game_tooltip.split('of the')[1].split('user reviews')[0])
return steam_score, num_of_reviews
def get_group_giveaways(group_webpage,
cookies,
existing_giveaways=None,
force_full_run=False,
start_date=None,
end_date=None):
if existing_giveaways is None:
existing_giveaways = dict()
LogUtils.log_info('Starting to process giveaways for group ' +
group_webpage)
group_giveaways = dict()
games = dict()
reached_end = False
giveaways_changed = True
reached_ended_giveaways = False
page_index = 1
while not reached_end and (giveaways_changed or not reached_ended_giveaways
or force_full_run):
giveaways_changed = False
reached_ended_giveaways = False
giveaways_page_url = group_webpage + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
page_index)
html_content = WebUtils.get_html_page(giveaways_page_url)
if html_content is None:
LogUtils.log_error('Cannot process page: ' + giveaways_page_url)
break
current_page_num = WebUtils.get_item_by_xpath(
html_content, u'.//a[@class="is-selected"]/span/text()')
if current_page_num and current_page_num != str(page_index):
break
if current_page_num:
LogUtils.log_info('Processing giveaways page ' +
get_page_num_str(current_page_num))
giveaway_elements = WebUtils.get_items_by_xpath(
html_content, u'.//div[@class="giveaway__summary"]')
if end_date:
earliest_end_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(
WebUtils.get_items_by_xpath(
giveaway_elements[-1], u'.//span/@data-timestamp')[0]))
if earliest_end_time and earliest_end_time > datetime.strptime(
end_date, '%Y-%m-%d'):
page_index += 1
continue
for giveaway_elem in giveaway_elements:
giveaway_not_started_yet = False
for end_time_text in WebUtils.get_items_by_xpath(
giveaway_elem, u'.//div/text()'):
if end_time_text == ' Begins in ':
giveaway_not_started_yet = True
partial_giveaway_link = WebUtils.get_item_by_xpath(
giveaway_elem, u'.//a[@class="giveaway__heading__name"]/@href')
giveaway_link = SteamGiftsConsts.get_giveaway_link(
partial_giveaway_link)
LogUtils.log_info('Processing ' + giveaway_link)
game_name = WebUtils.get_item_by_xpath(
giveaway_elem,
u'.//a[@class="giveaway__heading__name"]/text()',
default='').encode('utf-8')
winners = WebUtils.get_items_by_xpath(
giveaway_elem,
u'.//div[@class="giveaway__column--positive"]/a/text()')
poster = WebUtils.get_item_by_xpath(
giveaway_elem, u'.//a[@class="giveaway__username"]/text()')
timestamps = WebUtils.get_items_by_xpath(
giveaway_elem, u'.//span/@data-timestamp')
creation_time = None
end_time = None
if len(timestamps) >= 2:
if giveaway_not_started_yet:
creation_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(timestamps[0]))
else:
end_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(timestamps[0]))
creation_time = datetime.utcfromtimestamp(
StringUtils.normalize_float(timestamps[1]))
if winners:
reached_ended_giveaways = True
if len(winners) >= 3:
winners_content = WebUtils.get_html_page(
SteamGiftsConsts.get_giveaway_winners_link(
partial_giveaway_link),
cookies=cookies)
if winners_content is not None:
error_message = WebUtils.get_item_by_xpath(
winners_content,
u'.//div[@class="page__heading__breadcrumbs"]/text()')
if not error_message or error_message != 'Error':
current_winners_page_num = WebUtils.get_item_by_xpath(
winners_content,
u'.//a[@class="is-selected"]/span/text()')
LogUtils.log_info(
'Processing ' + giveaway_link + ' winners page ' +
get_page_num_str(current_winners_page_num))
winners.extend(
WebUtils.get_items_by_xpath(
winners_content,
u'.//p[@class="table__column__heading"]/a/text()'
))
if current_winners_page_num:
winners_page_index = 2
while True:
winners_url = SteamGiftsConsts.get_giveaway_winners_link(
partial_giveaway_link
) + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
winners_page_index)
winners_content = WebUtils.get_html_page(
winners_url, cookies=cookies)
if winners_content is None:
LogUtils.log_error(
'Cannot process page: ' + winners_url)
break
current_winners_page_num = WebUtils.get_item_by_xpath(
winners_content,
u'.//a[@class="is-selected"]/span/text()')
if current_winners_page_num and current_winners_page_num != str(
winners_page_index):
break
LogUtils.log_info(
'Processing ' + giveaway_link +
' winners page ' +
get_page_num_str(current_winners_page_num))
winners.extend(
WebUtils.get_items_by_xpath(
winners_content,
u'.//p[@class="table__column__heading"]/a/text()'
))
winners_page_index += 1
giveaway_entries = dict()
entries_content = None
if not giveaway_not_started_yet:
entries_content = WebUtils.get_html_page(
SteamGiftsConsts.get_giveaway_entries_link(
partial_giveaway_link),
cookies=cookies)
if entries_content is not None:
error_message = WebUtils.get_item_by_xpath(
entries_content,
u'.//div[@class="page__heading__breadcrumbs"]/text()')
if not error_message or error_message != 'Error':
current_entries_page_num = WebUtils.get_item_by_xpath(
entries_content,
u'.//a[@class="is-selected"]/span/text()')
LogUtils.log_info(
'Processing ' + giveaway_link + ' entries page ' +
get_page_num_str(current_entries_page_num))
process_entries(entries_content, giveaway_entries, winners)
if current_entries_page_num:
entries_page_index = 2
while True:
entries_url = SteamGiftsConsts.get_giveaway_entries_link(
partial_giveaway_link
) + SteamGiftsConsts.STEAMGIFTS_SEARCH_PAGE + str(
entries_page_index)
entries_content = WebUtils.get_html_page(
entries_url, cookies=cookies)
if entries_content is None:
LogUtils.log_error('Cannot process page: ' +
entries_url)
break
current_entries_page_num = WebUtils.get_item_by_xpath(
entries_content,
u'.//a[@class="is-selected"]/span/text()')
if current_entries_page_num and current_entries_page_num != str(
entries_page_index):
break
LogUtils.log_info(
'Processing ' + giveaway_link +
' entries page ' +
get_page_num_str(current_entries_page_num))
process_entries(entries_content, giveaway_entries,
winners)
entries_page_index += 1
else:
LogUtils.log_warning('Unable to process entries for ' +
giveaway_link)
# We can't access the GA data, but we can still know who won
for entry_user in winners:
giveaway_entries[entry_user] = GiveawayEntry(
entry_user,
datetime.utcfromtimestamp(0),
winner=True)
giveaway_groups = []
giveaway_groups_content = WebUtils.get_html_page(
SteamGiftsConsts.get_giveaway_groups_link(
partial_giveaway_link),
cookies=cookies)
if giveaway_groups_content is not None:
giveaway_groups = WebUtils.get_items_by_xpath(
giveaway_groups_content,
u'.//a[@class="table__column__heading"]/@href')
else:
LogUtils.log_warning('Unable to process groups for ' +
giveaway_link)
group_giveaway = GroupGiveaway(giveaway_link, game_name, poster,
creation_time, end_time,
giveaway_entries, giveaway_groups)
steam_game_link = WebUtils.get_item_by_xpath(
giveaway_elem, u'.//a[@class="giveaway__icon"]/@href')
game_value = float(
WebUtils.get_items_by_xpath(
giveaway_elem,
u'.//span[@class="giveaway__heading__thin"]/text()')[-1]
[1:-2])
games[game_name] = GameData(game_name, steam_game_link, game_value)
if giveaway_link not in existing_giveaways or not group_giveaway.equals(
existing_giveaways[giveaway_link]):
giveaways_changed = True
if giveaway_link in existing_giveaways and existing_giveaways[
giveaway_link].start_time != group_giveaway.start_time:
group_giveaway.start_time = existing_giveaways[
giveaway_link].start_time
group_giveaways[giveaway_link] = group_giveaway
if start_date:
if end_time and end_time < datetime.strptime(
start_date, '%Y-%m-%d'):
reached_end = True
break
if not current_page_num or reached_end:
break
page_index += 1
LogUtils.log_info('Finished processing giveaways for group ' +
group_webpage)
return group_giveaways, games
def get_and_encode_list(element, xpath):
results = []
for item in WebUtils.get_items_by_xpath(element, xpath):
results.append(item.encode('utf-8'))
return results