def test_mainpage_18(self):
main = MainPage(self.driver)
main.click_close_x_icon()
# Mouseover "SHOP" dropdown box
# Click on "HAIR" link box
actions = action_chains.ActionChains(self.driver)
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
actions.move_to_element(shop).perform()
hair = self.driver.find_element_by_xpath(MainPage.hair_link)
actions.move_to_element(hair).perform()
hair.click()
# Verify if the right page was opened
assert self.driver.title == "Hair Care Products | Sephora", "ERROR. Hair Care Products page wasn't opened."
# Go back to the main page
main.click_main_page()
# Mouseover "SHOP" dropdown box
# Click "TOOLS & BRUSHES" link box
actions = action_chains.ActionChains(self.driver)
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
actions.move_to_element(shop).perform()
tools_brushes = self.driver.find_element_by_xpath(
MainPage.tools_brushes_link)
actions.move_to_element(tools_brushes).perform()
tools_brushes.click()
# Verify if the right page was opened
assert self.driver.title == "Makeup Tools, Beauty Tools & Makeup Accessories | Sephora", "ERROR. Makeup Tools, Beauty Tools & Makeup Accessories page wasn't opened."
def userGender(ele):
peoID = ele.text #记录一下用户名
print(ele.text)
action_chains.ActionChains(driver).move_to_element(ele).perform() #只有这样才能拿到最新的action
EC = WebDriverWait(driver,3,0.3) #频率填0.2会出错。因为上面操作有改动driver的内容，而driver在第一个0.2时还没有正确刷新
cnt = 3
while(cnt>0):
try:
#有效弹出卡
EC.until(lambda x: x.find_element_by_xpath("//div[@id = 'zh-hovercard']//span[@class ='name']").text == peoID)
break
except Exception as e:
print("WebDriverWait出错",cnt)
cnt = cnt -1
if (cnt <= 0):
return
continue
tarDiv = driver.find_element_by_xpath("//div[@id = 'zh-hovercard']//div[@class ='upper']")
try: #有的没填性别
GenderIcon = tarDiv.find_element_by_class_name('icon')
except Exception as e:
print('没有性别图标')
return True
strGender = GenderIcon.get_attribute('class') #拿到字串，icon icon-profile-male
action_chains.ActionChains(driver).move_by_offset(50, 50).perform() #移开
if (strGender.endswith("female")):
print('女')
return True
elif(strGender.endswith("male")): #男
print('男')
else:
print('性别辨认出错')
return False
def test_mainpage_19(self):
main = MainPage(self.driver)
main.click_close_x_icon()
# Mouseover "SHOP" dropdown box
# Click on "FRAGRANCE" link box
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
actions = action_chains.ActionChains(self.driver)
actions.move_to_element(shop).perform()
fragrance = self.driver.find_element_by_xpath(MainPage.fragrance_link)
actions.move_to_element(fragrance).perform()
fragrance.click()
# Verify if the right page opened
assert self.driver.title == "Fragrance | Sephora", "ERROR. Fragrance page wasn't opened."
# Go back to the main page
main.click_main_page()
# Mouseover "SHOP" dropdown box
# Click "BATH & BODY" link box
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
bath_body = self.driver.find_element_by_xpath(MainPage.bath_body_link)
actions = action_chains.ActionChains(self.driver)
actions.move_to_element(shop).perform()
actions.move_to_element(bath_body).perform()
bath_body.click()
# Verify if the right page opened
assert self.driver.title == "Bath Products & Body Products | Sephora", "ERROR. Bath Products & Body Products page wasn't opened."
def test_mainpage_17(self):
wait = WebDriverWait(self.driver, 10)
main = MainPage(self.driver)
main.click_close_x_icon()
# Mouseover "SHOP" dropdown box
# Click on "MAKEUP" link box
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
makeup = self.driver.find_element_by_xpath(MainPage.makeup_link)
actions = action_chains.ActionChains(self.driver)
actions.move_to_element(shop).perform()
actions.move_to_element(makeup).perform()
makeup.click()
# actions.move_to_element(shop).move_to_element(makeup).click().perform()
# Verify if the right page was opened
assert self.driver.title == "Makeup | Sephora", "ERROR. Makeup page wasn't opened."
# Go back to the main page
main.click_main_page()
# Mouseover "SHOP" dropdown box
# Click "SKINCARE" link box
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
skincare = self.driver.find_element_by_xpath(MainPage.skincare_link)
actions = action_chains.ActionChains(self.driver)
actions.move_to_element(shop).perform()
actions.move_to_element(skincare).perform()
skincare.click()
# Verify if the right page was opened
assert self.driver.title == "Skincare Products | Sephora", "ERROR. Skincare page wasn't opened."
def test_mainpage_20(self):
main = MainPage(self.driver)
main.click_close_x_icon()
# Mouseover "SHOP" dropdown box
# Click on "GIFTS" link box
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
actions = action_chains.ActionChains(self.driver)
actions.move_to_element(shop).perform()
gifts_link = self.driver.find_element_by_xpath(MainPage.gifts_link)
actions.move_to_element(gifts_link).perform()
gifts_link.click()
# Verify if the right page was opened
assert self.driver.title == "Best Beauty Gifts in 2019 | Sephora", "ERROR. Best Beauty Gifts page wasn't opened."
# Go to the main page
main.click_main_page()
# Mouseover "SHOP" dropdown box
# Click "SALE" link box
shop = self.driver.find_element_by_xpath(MainPage.mouseover_shop)
sale_link = self.driver.find_element_by_xpath(MainPage.sale_link)
actions = action_chains.ActionChains(self.driver)
actions.move_to_element(shop).perform()
actions.move_to_element(sale_link).perform()
sale_link.click()
# Verify if the right page was opened
assert self.driver.title == "Makeup Sale | Beauty Sale | Sephora", "ERROR. Makeup Sale | Beauty Sale page wasn't opened."
def mkdir(self, folder_name):
"""Function which makes a new folder"""
#Wait untill New button is avaible in webpage
new = self.driver.find_elements_by_xpath("//button[@name='New']")
while len(new) == 0:
new = self.driver.find_elements_by_xpath("//button[@name='New']")
#If button is ready, click it
action = action_chains.ActionChains(self.driver)
action.click(new[0])
action.perform()
#Wait untill Folder button is avaible in webpage
folder = self.driver.find_elements_by_xpath("//button[@name='Folder']")
while len(folder) == 0:
folder = self.driver.find_elements_by_xpath(
"//button[@name='Folder']")
#If button is ready, click it
action = action_chains.ActionChains(self.driver)
action.click(folder[0])
action.perform()
#Wait and afterwards sent the folders name and create it
time.sleep(0.1)
action = action_chains.ActionChains(self.driver)
action.send_keys(folder_name)
action.send_keys(keys.Keys.ENTER)
action.perform()
def Log_Into_MS_Account(username, password, ChromeDriver):
MS_url = "https://login.live.com/"
# get the webpage
#This try catch block doesn't really do anything. If Chrome can't find the url it's not throwing an exception. Maybe use an assert on some html on the expected page?
try:
ChromeDriver.get(MS_url)
except Exception:
print(Exception)
sys.exit(1)
# Grab the Username id element. I found these IDs by manually looking at the source code on the page. Probably not how we should be doing it.
UserNameElement = ChromeDriver.find_element_by_id("i0116")
# Enter text into the elements text field
UserNameElement.send_keys(username)
# Grab the button element. I think we can search for "input type = submit" or something like that to find a button without the exact id.
ButtonElement = ChromeDriver.find_element_by_id("idSIButton9")
# Press the button
ButtonElement.send_keys(Keys.ENTER)
# password box is generated by javascript and not just static HTML. It is not a visible element that we can sent commands. Must use actions that send key presses like a person would.
action = action_chains.ActionChains(ChromeDriver)
time.sleep(1)
action.send_keys(password)
action.send_keys(Keys.ENTER)
action.perform()
time.sleep(1)
def select_check_box_format(cls, element_identifier_name,state,value_to_format):
is_selected = False
try:
__obj_wait_extension = WaitExtensions()
by = Page_elements().get_element_identifier(element_identifier_name)
by = eval(by)
__val2 = by[1]
if value_to_format != None : # check if there is an option to format
by[1] = __val2.format(value_to_format)
if (__obj_wait_extension.Wait_for_element_visible(by, 10)):
ele = Env_setup.get_driver().find_element(eval(by[0]), by[1])
action_chains.ActionChains(Env_setup.get_driver()).move_to_element(ele).perform()
if (state.upper() == "ON"):
if ele.is_selected() :
cls.log.info("Check box selection is successful '{}'".format(element_identifier_name))
is_selected = True
else:
ele.click() # if check box is not selected, then select
is_selected = True
else: # to un select check box
ele.click()
cls.log.info("Check box is unselected {}".format(element_identifier_name))
except webdriver_exceptions.NoSuchElementException as e:
cls.log.exception("Exception thrown in 'select_check_box_format' method '{}' ".format(e))
raise e
except Exception as a:
cls.log.exception("Exception thrown in 'select_check_box_format' method '{}' ".format(a))
raise a
cls.log.exception("Value of 'is_selected' is '{}' ".format(is_selected))
return is_selected
def submit_solution(problemLink, submissionFile, driver, language):
driver.get(problemLink)
time.sleep(5)
temp = driver.find_element_by_xpath(
'//*[@id="cc-footer-div"]/div[2]/div[1]/ul/li[1]/a')
action = action_chains.ActionChains(driver)
action.move_to_element(temp).perform()
print("======================")
with open(submissionFile, "r") as file:
code = file.read()
print(code)
print("=====================")
area = driver.find_element_by_xpath('//*[@id="edit-program"]')
area.click()
area.send_keys(keys.Keys.CONTROL + "a")
area.send_keys(keys.Keys.DELETE)
area.send_keys(code)
time.sleep(10)
# select programming language
driver.find_element_by_xpath(
"//select[@id='edit-language']/option[text()='" + language +
"']").click()
button = driver.find_element_by_xpath('//*[@id="edit-submit-1"]')
driver.execute_script("arguments[0].click();", button)
time.sleep(20)
# logout
driver.find_element_by_link_text('Logout').click()
def get_all_security_id(self):
# GOING TO COMPANY FACTSHEET DATA PAGE
action_chains.ActionChains(self.browser).move_to_element(self.browser.find_element_by_class_name('li-company')).perform()
sleep(1)
self.browser.find_element_by_xpath('//a[@href="/factsheet.html"]').click()
# INPUT SYMBOL
## TO DO: FINDING ALL SYMBOL
symbol_master = {}
all_symbol = np.array(pd.read_csv('company_metadata.csv', sep='|')['Symbol'])
n = 0
all_symbol_n = len(all_symbol)
# LOOP OVER SYMBOL
for symbol in all_symbol:
n += 1
sys.stdout.write("\r{0}/{1}".format(str(n), str(all_symbol_n)))
sys.stdout.flush()
try:
# CLEAR INPUT TEXT AND INPUT SYMBOL
self.browser.find_element_by_name('symbol').clear()
self.browser.find_element_by_name('symbol').send_keys(symbol)
sleep(1)
self.browser.find_element_by_name('submit').click()
# SAVE RESULT IN DICT
symbol_master[symbol] = self._get_security_id_setsmart()
except:
print('\n{}'.format(symbol))
# SAVE AS JSON FILE
## TO DO: WHERE TO SAVE THE MASTER
with open('symbol_id_setsmart.json', 'w') as fp:
json.dump(symbol_master, fp)
return symbol_master
def BruteLogin(user, pwd):
browser.get('http://xxx.edu.cn/login.jsp')
browser.implicitly_wait(20)
action = action_chains.ActionChains(browser)
elem = browser.find_element_by_name("j_username")
elem.send_keys(user)
action.perform()
elem = browser.find_element_by_name("j_password")
elem.send_keys(pwd)
action.perform()
elem = browser.find_element_by_name("btn_submit")
#print browser.current_window_handle
#elem.click()
action.send_keys("document.getElementsByName('btn_submit')[0].click()" +
keys.Keys.ENTER)
action.perform()
time.sleep(1)
if '当前用户' in browser.page_source:
print 'Login Success:' + user + '|' + pwd
sys.exit()
else:
print 'LoginFaild!'
def play_game(website):
# Create an instance of a Chrome session
browser = webdriver.Chrome()
browser.get(website)
# Create an instance of an action chain
actions = action_chains.ActionChains(browser)
retryElem = browser.find_element_by_class_name("retry-button")
while True:
# Retry element always gets an exception, check page src before and
# after keypress to know when game-over dialogue is present
stopCheck = browser.page_source
actions.send_keys(keysTuple).perform()
stopCheck2 = browser.page_source
# If no change to source we know we got game-over
# Exceptions for the ODD time they occur
if stopCheck == stopCheck2:
# sleep to confirm the game-over dialogue is shown
#sleep(4)
try:
retryElem.click()
except NoSuchElementException:
print("Passing No such element...")
pass
except ElementNotVisibleException:
print("Passing element not visible..")
pass
def __init__(self, *args, **kwargs):
self.myPyDriver = webdriver.Firefox()
#wait setup
self.wait = WebDriverWait(self.myPyDriver, 15)
#init Action chains
self.action = action_chains.ActionChains(self.myPyDriver)
#init time stamp
self.now = datetime.datetime.now()
self.date = self.now.strftime('%H%M%S_%m%d%y')
config_object = config.config()
selector_object = Selectors.Selectors()
self.url = config_object.url
self.usernameText = config_object.usernameText
self.passwordText = config_object.passwordText
self.name_userName = selector_object.name_userName
self.name_pwInput = selector_object.name_pwInput
self.name_submit = selector_object.name_submit
self.doneButton = selector_object.doneButton
self.className_IconHome = selector_object.className_IconHome
self.myPyDriver.get(self.url)
self.assertTrue( self.myPyDriver.title == "SpaceRunner" )
self.wait.until(EC.presence_of_element_located((By.NAME,self.name_userName)))
self.userNameField = self.myPyDriver.find_element_by_name(self.name_userName)
self.userNameField.send_keys(self.usernameText)
self.userPassword = self.myPyDriver.find_element_by_name(self.name_pwInput)
self.userPassword.send_keys(self.passwordText)
self.loginSubmitButton = self.myPyDriver.find_element_by_name(self.name_submit)
self.loginSubmitButton.click()
def checkStatus(driver, urlNumber, framenumber):
pname = getPName(sys.argv[-1])
player_status = driver.execute_script("return document.getElementById('movie_player').getPlayerState()")
condition = {
-1: "unstarted",
0: "ended",
1: "playing",
2: "paused",
3: "buffering",
5: "video cued",
}
print("Video Status:", player_status, condition[player_status])
if(player_status == -1):
print("Video still hanging: sending click!");
import selenium.webdriver.common.action_chains as ac
playerElement = driver.find_element_by_css_selector("#player-container")
previewfname = buildScreenName('{}-{}-preview'.format(pname, urlNumber))
playerElement.screenshot(previewfname)
actions = ac.ActionChains(driver)
actions.move_to_element(playerElement)
actions.click()
actions.perform()
framenumber = 0;
elif(player_status == 0):
print("Video completed reproduction, closing")
return -1
elif(player_status == 1):
framenumber += 1
framecopy = buildScreenName("{}-{}-snap-{}".format(pname, urlNumber, framenumber) )
playerElement = driver.find_element_by_css_selector("#player-container")
playerElement.screenshot(framecopy)
else:
print("Ignored condition", condition[player_status])
return framenumber
def login(self):
# click on login link
self.browser.find_element(
By.XPATH, "//div[@id='service-navigation']/div/a[2]").click()
# make sure the login input fields are loaded and visible by simulating a click
assert 'login_email_username' in self.browser.page_source
action = action_chains.ActionChains(self.browser)
action.send_keys(keys.Keys.COMMAND + keys.Keys.ALT + 'i')
action.perform()
time.sleep(3)
action.send_keys(keys.Keys.ENTER)
action.send_keys(
"document.querySelector('#login_email_username').click()" +
keys.Keys.ENTER)
action.perform()
# login
self.browser.find_element_by_id('login_email_username').send_keys(
self.username)
self.browser.find_element_by_id('login_password').send_keys(
self.password)
self.browser.find_element_by_id('login_basic').submit()
time.sleep(3)
self.logger.info("Logged in.")
def hit_enter():
action = action_chains.ActionChains(driver)
action.send_keys(Keys.COMMAND + Keys.ALT + 'i')
action.perform()
time.sleep(1)
action.send_keys(Keys.ENTER)
action.perform()
def submit_solution (problemLink,submissionFile,driver) :
driver.get(problemLink)
mode = driver.find_element_by_xpath('//*[@id="edit-submit"]')
if mode.get_attribute("value") == 'Switch to Non-IDE mode':
mode.click()
time.sleep(5)
toggle = driver.find_element_by_id('edit_area_toggle_checkbox_edit-program')
temp = driver.find_element_by_xpath('//*[@id="cc-footer-div"]/div[2]/div[1]/ul/li[1]/a')
action = action_chains.ActionChains(driver)
action.move_to_element(temp).perform()
toggle.click()
print("======================")
with open(submissionFile, "r") as file:
code = file.read()
print(code)
print("=====================")
area = driver.find_element_by_xpath('//*[@id="edit-program"]')
area.click()
area.send_keys(keys.Keys.CONTROL + "a")
area.send_keys(keys.Keys.DELETE)
area.send_keys(code)
time.sleep(10)
button = driver.find_element_by_xpath('//*[@id="edit-submit-1"]')
driver.execute_script("arguments[0].click();", button)
def link_generator_onefootball(link_dict):
match_link_dict = {}
for link in link_dict:
id_counter = 0
try:
driver = webdriver.Firefox()
driver.get(link_dict[link])
WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.CLASS_NAME, 'mb-xxl')))
WebDriverWait(driver, 0.5)
page_sources = []
for counter in range(0, 40):
left_arrow = driver.find_element_by_xpath('/html/body/of-app/main/of-competition-nav/div/div/of-competition-tab-matches/div/section/of-competition-matches/div[1]/nav/span[1]')
clicking = action_chains.ActionChains(driver).move_to_element(left_arrow).click(left_arrow)
clicking.perform()
WebDriverWait(driver, 15) #wait for page loading
page_sources.append(driver.page_source)
finally:
driver.close()
page_sources = list(set(page_sources)) # remove duplicates
for counter, page_source in enumerate(page_sources):
page_soup = bs.BeautifulSoup(page_source, 'lxml')
match_links = []
for match_link in page_soup.find_all(class_="match-cards LARGE"):
for actual_link in match_link.find_all('a'):
match_links.append(actual_link.get('href'))
match_link_dict[link + ' ' + str(counter)] = match_links
return match_link_dict ##flat dictionary, entries: <League> [0-9]+
def edit_environment(self, old_name, new_name):
el_td = self.driver.find_element_by_css_selector(
'tr[data-display="{0}"] '.format(old_name) +
'td[data-cell-name="name"]')
el_pencil = el_td.find_element_by_css_selector(
'button.ajax-inline-edit')
# hover to make pencil visible
hover = action_chains.ActionChains(self.driver).move_to_element(el_td)
hover.perform()
el_pencil.click()
# fill in inline input
el_inline_input = self.driver.find_element_by_css_selector(
'tr[data-display="{0}"] '.format(old_name) +
'td[data-cell-name="name"] .inline-edit-form input')
el_inline_input.clear()
el_inline_input.send_keys(new_name)
# click submit
el_submit = self.driver.find_element_by_css_selector(
'tr[data-display="{0}"] '.format(old_name) +
'td[data-cell-name="name"] .inline-edit-actions' +
' button[type="submit"]')
el_submit.click()
def doubleclick_rte_element(self, elt_css_selector):
self._tester.driver.switch_to_frame(AddLesson.RTE_EDITOR_ID)
target = self.find_element_by_css_selector(elt_css_selector)
action_chains.ActionChains(
self._tester.driver).double_click(target).perform()
self._tester.driver.switch_to_default_content()
return self
def oneway_search(self, driver, depart, dest, day, month, year):
# choose to select one way.
one_way_xpath = '//*[@id="oneWay"]'
element = driver.find_element_by_xpath(one_way_xpath)
element.click()
depart_xpath = '//*[@id="origin_O_0"]'
dest_xpath = '//*[@id="destination_O_0"]'
date_xpath = '//*[@id="returnDateLabel"]/div[1]/div'
empty_xpath = '//*[@id="bookingMagnetForm"]/fieldset/div[2]/div[1]'
search_xpath = '//*[@id="magnet-fields-wrapper"]/div[3]/div[3]/input'
# To edit the date
date_field = driver.find_element_by_xpath(date_xpath)
date_field.click()
# find the correct table (month + year)
month_now = datetime.datetime.now().month
year_now = datetime.datetime.now().year
click_num = (year - year_now) * 12 + (month - month_now)
for i in range(0, click_num):
time.sleep(0.5)
next_field = driver.find_element_by_xpath(
'//*[@id="ui-datepicker-div"]/a[2]')
next_field.click()
# select the correct date
if (day < 10):
day = '0' + str(day)
date_button_xpath = (
'//td[@data-handler="selectDay"][@data-date="{0}"][@data-month="{1}"][@data-year="{2}"]/span[1]'
).format(day, str(month - 1), str(year))
general_button = '//*[@id="ui-datepicker-div"]/a[2]/span'
wait = WebDriverWait(driver, 20)
element = wait.until(
EC.element_to_be_clickable((By.XPATH, general_button)))
button = driver.find_element_by_xpath(date_button_xpath)
action_chains.ActionChains(driver).move_to_element(
button).click().perform()
button.click()
# to edit depart, dest fields
empty_field = driver.find_element_by_xpath(empty_xpath)
depart_field = driver.find_element_by_xpath(depart_xpath)
depart_field.send_keys(depart)
time.sleep(1)
depart_field.send_keys(keys.Keys.ENTER)
dest_field = driver.find_element_by_xpath(dest_xpath)
dest_field.send_keys(dest)
time.sleep(1)
dest_field.send_keys(keys.Keys.ENTER)
# Press search
search_field = driver.find_element_by_xpath(search_xpath)
search_field.click()
return driver
def doubleclick_rte_element(self, elt_css_selector, index=0):
self._tester.driver.switch_to_frame(
CourseContentElement.RTE_EDITOR_FORMAT % index)
target = self.find_element_by_css_selector(elt_css_selector)
action_chains.ActionChains(
self._tester.driver).double_click(target).perform()
self._tester.driver.switch_to_default_content()
return self
def double_click(self):
"""
using action chain
"""
actionWrapper(
actions.ActionChains(self.driver).double_click(self).perform, None,
[self.scroll_into_view],
self.selector.statement + ": iz-dbl-click failed")
def move_element_by_offset(self, x, y):
"""
moves the element by offset of x and y.
"""
actionWrapper(
actions.ActionChains(self.driver).drag_and_drop_by_offset(
self, x, y).perform, [self.scroll_into_view], None,
self.selector.statement + ": iz-move failed")
def move_to_me(self):
"""
moves the cursor to the element's location
"""
actionWrapper(
actions.ActionChains(self.driver).move_to_element(self).perform,
[self.scroll_into_view], None,
self.selector.statement + ": iz-move failed")
def scrap_following(self):
browser = webdriver.PhantomJS(
executable_path=
'C:/Users/ShuaibReeyaz/Downloads/phantomjs-2.1.1-windows/bin/phantomjs'
)
browser.get("https://twitter.com/" + self.query[1:] + "/following")
action = action_chains.ActionChains(browser)
time.sleep(2)
username = browser.find_element_by_css_selector(
'.js-username-field.email-input.js-initial-focus')
username.send_keys('*****@*****.**')
password = browser.find_element_by_css_selector('.js-password-field')
password.send_keys('ilias2019!')
form = browser.find_element_by_css_selector(
'.submit.EdgeButton.EdgeButton--primary.EdgeButtom--medium')
form.submit()
SCROLL_PAUSE_TIME = 0.5
# Get scroll height
last_height = browser.execute_script(
"return document.body.scrollHeight")
while True:
# Scroll down to bottom
browser.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
# Wait to load page
time.sleep(SCROLL_PAUSE_TIME)
# Calculate new scroll height and compare with last scroll height
new_height = browser.execute_script(
"return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
element = browser.find_element_by_xpath("//body")
page = Selector(text=element.text)
info = (element.text).split()
self.saveUserPath = settings['SAVE_USER_FOLLOWERS_PATH']
mkdirs(self.saveUserPath)
savePath = os.path.join(self.saveUserPath, "following" + ".txt")
with open(savePath, 'a+') as f:
for i in info:
if i.startswith('@') and len(i) > 1 and i != self.query:
f.write(i)
f.write("\n")
savePath = os.path.join(self.saveUserPath, "scrapped_users" + ".txt")
with open(savePath, 'a+') as f:
f.write(self.query)
f.write("\n")
def commentary_extraction_onefootball(link):
print(link)
scroll_wait = 1.5
match_dict = {}
try:
driver = webdriver.Firefox()
driver.get(link)
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME, "of-match-live-experience")))
load_more = driver.find_element_by_xpath('/html/body/div[6]/div/div/div/div/div/div[1]/div/section/div[8]/article/of-pager/div/div/div/button')
driver.execute_script('window.scrollTo(0,arguments[0]);', load_more.location['y']-100)
WebDriverWait(driver, 1).until(EC.visibility_of(load_more))
load_more_position = load_more.location['y']
scroll_position = 100
while scroll_position < load_more_position - 150:
driver.execute_script('window.scrollTo(0,arguments[0]);', scroll_position)
WebDriverWait(driver, 0.5)
scroll_position += 30 + randint(-10, 10)
driver.execute_script('window.scrollTo(0,arguments[0]);', load_more_position - 150)
load_more = driver.find_element_by_xpath('/html/body/div[6]/div/div/div/div/div/div[1]/div/section/div[8]/article/of-pager/div/div/div/button')
WebDriverWait(driver, 1)
load_more_click = action_chains.ActionChains(driver).move_to_element(load_more).click(load_more)
load_more_click.perform()
# driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
WebDriverWait(driver, scroll_wait)
last_height = driver.execute_script("return window.pageYOffset")
while True:
driver.execute_script("window.scrollTo(0, arguments[0]);", scroll_position)
WebDriverWait(driver, scroll_wait)
new_height = driver.execute_script("return window.pageYOffset")
scroll_position += 100
if new_height == last_height:
break
last_height = new_height
hand_over = driver.page_source
finally:
driver.close()
page_soup = bs.BeautifulSoup(hand_over, 'lxml')
comment_soup = page_soup.find_all('li')
match_date = comment_soup[0].find('time').get('datetime')
commentary = []
for child in comment_soup:
timestamp = child.find(class_='time text-center').string
comment = child.find(class_='action-description').string
commentary.append([str(timestamp), str(comment)])
match_dict["commentary onefootball"] = commentary
match_info_soup = page_soup.find(class_="content-header-body").find(class_="of-row pt-lg pb-none")
home_team = match_info_soup.find(class_="team team-home").get('title')
away_team = match_info_soup.find(class_="team team-away").get('title')
score = match_info_soup.find(class_="game-result mt-sm mb-sm text-center").string
home_score, away_score = score.split(' : ')
match_dict["home team"] = str(home_team)
match_dict["away team"] = str(away_team)
match_dict["home score"] = str(home_score)
match_dict["away score"] = str(away_score)
#match_date = page_soup.find('div', attrs = {'ng-if': '::vm.displayDate'}).find(class_="m-none game-info-small-box").string
match_dict["match date"] = str(datetime.strptime(match_date, '%Y-%m-%dT%H:%M:%SZ').date())
return match_dict
def doubleclick_rte_element(self, elt_css_selector, index=0):
iframe = self.find_element_by_css_selector('.yui-editor-editable',
index=index)
self._tester.driver.switch_to_frame(iframe)
target = self.find_element_by_css_selector(elt_css_selector)
action_chains.ActionChains(
self._tester.driver).double_click(target).perform()
self._tester.driver.switch_to_default_content()
return self
def verify_hover_arrow(self, column, sort_dir, arrow):
col_hdr_selector = self._scrolled_col_hdr_id_sel(column)
fixed_hdr_selector = self._fixed_col_hdr_id_sel(column)
col_hdr_th = self.find_element_by_css_selector(col_hdr_selector)
action_chains.ActionChains(
self._tester.driver).move_to_element(col_hdr_th).perform()
self._check_hover_arrow_state(col_hdr_selector, sort_dir, arrow)
self._check_hover_arrow_state(fixed_hdr_selector, sort_dir, arrow)
return self
def tooltip_match_pops_up(driver):
count_div = self.find_element_by_css_selector(count_div_selector)
action_chains.ActionChains(
self._tester.driver).move_to_element(count_div).perform()
tooltip_div = self.find_element_by_css_selector(tooltip_selector)
match = re.match(tooltip, tooltip_div.text.strip())
if not match:
time.sleep(reload_sleep)
self.load(self._base_url)
return match
