def test_solution_cell(browser):
_load_notebook(browser)
_activate_toolbar(browser)
# make sure the toolbar appeared
_wait(browser).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".celltoolbar select")))
# does the nbgrader metadata exist?
assert _get_metadata(browser) is None
# make it a solution cell
_select_solution(browser)
assert _get_metadata(browser)['solution']
assert not _get_metadata(browser)['grade']
assert not _get_metadata(browser)['locked']
# wait for the id field to appear
_wait(browser).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id")))
# set the id
_set_id(browser)
assert "foo" == _get_metadata(browser)['grade_id']
# make it nothing
_select_none(browser)
assert not _get_metadata(browser)['solution']
assert not _get_metadata(browser)['grade']
assert not _get_metadata(browser)['locked']
def test_tests_cell(browser, port):
_load_notebook(browser, port)
_activate_toolbar(browser)
# does the nbgrader metadata exist?
assert _get_metadata(browser) is None
# make it autograder tests
_select_tests(browser)
assert not _get_metadata(browser)['solution']
assert _get_metadata(browser)['grade']
assert _get_metadata(browser)['locked']
# wait for the points and id fields to appear
_wait(browser).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-points")))
_wait(browser).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id")))
WebDriverWait(browser, 30).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".lock-button")))
# set the points
_set_points(browser)
assert 2 == _get_metadata(browser)['points']
# set the id
_set_id(browser)
assert "foo" == _get_metadata(browser)['grade_id']
# make it nothing
_select_none(browser)
assert not _get_metadata(browser)['solution']
assert not _get_metadata(browser)['grade']
assert not _get_metadata(browser)['locked']
def findLineOfCertainStatus(driver, status):
try:
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div[@class='table']/div/div/table/tbody")))
except TimeoutException as e:
logCommon.error('None unacked records in the table.')
tbody = driver.find_element_by_xpath("//div[@class='table']/div/div/table/tbody")
trs = WebDriverWait(tbody, 10).until(EC.presence_of_all_elements_located((By.XPATH, ".//tr")))
# trs = tbody.find_elements_by_xpath(".//tr")
for tr in trs:
# tds = tr.find_elements_by_xpath(".//td")
tds = WebDriverWait(tr, 10).until(EC.presence_of_all_elements_located((By.XPATH, ".//td")))
if (0 == status):
if ("未清除" == tds[5].get_attribute("innerHTML").encode('utf-8').strip()):
print "告警代码：" + tds[1].get_attribute("innerHTML").encode('utf-8') + " 未清除"
return tr
elif (2 == status):
if ("未确认" == tds[6].get_attribute("innerHTML").encode('utf-8').strip()):
# print "告警代码：" + tds[1].get_attribute("innerHTML").encode('utf-8') + " 未确认"
logCommon.info("ID: " + str(tds[1].get_attribute("innerHTML").encode('utf-8')) + " not acked")
return tr
elif (4 == status):
if ("未确认" == tds[6].get_attribute("innerHTML").encode('utf-8').strip() and "未清除" == tds[5].get_attribute(
"innerHTML").encode('utf-8').strip()):
print "告警代码：" + tds[1].get_attribute("innerHTML").encode('utf-8') + " 未确认并且未清除"
return tr
return False
def test_check_countries(self):
wait = WebDriverWait(self.driver, 5)
self.sign_in()
self.driver.get(COUNTRIES_PAGE)
current_countries = []
not_null_zones_country_links = []
country_rows = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".row")))
for row in country_rows:
country = row.find_element_by_css_selector("td:nth-child(5) a")
current_countries.append(country.text)
zones = row.find_element_by_css_selector("td:nth-child(6)")
if int(zones.text) > 0:
not_null_zones_country_links.append(country.get_attribute("href"))
expected_countries = sorted(current_countries)
self.assertListEqual(current_countries, expected_countries)
for link in not_null_zones_country_links:
self.driver.get(link)
current_country_zones = [x.text for x in wait.until(EC.presence_of_all_elements_located(
(By.XPATH, "(.//*[@id=\"table-zones\"]//td[3])[position() < last()]")
))]
expected_country_zones = sorted(current_country_zones)
self.assertListEqual(current_country_zones, expected_country_zones)
def test_locked_cell(browser, port):
_load_notebook(browser, port)
_activate_toolbar(browser)
# does the nbgrader metadata exist?
assert _get_metadata(browser) is None
# make it locked
_select_locked(browser)
assert not _get_metadata(browser)['solution']
assert not _get_metadata(browser)['grade']
assert _get_metadata(browser)['locked']
# wait for the id and lock button to appear
WebDriverWait(browser, 30).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id")))
WebDriverWait(browser, 30).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".lock-button")))
# set the id
_set_id(browser)
assert "foo" == _get_metadata(browser)['grade_id']
# make sure the metadata is valid
_save_and_validate(browser)
# make it nothing
_select_none(browser)
assert not _get_metadata(browser)
_save_and_validate(browser)
def test_manual_cell(browser, port):
_load_notebook(browser, port)
_activate_toolbar(browser)
# does the nbgrader metadata exist?
assert _get_metadata(browser) is None
# make it manually graded
_select_manual(browser)
assert _get_metadata(browser)['solution']
assert _get_metadata(browser)['grade']
assert not _get_metadata(browser)['locked']
# wait for the points and id fields to appear
_wait(browser).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-points")))
_wait(browser).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id")))
# set the points
_set_points(browser)
assert 2 == _get_metadata(browser)['points']
# set the id
_set_id(browser)
assert "foo" == _get_metadata(browser)['grade_id']
# make sure the metadata is valid
_save_and_validate(browser)
# make it nothing
_select_none(browser)
assert not _get_metadata(browser)
_save_and_validate(browser)
def test_correct_tabs_are_displayed(self):
"""Tabs with the class admin-only should not be displayed, and tabs
with the class teacher-only should be displayed
"""
try:
admin_only_elements = WebDriverWait(self.browser, 10).until(
expected_conditions.presence_of_all_elements_located((By.CLASS_NAME, "admin-only"))
)
except TimeoutException:
admin_only_elements = []
teacher_only_elements = WebDriverWait(self.browser, 10).until(
expected_conditions.presence_of_all_elements_located((By.CLASS_NAME, "teacher-only"))
)
# Make sure nav bar is expanded e.g. in a small screen
try:
navbar_expand = self.browser.find_element_by_class_name("navbar-toggle")
self.browser_activate_element(elem=navbar_expand)
# Wait for the animation to finish
WebDriverWait(self.browser, 3).until(
expected_conditions.visibility_of_element_located((By.CLASS_NAME, "nav"))
)
except ElementNotVisibleException:
# browser_activate_element could throw this, meaning nav bar is already visible
pass
for el in admin_only_elements:
self.assertFalse(el.is_displayed(), "Elements with `admin-only` class should not be displayed!")
for el in teacher_only_elements:
self.assertTrue(el.is_displayed(), "Elements with `teacher-only` class should be displayed!")
def login(self):
url = 'https://mp.weixin.qq.com/'
self._browser.get(url)
self._browser.implicitly_wait(3)
try:
input_username = self._wait.until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_input_panel > div:nth-child(1) > div > span > input")))
input_username[0].clear()
input_username[0].send_keys(self.username)
input_password = self._wait.until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_input_panel > div:nth-child(2) > div > span > input")))
input_password[0].clear()
input_password[0].send_keys(self.password)
##记住我
remember_me = self._wait.until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_help_panel > label > i")))
remember_me[0].click()
submit = self._wait.until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_btn_panel > a")))
submit[0].click()
time.sleep(10)
##存储cookies到本地
self._browser.get(url)
for cookie_item in self._browser.get_cookies():
self._cookie[cookie_item['name']] = cookie_item['value']
cookie_str = json.dumps(self._cookie)
with open(path.join(path.abspath('.'),'cookie.txt'), 'w+', encoding='utf-8') as f:
f.write(cookie_str)
except TimeoutException:
print("Time Out")
def data_extractor(input_driver):
"""
HELPER FUNCTION
takes the driver object and extracts the page content data and cleans it if
present and loaded correctly then returns the processed data and
state : True/False
(driver object) --> (boolean, dictionary)
"""
try:
final_dict = {}
# retrieve data by class Name for the name fields
name_list = [' ']
namefields = waiter(input_driver, 5).until(
EC.presence_of_all_elements_located(
(By.CLASS_NAME, "RightAlignBlack")))
for name in namefields:
name_list.append(str(name.text))
#data corresponding to name fields
data_list = []
datafields = waiter(input_driver, 5).until(
EC.presence_of_all_elements_located(
(By.CLASS_NAME, "LeftAlignGreen")))
for data in datafields:
data_str = str(data.text)
if data_str != ' : ':
data_list.append(data_str)
#handling the special case of the name field: Standard Seating Capacity
capacity_namexpath = """
//*[@id="ctl00_UpdatePanel1"]/table/tbody/tr[5]/td/table/tbody/tr[6]
/td[2]/fieldset/table/tbody/tr[3]/td[5]"""
capacity_namefield = waiter(input_driver, 5).until(
EC.presence_of_element_located((By.XPATH, capacity_namexpath)))
cs_namefield = str(capacity_namefield.text)
#getting special case of the data field for Standard Seating Capacity
capacity_dataxpath = """
//*[@id="ctl00_UpdatePanel1"]/table/tbody/tr[5]/td/table/tbody/tr[6]
/td[2]/fieldset/table/tbody/tr[3]/td[7]"""
capacity_datafield = waiter(input_driver, 5).until(
EC.presence_of_element_located((By.XPATH, capacity_dataxpath)))
cs_datafield = str(capacity_datafield.text)
#creating the clean data and enter in dictionary
pos = 0
offset = 0
for nameval in name_list:
if pos == 0 or pos == 1:
pos += 1
continue
if cs_datafield == data_list[pos + offset]:
offset += 1
final_dict[nameval] = data_list[pos + offset]
pos += 1
#finally add the exception case data to dictionary
final_dict[cs_namefield] = cs_datafield
return(True, final_dict)
except TimeoutException:
return(False, None)
def find_elements(self,by,locator,timeout=configfile.timeout_find_element):
if by == 'id':
elements = WebDriverWait(self.driver, timeout).until(
EC.presence_of_all_elements_located((By.ID,locator)))
elif by == 'xpath':
elements = WebDriverWait(self.driver, timeout).until(
EC.presence_of_all_elements_located((By.XPATH,locator)))
elif by == 'name':
elements = WebDriverWait(self.driver, timeout).until(
EC.presence_of_all_elements_located((By.NAME,locator)))
elif by == 'class':
elements = WebDriverWait(self.driver, timeout).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,locator)))
return elements
def clico_menu_dashboards(context):
#esperando elementos do menu
wait = WebDriverWait(context.driver, 10)
wait.until(EC.presence_of_all_elements_located((By.XPATH, "//div[contains(@class, 'x-btn x-box-item x-toolbar-item x-btn-custom-toolbar-small x-noicon x-btn-noicon x-btn-custom-toolbar-small-noicon')]")))
#selencionado o menu admnistração
menu_lista = context.driver.find_elements_by_xpath("//div[contains(@class, 'x-btn x-box-item x-toolbar-item x-btn-custom-toolbar-small x-noicon x-btn-noicon x-btn-custom-toolbar-small-noicon')]")
admin = list(menu_lista)
menu_admin = admin[1]
menu_admin.click()
wait = WebDriverWait(context.driver, 20)
wait.until(EC.presence_of_all_elements_located((By.XPATH, "//span[contains(@class, 'x-menu-item-text')]")))
menu_admin_lista = list(context.driver.find_elements_by_xpath("//span[contains(@class, 'x-menu-item-text')]"))
menu_admin_dashboards = menu_admin_lista[9]
menu_admin_dashboards.click()
def Click_Element_By_Link_Text(_name,parent=False,uppercase=False):
'''
Use this function only if you are sure that there wont be any conflicting Name.
If possible use Click_Element_By_ID
'''
sModuleInfo = inspect.stack()[0][3] + " : " + inspect.getmoduleinfo(__file__).name
try:
CommonUtil.TakeScreenShot(sModuleInfo, local_run)
#Find all elements containing the name
CommonUtil.ExecLog(sModuleInfo, "Trying to find element by Link Text: %s"%_name, 1,local_run)
if isinstance(parent, (bool)) == True:
if uppercase:
allElements = WebDriverWait(sBrowser, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT, _name.upper())))
else:
allElements = WebDriverWait(sBrowser, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT,_name)))
else:
if uppercase:
allElements = WebDriverWait(parent, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT, _name.upper())))
else:
allElements = WebDriverWait(parent, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT, _name)))
if allElements == []:
CommonUtil.ExecLog(sModuleInfo, "Could not find your element by Link Text: %s"%_name, 3,local_run)
return "failed"
else:
if len(allElements) > 1:
CommonUtil.ExecLog(sModuleInfo, "Found more than one element and will use the first one. ** if fails, try providing parent element or try by ID** ", 2, local_run)
for each in allElements:
if uppercase:
if (WebDriverWait(each, WebDriver_Wait).until(lambda driver : each.text==_name.upper())) == True:
Element = each
CommonUtil.ExecLog(sModuleInfo, "Found your element by Link Text: %s. Using the first element found to click"%_name, 1,local_run)
break
#Now we simply click it
else:
if (WebDriverWait(each, WebDriver_Wait).until(lambda driver : each.text==_name)) == True:
Element = each
CommonUtil.ExecLog(sModuleInfo, "Found your element by Link Text: %s. Using the first element found to click"%_name, 1,local_run)
break
Element.click()
CommonUtil.TakeScreenShot(sModuleInfo, local_run)
CommonUtil.ExecLog(sModuleInfo, "Successfully clicked your element: %s"%_name, 1,local_run)
return "passed"
except Exception, e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
Error_Detail = ((str(exc_type).replace("type ", "Error Type: ")) + ";" + "Error Message: " + str(exc_obj) +";" + "File Name: " + fname + ";" + "Line: "+ str(exc_tb.tb_lineno))
CommonUtil.ExecLog(sModuleInfo, "Unable to expand menu: %s. Error: %s"%(_name,Error_Detail), 3,local_run)
return "failed"
def get_img_code(self):
global driver
if driver:
driver.close()
firefox_path = self.find_firefox_path()
driver = webdriver.Firefox(executable_path=firefox_path)
driver.maximize_window()
actions = ActionChains(driver)
url = "https://inv-veri.chinatax.gov.cn/index.html"
driver.get(url) #打开网页
WebDriverWait(driver, 5, 0.5).until(ec.presence_of_all_elements_located((By.ID, "kprq")))
fpdm = driver.find_element_by_id("fpdm")
fpdm.send_keys(self.invoice_code)
time.sleep(1)
actions.click(fpdm)#提交发票代码
fphm = driver.find_element_by_id("fphm")
fphm.send_keys(self.name) #提交发票号码
date = self.invoice_date.replace('-', '') # 转换日期格式
date_in = driver.find_element_by_id("kprq")#提交发票日期
date_in.clear()
date_in.send_keys(date)
if self.invoice_type == 'zy':
driver.find_element_by_id("kjje").send_keys(str(self.invoice_amount))#提交发票金额
if self.invoice_type == 'pt':
driver.find_element_by_id("kjje").send_keys(str(self.invoice_heck_code)[-6:]) # 提交校验码后六位
if self.invoice_type == 'dz':
driver.find_element_by_id("kjje").send_keys(str(self.invoice_heck_code)[-6:]) # 提交校验码后六位
time.sleep(1)
WebDriverWait(driver, 5, 0.5).until(ec.presence_of_all_elements_located((By.ID, "yzminfo")))
# 取图片
try:
imgelement = driver.find_element_by_id("yzm_img")
except:
actions.click(fpdm)
actions.click(fphm)
imgelement = driver.find_element_by_id("yzm_img")
finally:
img = imgelement.get_attribute('src').split(",")[-1]
# 取图片说明
img_note = driver.find_element_by_id("yzminfo").text
print img
self.write({'img': img, 'img_note': img_note})
# 处理base64代码不足报错
missing_padding = 4 - len(img) % 4
if missing_padding:
img += b'=' * missing_padding
imgdata = base64.b64decode(img)
file = open('./log/out.png', 'wb')
file.write(imgdata)
def parseItem(self, url):
'''
Parsing item with url
Args: url of item
url: url of item
Returns: item, in dictionary
'''
# take care of videos when choosing image
self.web_driver.get(url)
try:
# so after all this is probably the right way to find elements
elem = WebDriverWait(self.web_driver, 10).until(
EC.presence_of_all_elements_located((By.XPATH, '//input[contains(@class, "thumbs")]'))
)
except Exception as e:
print e
item = {}
item["detail_image_path"] = ""
images = []
for i in elem:
images.append(i.get_attribute('src'))
item["detail_image_path"] = "|".join(images)
item["suitable_images_index"] = 1
item['white_suitable_images_index'] = 0
item['white_suitable_images'] = ""
try:
# so after all this is probably the right way to find elements
elem = WebDriverWait(self.web_driver, 10).until(
EC.presence_of_all_elements_located((By.XPATH, '//div[contains(@id, "tabWindow")]'))
)
except Exception as e:
print e
if len(elem) >= 1:
item["info"] = elem[0].text
try:
# so after all this is probably the right way to find elements
elem = WebDriverWait(self.web_driver, 10).until(
EC.presence_of_all_elements_located((By.XPATH, '//p[contains(@class, "description")]'))
)
except Exception as e:
print e
if len(elem)>=1:
item["description"] = elem[0].text
return item
def add_to_cart(self, index):
self.wait.until(EC.presence_of_all_elements_located(
(By.CLASS_NAME, 'js-product-to-cart')
))[index].click()
self.wait.until(EC.visibility_of_element_located(
(By.CLASS_NAME, 'js-cart-wrapper')
))
def getupc(data, sleeptime):
display = Display(visible=0, size=(800, 600))
display.start()
a = webdriver.Firefox()
a.get('https://www.google.com/ncr')
time.sleep(sleeptime)
search = WebDriverWait(a, 5).until(EC.element_to_be_clickable((By.XPATH, "//input[@type='text']")))
for i in data:
ActionChains(a).move_to_element(search).click(search).send_keys(i['name'] + ' upc', Keys.ENTER).perform()
time.sleep(sleeptime)
contents = WebDriverWait(a, 5).until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='g']")))
try:
upc = next(
(re.split(r'/', href.find_element_by_tag_name('a').get_attribute('href'))[-1] for
href in contents if
href.find_element_by_tag_name('a').get_attribute('href').startswith(
'http://www.upcitemdb.com/upc')))
i['upc'] = upc
except StopIteration:
pass
search = WebDriverWait(a, 5).until(EC.element_to_be_clickable((By.XPATH, "//input[@type='text']")))
search.clear()
a.close()
display.stop()
return data
def getHtml(word,driver):
driver.get('http://www.shufazidian.com')
elem = driver.find_element_by_name('wd')
select = Select(driver.find_element_by_name('sort'))
select.select_by_value('9')
# print('1, ', ctime())
elem.send_keys(word)
# wait = WebDriverWait(driver,5)
elem.send_keys(Keys.RETURN)
# driver.implicitly_wait(5)
# try:
# element = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.mbpho')))
# print('3, ',ctime())
# except Exception,e:
# print(e)
# print('4, ', ctime())
delay = 7 # seconds
try:
WebDriverWait(driver, delay).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.mbpho')))
except TimeoutException:
# print >>f , "TimeoutException Loading took too much time!"
pass
time.sleep(5)
return driver.page_source
def find_paper_in_scholar(cnx, title, id, key):
driver.get(SCHOLAR)
try:
search_box = WebDriverWait(driver, WAIT_TIME).until(EC.presence_of_element_located((By.ID, "gs_hp_tsi")))
search_box.send_keys(title + Keys.RETURN)
if in_captcha_page():
print "redirected to captcha"
try:
#wait
scholar_hits = WebDriverWait(driver, WAIT_TIME).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "gs_ri")))
time.sleep(2)
for hit in scholar_hits:
info_hit = hit.find_element_by_class_name("gs_rt")
try:
hit_link = info_hit.find_element_by_tag_name("a")
title_hit = hit_link.text
leveh_distance = nltk.metrics.edit_distance(re.sub(r'\s+', '', title_hit.lower()), re.sub(r'\s+', '', title.lower()))
if leveh_distance <= 6:
if leveh_distance > 1:
logging.warning("match: " + title_hit + " ******* " + title + " ******* " + str(leveh_distance))
add_abstract_to_paper(hit_link, cnx, id, key)
break
except NoSuchElementException:
continue
except TimeoutException:
logging.warning("abstract not found for " + title)
except TimeoutException:
logging.info("Google Scholar not loaded")
def FindCssElements(self, css, obj=None):
"""Find web elements using CSS elements.
Args:
css: string, css elements to search for.
obj: web element object to search within.
Returns:
list of web element objects that contain css.
"""
try:
self.GetWait().until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,
css)))
except TimeoutException:
self.logger.error('Timed out looking for css: %s', css)
return None
else:
try:
if obj:
elements = obj.find_elements_by_css_selector(css)
else:
elements = self.driver.find_elements_by_css_selector(css)
except NoSuchElementException:
self.logger.error('Element css %s not found', css)
return None
return elements
def next_level(self):
self.assert_success()
self.browser.find_element_by_id("next_level_button").click()
WebDriverWait(self.browser, 10).until(
presence_of_all_elements_located((By.ID, "blockly_tab"))
)
return self
def run_alarms(self, driver, test_log):
gui_lib = Utils(driver)
self.login_manager.login()
test_log.start('Alarms')
test_helper = TestHelper(test_log, driver)
driver.switch_to_default_content()
gui_lib.click_element('menu_node_status')
gui_lib.click_element('menu_node_alarms')
driver.switch_to_frame('frame_content')
# t = driver.find_element(By.XPATH, "//body/fieldset/legend").text
# print('test', t)
buttons = driver.find_elements_by_tag_name('button')
# try test for visibility on the function below as it finds the buttons
# before they are onscreen
# (EC.visibility_of_element_located(
WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.TAG_NAME, 'button')))
# check that there are 3 buttons
test_helper.assert_true(len(buttons) != 3, 'Not all buttons displayed', 'Check that all buttons displayed')
# test that the buttons have text on them
for btn in buttons:
test_helper.assert_true(len(btn.text.strip()) == 0, 'No text on button', 'Check buttons have text')
print(len(btn.text.strip()))
# test_log.close()
self.login_manager.logout()
def browser_check_django_message(self, message_type=None, contains=None, exact=None, num_messages=1):
"""Both central and distributed servers use the Django messaging system.
This code will verify that a message with the given type contains the specified text."""
# Get messages (and limit by type)
if num_messages > 0:
messages = WebDriverWait(self.browser, 5).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "alert")))
else:
messages = []
if isinstance(contains, str): # make sure we only look at messages we're looking for.
messages = [message for message in messages if contains in message.text]
# Check that we got as many as expected
if num_messages is not None:
msg = "Make sure there are %d message(s), type='%s'." % \
(num_messages, message_type if message_type else "(any)")
self.assertEqual(num_messages, len(messages), msg)
for i, message in enumerate(messages):
if type(contains) == list:
contain = contains[i]
else:
contain = contains
if type(exact) == list:
exac = exact[i]
else:
exac = exact
if contain is not None:
self.assertIn(contain, message.text, "Make sure message contains '%s'" % contain)
if exac is not None:
self.assertEqual(exac, message.text, "Make sure message = '%s'" % exac)
def FindClasses(self, classname, obj=None):
"""Find all web elements with classname.
Args:
classname: string, class to search for.
obj: web element object to search within.
Returns:
list of web element objects containing classname.
"""
try:
self.GetWait().until(EC.presence_of_all_elements_located((By.CLASS_NAME,
classname)))
except TimeoutException:
self.logger.error('Timed out looking for class: %s', classname)
return None
else:
try:
if obj:
elements = obj.find_elements_by_class_name(classname)
else:
elements = self.driver.find_elements_by_class_name(classname)
except NoSuchElementException:
self.logger.error('Error finding %s class name.', classname)
return None
return elements
def FindTags(self, tagname, obj=None):
"""Find web elements using tagname.
Args:
tagname: string, tag to search for.
obj: web element object to search within.
Returns:
list of web element objects containing tagname.
"""
try:
self.GetWait().until(EC.presence_of_all_elements_located((By.TAG_NAME,
tagname)))
except TimeoutException:
self.logger.error('Timed out finding names: %s', tagname)
return None
else:
try:
if obj:
elements = obj.find_elements_by_tag_name(tagname)
else:
elements = self.driver.find_elements_by_tag_name(tagname)
except NoSuchElementException:
self.logger.error('Error finding tag: %s', tagname)
return None
return elements
def action_on_richdata(driver):
""" Method to perform actions on rich data"""
try:
WebDriverWait(driver, 30).until(EC.presence_of_all_elements_located(driver.find_elements_by_xpath("//*")))
except:
wait_to_load_webelement(driver,"xpath","//*")
##wait_to_load_webelement(driver,"tagname","a")
all_elements_list = driver.find_elements_by_tag_name("a")
## time.sleep(8)
for item_all_elements_list in all_elements_list:
if str(item_all_elements_list.get_attribute("class")).split("-")[0] == 'colorbox':
try:
element = item_all_elements_list.find_element_by_tag_name("img")
element.click()
## close_button = self.driver.find_element_by_class_name("fancybox-close")
## close_button.click()
time.sleep(5)
except :
item_all_elements_list.click()
time.sleep(5)
wait_to_load_webelement(driver, "id", "fancybox-close")
## time.sleep(2)
close_button = driver.find_elements_by_class_name("fancybox-close")
for btn in close_button:
btn.click()
break
time.sleep(5)
def size_options(self):
try:
return self.wait.until(EC.presence_of_all_elements_located(
self.SIZE_OPTIONS
))
except TimeoutException:
return []
def test_check_all_admin_pages(self):
wait = WebDriverWait(self.driver, 5)
self.driver.get(SIGN_IN_PAGE)
self.driver.find_element_by_name("username").send_keys(LOGIN)
self.driver.find_element_by_name("password").send_keys(PASSWORD)
self.driver.find_element_by_name("login").click()
first_menu_elements = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#app->a")))
first_menu_links = [x.get_attribute("href") for x in first_menu_elements]
for link in first_menu_links:
self.driver.get(link)
first_menu_item_header = self.driver.find_elements_by_css_selector("#content>h1")
self.assertTrue(first_menu_item_header)
second_menu_elements = self.driver.find_elements_by_css_selector(".docs a")
second_menu_links = [x.get_attribute("href") for x in second_menu_elements]
if second_menu_elements:
for link in second_menu_links:
self.driver.get(link)
second_menu_item_header = self.driver.find_elements_by_css_selector("#content>h1")
self.assertTrue(second_menu_item_header)
def test_question_page(self):
# login
self.client.get('http://*****:*****@example.com')
self.client.find_element_by_name('password').send_keys('cat')
self.client.find_element_by_id('submit').click()
# navigate to a question page
time.sleep(2)
self.client.find_element_by_id('explore').click()
question_links = WebDriverWait(self.client, 10).until(
EC.presence_of_all_elements_located((
By.CLASS_NAME, 'question_link'))
)
random.choice(question_links).click()
# follow the question
WebDriverWait(self.client, 10).until(
lambda c: len(c.window_handles) == 2)
self.client.switch_to_window(self.client.window_handles[-1])
time.sleep(5)
follow_button = WebDriverWait(self.client, 10).until(
EC.presence_of_element_located((
By.XPATH, '//div[@class="side-section"]/button'))
)
follow_button.click()
time.sleep(5)
self.assertTrue('btn-white' in follow_button.get_attribute('class'))
def sell_orders():
scroll_down()
#xpath = '//tr[contains(@class,"sell_tr")]'
while True:
logging.debug("Getting sell rows")
xpath = '//table[@id="md-sell"]/tbody/tr[position() <= 5]'
trs = WebDriverWait(driver, 20).until(
EC.presence_of_all_elements_located((By.XPATH, xpath)))
trs = reversed(trs)
logging.debug("Looping through rows")
try:
tr = trs.next()
_tds = tr.find_elements_by_tag_name('td')
tds = []
print "tds = {0}".format(pp.pformat(_tds))
for td in _tds:
tds.append(float(td.text))
so = SellOrder(*tds[0:2])
logging.debug("yielding sell order: {0}".format(so))
yield so
except StaleElementReferenceException:
logging.debug("StaleElementReferenceException!")
continue
def parseC(self, response):
chemical_item = Chemical()
###Create Item
###Name
name_url = response.url + "#section=Top"
self.sel.get(name_url)
time.sleep(4)
chemical_item['name']= WebDriverWait(self.sel, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div/div/div[1]/div[2]/div/h1/span'))).text
###CAS
cas_url = response.url + "#section=CAS"
self.sel.get(cas_url)
time.sleep(1)
chemical_item['cas'] = WebDriverWait(self.sel, 5).until(EC.presence_of_element_located((By.XPATH,'//ol/li[3]/ol/li[3]/ol/li[1]/div[2]/div[1]'))).text
###GHS CLASSIFICATION
hazard_url = response.url + "#section=GHS-Classification"
self.sel.get(hazard_url)
time.sleep(1)
try:
thumbnail_container = WebDriverWait(self.sel, 5).until(EC.presence_of_all_elements_located((By.XPATH,'//ol/li[1]/ol/li[1]/div[2]/div[1]/div/img')))
urls = []
for element in thumbnail_container:
urls.append(element.get_attribute('src'))
chemical_item['image_urls']=urls
except TimeoutException:
chemical_item['image_urls']= []
self.chemical_info.append(chemical_item)
time.sleep(3)
return chemical_item
def presence_of_all_elements_located(self, selector):
'''判断是否至少有1个元素存在于dom树中，如果定位到就返回列表'''
locator = self.handle_locator(selector)
return WebDriverWait(self.driver, 10).until(
EC.presence_of_all_elements_located(locator))
def test_edit_promotion_method_(self):
driver = self.driver
self.authorizatuion(driver)
WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//a[@href='/superadmin/promotions']"))).click()
time.sleep(2)
list_promotions = WebDriverWait(driver, 10).until(ec.presence_of_all_elements_located((By.XPATH, "//mat-row[@class='mat-row']")))
list_promotions[randint(0, len(list_promotions)-1)].click()
time.sleep(2)
#WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//*[@id='portal-classic-content']/app-super-promotion-view/app-promotion-view/div/mat-card/mat-card-content/div[1]/form/div[1]/app-gallery-picker/div/div[1]/button"))).click()
#time.sleep(2)
file_dicitionary = {0: "/Users/rufina/Desktop/dishs/BjJ6inaYiWam0GGViLFHLQ-double.jpg",
1: "/Users/rufina/Desktop/dishs/4Rve51WmWfk.jpg", 2: "/Users/rufina/Desktop/dishs/2531.jpg",
3: "/Users/rufina/Desktop/dishs/salat_kinoa.jpg__1499258543__50030.jpg",
4: "/Users/rufina/Desktop/dishs/4703.jpg", 5: "/Users/rufina/Desktop/dishs/caption (1).jpg"}
#WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//input[@type='file']"))).send_keys(file_dicitionary[randint(0, len(file_dicitionary) - 1)])
#time.sleep(2)
name_promotion = WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//input[@placeholder='Название акции']")))
name_promotion.clear()
name_promotion.send_keys(self.my_metho_randem_stroka(randint(4, 7), 3))
time.sleep(2)
text_promotion = WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//textarea[@placeholder ='Описание акции']")))
text_promotion.clear()
text_promotion.send_keys(self.my_metho_randem_stroka(randint(5,10), randint(3, 5)))
time.sleep(2)
WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//input[@placeholder='От']"))).click()
time.sleep(2)
list_dates_from = WebDriverWait(driver, 10).until(ec.presence_of_all_elements_located((By.XPATH, "//td[@class='mat-calendar-body-cell ng-star-inserted']")))
list_dates_from[randint(0, len(list_dates_from )-1)].click()
time.sleep(2)
WebDriverWait(driver, 10).until(
ec.presence_of_element_located((By.XPATH, "//input[@placeholder='До']"))).click()
time.sleep(2)
list_dates_to = WebDriverWait(driver, 10).until(
ec.presence_of_all_elements_located((By.XPATH, "//td[@class='mat-calendar-body-cell ng-star-inserted']")))
list_dates_to[randint(0, len(list_dates_to) - 1)].click()
time.sleep(2)
WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//button[@class='mat-button mat-flat-button']"))).click()
time.sleep(5)
def testShouldFailToFindElementsWhenExplicitWaiting(driver, pages):
pages.load("dynamic.html")
with pytest.raises(TimeoutException):
WebDriverWait(driver, 0.7).until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "redbox")))
)
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = USER_AGENT
print('Setting up Phantom JS webdriver...')
###########################################
browser = webdriver.PhantomJS(desired_capabilities=dcap)
print('Fetching - ', TARGET)
############################
browser.get('https://orgsync.com/412/community/calendar?view=upcoming')
print('Waiting for JS to load...')
##################################
wait = WebDriverWait(browser, 10)
wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'osw-events-list-date')))
events_module = browser.find_element_by_class_name('osw-events-index-main')
print('Found events module')
# Constants specific to Orgsync DOM
DATE_CLASS = 'osw-events-list-date'
EVENT_CLASS = 'osw-events-list-item-content'
TITLE_CLASS = 'osw-events-list-date-header'
NAME_CLASS = 'osw-events-list-item-title'
TIME_CLASS = 'osw-events-list-item-time'
ORG_CLASS = 'osw-events-list-item-portal-name'
# Parse remaining with BS4
html = events_module.get_attribute('outerHTML')
soup = BeautifulSoup(html, 'html.parser')
dates = soup.find_all('div', attrs={'class': DATE_CLASS})
def linkedin(url3,searched_word,job_fcn,sen_level):
print('\nLinkedin: Open URL...')
driver = webdriver.Chrome(chrome_driver, options=options)
driver.get(url3)
# driver.maximize_window()
# driver.set_window_size(1470, 1033)
#%% Linkedin: Login
print('Linkedin: Logging in...')
idd = '*****@*****.**'
pw = 'awesomeMe2020'
WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.ID,'username')))
driver.find_element_by_id('username').clear()
driver.find_element_by_id('username').send_keys(idd)
driver.find_element_by_id('password').clear()
driver.find_element_by_id('password').send_keys(pw)
driver.find_element_by_class_name('btn__primary--large.from__button--floating').click()
#%% Linkedin: Click sales Navigator
print('Linkedin: Open Sales Navigator...')
main_page = driver.window_handles[0]
sales_navi = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'global-nav__primary-item.global-nav__spotlight')))
sales_navi.click()
sales_navi_windows_handle = driver.window_handles[1]
driver.switch_to.window(sales_navi_windows_handle)
#%% Linkedin: Click Advanced Search
print('Linkedin: Filling in Advanced search...')
# adv_search = WebDriverWait(driver,5).until(
# EC.presence_of_element_located((By.CLASS_NAME,'artdeco-dropdown__trigger.color-white.t-14.t-bold.artdeco-dropdown__trigger--non-button.flex.align-items-flex-end.artdeco-dropdown__trigger--placement-bottom.ember-view')))
adv_search = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'artdeco-dropdown.artdeco-dropdown--placement-bottom.artdeco-dropdown--justification-left.ember-view')))
adv_search.click()
time.sleep(0.2)
leads = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'artdeco-dropdown__item.artdeco-dropdown__item--is-dropdown.ember-view')))
leads.click()
#%% Linkedin: Obtaining all filter handles
filt_list = [
'custom lists',
'past lead',
'geography',
'relationship',
'industry',
'school',
'profile language',
'first name',
'last name',
'seniority level',
'years in current position',
'years at current company',
'function',
'title current',
'years of experience',
'company current',
'company headcount',
'company type',
'past company',
'became a member',
'groups find people',
'tags',
'posted content keywords',
]
filt_containers = WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'flex.pt4.ph4.pb3.flex-wrap.cursor-pointer')))
geography_handle = filt_containers[2]
# # custom_list = filt_containers[3]
# industry = filt_containers[4]
senioritylevel_handle = filt_containers[9]
function_handle = filt_containers[12]
companycurrent_handle = filt_containers[15]
# for item in filt_containers:
# a =' '.join(item.text.replace('\n', ' ').lower().split()[:2])
# exec(a.replace(' ','')+'=item',globals(),locals())
# exec()
# for selection in filt_list:
# for i in range(0,len(filt_containers)):
# if selection.lower() in filt_containers[i].text.replace('\n', ' ').lower():
# # print(filt_containers[i].text.replace('\n', ' ').lower())
# print(filt_list[i].replace(' ','')+f'_handle=filt_containers[{i}]')
# # exec(filt_list[i].replace(' ','')+f'_handle=filt_containers[{i}]')
# break
#%% Linkedin: Filter by Geography
def geo():
geography_handle.click()
geo_searched_region1 = 'Malaysia'
geo_searched_region2 = 'Singapore'
geo_enter = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'text-input--no-border.mb4.ember-text-field.ember-view')))
geo_enter.clear()
geo_enter.send_keys(geo_searched_region1)
time.sleep(1)
geo1 = driver.find_elements_by_class_name("button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left")
geo1[0].click()
# Entering second location
geo_enter.clear()
geo_enter.send_keys(geo_searched_region2)
time.sleep(0.5)
geo1 = driver.find_elements_by_class_name("button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left")
geo1[0].click()
time.sleep(0.5)
# geo()
#%% Linkedin: Filter by seniority level
print('Linkedin: Filter Seniority level')
senioritylevel_handle.click()
# senior_containers = WebDriverWait(driver,5).until(
# EC.presence_of_all_elements_located((By.CLASS_NAME,'ph4.pb4.ember-view')))
# senior_containers = WebDriverWait(driver,5).until(
# EC.presence_of_all_elements_located((By.CLASS_NAME,'button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left')))
for selection in sen_level:
# print(selection.lower())
senior_containers = WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left')))
for i in range(0,len(senior_containers)):
# print(i)
if selection.lower() in senior_containers[i].text.lower():
senior_containers[i].click()
break
#%% Linkedin: Filter by current company
print('Linkedin: Filter Company')
driver.find_element_by_class_name('advanced-search-filter__list').click()
companycurrent_handle.click()
company_enter = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'text-input--no-border.ember-text-field.ember-view')))
for company in company_names[0:15]:
# print(company)
company_enter.clear()
company_enter.send_keys(company)
company_enter.send_keys(Keys.ENTER)
#%% Linkedin: Filter by function
print('Linkedin: Filter Job function')
function_handle.click()
function_enter = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'text-input--no-border.mb4.ember-text-field.ember-view')))
function_enter.clear()
function_enter.send_keys(job_fcn)
function_click = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left')))
function_click.click()
#%% Linkedin: Start search
print('Linkedin: Start search for suspects')
start_search = driver.find_element_by_class_name('button-primary-medium')
start_search.click()
driver.set_window_size(1657, 923) #just to load all data
time.sleep(2)
#%% Linkedin: Initialisation
name = []
position = []
company = []
location = []
searched_keyword = []
email = []
found_email = "*****@*****.**"
page = 1
#%% Linkedin: Scraping prospects
prospect_container = WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'pv5.ph2.search-results__result-item')))
f = driver.find_element_by_class_name('search-results__pagination-list').find_elements_by_tag_name('li')
print('Linkedin: Webscraping starts')
count= 0
try:
total_page = int(f[-1].find_element_by_tag_name('button').text)
except Exception as e:
print(e)
total_page = 1
finally:
while page <= total_page:
articles = WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'pv5.ph2.search-results__result-item')))
for article in articles:
count = count + 1
ActionChains(driver).move_to_element(article).perform()
"""Name"""
try:
article.find_element_by_class_name('result-lockup__name') # Name
article.find_element_by_class_name('result-lockup__highlight-keyword') # Position
article.find_element_by_class_name('result-lockup__misc-item') # Location
article.find_element_by_class_name('result-lockup__position-company') # Company
except NoSuchElementException:
# print('var not defined')
# name.append("NA")
# position.append("NA")
# location.append("NA")
# company.append("NA")
continue
else:
name.append(article.find_element_by_class_name('result-lockup__name').text)
position.append(article.find_element_by_class_name('result-lockup__highlight-keyword').find_element_by_class_name('t-14.t-bold').text)
location.append(article.find_element_by_class_name('result-lockup__misc-item').text)
company.append(article.find_element_by_class_name('result-lockup__position-company').find_element_by_tag_name('span').text)
searched_keyword.append(searched_word)
email.append(found_email)
print(f'Suspect {count}')
next_page = driver.find_element_by_class_name('search-results__pagination-next-button')
next_page.find_element_by_class_name('v-align-middle').click()
page = page + 1
#%% Linkedin: Close website
driver.close()
driver.switch_to.window(main_page)
driver.close()
return name,position,location,company,searched_keyword,email
def find_elements(self, locator, time=10):
return WebDriverWait(self.driver, time).until(EC.presence_of_all_elements_located(locator),
message=f"Can't find elements by locator {locator}")
def parse_list(self, url, search_company):
"""
解析列表页，获取部分信息
:param url: search url
:return: item（含部分信息）以及详情url
"""
while True:
self.driver.get(url)
current_url = self.driver.current_url
if 'login' in current_url:
self.login()
time.sleep(2)
else:
break
time.sleep(2)
try:
self.detail_yzm()
except Exception:
pass
try:
nodes = self.wait.until(EC.presence_of_all_elements_located((By.XPATH, '//*[@id="searchlist"]//tbody/tr')))
if len(nodes) == 0:
detail_url = None
item = None
return detail_url, item
except Exception as E:
detail_url = None
item = None
return detail_url, item
if not self.accurate:
item = {}
if self.type != '3':
get_company = self.driver.find_element_by_xpath(
'//*[@id="searchlist"]//tr[1]/td[2]/a').text.replace('(', '（').replace(')', '）')
item['search_company'] = search_company
try:
item['company_name'] = get_company
except:
item['company_name'] = ' '
try:
item['another_name'] = self.driver.find_element_by_xpath(
'//*[@id="searchlist"]//tbody/tr[1]//i/..').text
except:
item['another_name'] = ' '
try:
item['company_url'] = self.driver.find_element_by_xpath(
'//*[@id="searchlist"]//tbody/tr[1]/td[2]/a').get_attribute('href')
except:
item['company_url'] = ' '
try:
item['telphone'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]//span[contains(text(),"电话：")]').text.replace('电话：', '')
except:
item['telphone'] = ' '
try:
nodes = self.driver.find_elements_by_xpath('//*[@id="phoneModal"]/div/div/div[2]/div/div[1]')
item['tel_more'] = ';'.join([node.text for node in nodes])
except:
item['tel_more'] = ' '
try:
item['email'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]//p[contains(text(),"邮箱：")]').text.strip().split(' ')[0].replace('邮箱：', '')
except:
item['email'] = ' '
try:
item['法定代表人'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[1]').text.split('注册资本')[0].replace('法定代表人：', '').strip()
except:
item['法定代表人'] = ' '
try:
capital = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[1]/span[1]').text.replace('注册资本：', '')
pattern = re.compile(r'(\d+\.?\d*)')
num = pattern.findall(capital)[0]
if '人民币' in capital:
item['注册资本'] = capital
elif '美元' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*6.8)).replace('美元', '人民币')
elif '日元' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*0.06)).replace('日元', '人民币')
elif '欧元' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*7.9)).replace('欧元', '人民币')
elif '港元' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*0.87)).replace('港元', '人民币')
elif '韩元' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*0.006)).replace('韩元', '人民币')
elif '新加坡元' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*5)).replace('新加坡元', '人民币')
elif '瑞士法郎' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*6.9)).replace('瑞士法郎', '人民币')
elif '新台币' in capital:
item['注册资本'] = capital.replace(num, str(int(num)*0.22)).replace('新台币', '人民币')
else:
item['注册资本'] = capital
except:
item['注册资本'] = ' '
pattern = re.compile(r'(\d+\.?\d*)')
if '股本' not in item['注册资本']:
n = pattern.findall(item['注册资本'])
if n:
item['注册资本（万）'] = n[0]
else:
item['注册资本（万）'] = ' '
else:
item['注册资本（万）'] = ' '
try:
item['成立时间'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[1]/span[2]').text.replace('成立时间：', '').split('-')[0]
except:
item['成立时间'] = ' '
item['成立时间（new）'] = item['成立时间'] + '年'
try:
item['公司地址'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[3]').text.replace('地址：', '')
except:
item['公司地址'] = ' '
try:
item['状态'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[3]/span').text
except:
item['状态'] = ' '
detail_url = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/a'))).get_attribute('href')
return detail_url, item
# 精准匹配
if self.accurate == 'on':
for node in nodes:
company = node.find_element_by_xpath('./td[2]//a').text
if search_company.replace('（', '(').replace('）', ')') == company.replace('（', '(').replace('）', ')'):
item = {}
if self.type != '3':
item['search_company'] = search_company
item['company_name'] = company.replace('(', '（').replace(')', '）')
try:
item['company_url'] = node.find_element_by_xpath(
'./td[2]/a').get_attribute('href')
except:
item['company_url'] = ' '
try:
item['another_name'] = node.find_element_by_xpath(
'.//i/..').text
except:
item['another_name'] = ' '
try:
item['telphone'] = node.find_element_by_xpath(
'.//span[contains(text(),"电话：")]').text.strip().replace('电话：', '')
except:
item['telphone'] = ' '
try:
nodes = self.driver.find_elements_by_xpath('//*[@id="phoneModal"]/div/div/div[2]/div/div[1]')
item['tel_more'] = ';'.join([node.text for node in nodes])
except:
item['tel_more'] = ' '
try:
item['email'] = node.find_element_by_xpath(
'.//p[contains(text(),"邮箱：")]').text.strip().replace('邮箱：', '')
except:
item['email'] = ' '
try:
item['法定代表人'] = node.find_element_by_xpath(
'./td[2]/p[1]').text.split('注册资本')[0].replace('法定代表人：', '').strip()
except:
item['法定代表人'] = ' '
try:
item['注册资本'] = node.find_element_by_xpath(
'./td[2]/p[1]/span[1]').text
except:
item['注册资本'] = ' '
try:
item['成立时间'] = node.find_element_by_xpath(
'./td[2]/p[1]/span[2]').text
except:
item['成立时间'] = ' '
try:
item['公司地址'] = node.find_element_by_xpath(
'./td[2]/p[3]').text.replace('地址：', '')
except:
item['公司地址'] = ' '
try:
item['状态'] = node.find_element_by_xpath(
'./td[3]/span').text
except:
item['状态'] = ' '
detail_url = node.find_element_by_xpath('./td[2]/a').get_attribute('href')
return detail_url, item
return None, None
def america():
options = webdriver.ChromeOptions()
options.add_argument("headless")
options.add_argument("disable-gpu")
driver = webdriver.Chrome("C:/selenium/chromedriver", options=options)
url = "https://kr.investing.com/rates-bonds/u.s.-10-year-bond-yield-historical-data" # 인베스팅_10년물 채권 수익률
driver.get(url)
price_list = []
# 일주일치 price 데이터 얻기 - ok
for i in range(1, 8):
price_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[2]"
price_element = WebDriverWait(driver, 25). \
until(EC.presence_of_all_elements_located((By.XPATH, price_xpath)))
price_list.append(price_element[0].text)
change_list = []
# 일주일치 change 데이터 얻기 - ok
for i in range(1, 8):
change_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[6]"
change_element = WebDriverWait(driver, 25). \
until(EC.presence_of_all_elements_located((By.XPATH, change_xpath)))
change_list.append(change_element[0].text)
change_integrate = 0
# 14일치 change 데이터 얻기 - ok
for i in range(1, 15):
change_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[6]"
change_element = WebDriverWait(driver, 25). \
until(EC.presence_of_all_elements_located((By.XPATH, change_xpath)))
change_element = change_element[0].text
change_element = change_element[:-1] # 마지막에 %를 제거하기
change_integrate = change_integrate + float(change_element)
# 일주일치 Date 데이터 얻기 - ok
date_list = []
for i in range(1, 8):
date_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[1]"
date_element = WebDriverWait(driver, 25). \
until(EC.presence_of_all_elements_located((By.XPATH, date_xpath)))
date_list.append(date_element[0].text)
# 출력할 데이터 - ok
show_list = []
show_list.append("미국 10년 채권: ")
show_list.append("14일치 누적: " + str(round(change_integrate, 2)) + "%")
show_list.append(change_list)
show_list.append(driver.current_url)
print(show_list)
driver.close()
# 몇 일 연속으로 오르고 or 내렸는지 화인
plus = 0 # 플러스일 경우 +1
minus = 0 # 마이너스일 경우 +1
plus_plus = 0 # 연속 +
minus_minus = 0 # 연속 -
stoper = 0
for i in range(0, 7):
singleData = change_list[i]
# 수급의 플러스, 마이너스 확인하기
if singleData[0] == "-":
minus += 1
else:
plus += 1
# 순매도의 경우에는 부호 -가 붙지만
# 순매수의 경우에는 부호가 붙지않아서 붙여줌
singleData = "+" + str(singleData)
# 연속 + or - 확인하기
if i == 0 and plus == 1:
plus_plus = 1
elif i == 0 and minus == 1:
minus_minus = 1
# 부호의 경우 마이너스는 -가 붙지만 플러스는 바로 숫자로 표기됨
if i >= 1 and singleData[
0] == "+" and plus_plus >= 1 and stoper == 0:
plus_plus += 1
elif i >= 1 and singleData[
0] == "-" and plus_plus >= 1 and stoper == 0:
stoper = 1
elif i >= 1 and singleData[
0] == "-" and minus_minus >= 1 and stoper == 0:
minus_minus += 1
elif i >= 1 and singleData[
0] == "+" and minus_minus >= 1 and stoper == 0:
stoper = 1
return change_integrate
driver = webdriver.Chrome(path + "\chromedriver.exe")
driver.get("https://play.typeracer.com")
driver.maximize_window()
while True:
target = input("What is your target Words Per Minute? ")
try:
target = int(target)
break
except ValueError:
print("Please type an integer")
while True:
try:
stuff = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located(
(By.XPATH, "//span[@unselectable= 'on']")))
words = []
for i in stuff:
words.append(i.text)
if len(words) == 3:
words[0] = words[0] + words[1]
words.pop(1)
words = ' '.join(words)
words = words.split()
if words[1] == ",":
words[0] = words[0] + words[1]
words.pop(1)
speed = (((len(words) * 60) / target) / len(words)) - 0.01
if not words:
def find_elements(self, locator, timeout=10):
'''定位一组元素'''
elements = WebDriverWait(self.driver, timeout, 1).until(EC.presence_of_all_elements_located(locator))
return elements
def send_private_address_hidden_amount(self):
#Send TRY login process
driver = Browser().open_browser()
#Enter the page
driver.get('https://wallet.trias.one/')
driver.find_element_by_xpath(
'/html/body/div/div/main/div/a[2]/div[1]/h2').click()
#Gets the file name under the downloadfile folder
dir = os.path.abspath('..')
url = dir + "\\downloadfile\\"
files = GetFileName().getfilename(url)
#Calculate folder length
len1 = len(os.listdir(url))
v1 = url + files[len1 - 2]
wait = WebDriverWait(driver, 60)
wait.until(
EC.presence_of_all_elements_located(
(By.XPATH, '//*[@id="fselector"]')))
#Upload a file
driver.find_element_by_xpath('//*[@id="fselector"]').send_keys(v1)
driver.implicitly_wait(5)
#Enter the password
driver.find_element_by_xpath(
'//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input'
).send_keys(123456789)
wait.until(
EC.element_to_be_clickable(
(By.XPATH, '//*[contains(text(),"UNLOCK")]')))
driver.find_element_by_xpath(
'//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input'
).send_keys(Keys.TAB)
#login
time.sleep(9)
try:
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/article[1]/div[2]/article/div/div[3]/a'
).click()
except:
print('This element was not found')
#Transaction process
# Get account amount information
time.sleep(2)
for i in range(50):
time.sleep(1)
account_total_amount = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[1]/div[2]/ul/li/span'
).text
public_address_amount = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[1]/ul/li/span'
).text
public_address_hideen_amount = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[2]/ul/li/span'
).text
private_address_amount = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[3]/ul/li/span'
).text
private_address_hidden_amount = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[4]/ul/li/span'
).text
if int(account_total_amount) != 0:
break
#Get the name and address of the transfer target account
splits = files[len1 - 1].split('--')
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[3]/article/article[1]/div[1]/div[1]/input'
).send_keys(splits[2])
#Random transfer amount
rand = random.randint(1, 150)
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[3]/article/article[1]/section/div[2]/div/input'
).send_keys(rand)
#Click the hide address button
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[3]/article/article[1]/a[1]'
).click()
#Click the hide amount button
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[3]/article/article[1]/a[2]'
).click()
#Click the trade button
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[3]/article/article[1]/div[2]/div/a'
).click()
time.sleep(3)
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[3]/article/article[2]/section/section/div[2]/button[2]'
).click()
#Wait for the trade page to change
try:
wait.until(
EC.presence_of_all_elements_located((
By.XPATH,
'/html/body/div/div/main/div/div[1]/article[2]/section/a[2]/span'
)))
except:
print('Transaction timeout')
time.sleep(2)
for i in range(100):
time.sleep(2)
#enter the homepage
driver.find_element_by_xpath(
'/html/body/div/div/div[2]/div[2]/div/ul/li[1]/a/span').click(
)
#Check the view account info
driver.find_element_by_xpath(
'/html/body/div/div/div[2]/div[2]/div/ul/li[4]/a/span').click(
)
#Upload a file
driver.find_element_by_xpath('//*[@id="fselector"]').send_keys(v1)
driver.implicitly_wait(5)
#Enter the password
driver.find_element_by_xpath(
'//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input'
).send_keys(123456789)
wait.until(
EC.element_to_be_clickable(
(By.XPATH, '//*[contains(text(),"UNLOCK")]')))
driver.find_element_by_xpath(
'//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input'
).send_keys(Keys.TAB)
#login
time.sleep(8)
try:
driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/article[1]/div[2]/article/div/div[3]/a'
).click()
except:
print('This element was not found')
#Get account_total_amount again
time.sleep(2)
for i in range(50):
account_total_amount_second = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[1]/div[2]/ul/li/span'
).text
time.sleep(2)
if int(account_total_amount_second) != 0:
break
if int(account_total_amount_second) != int(account_total_amount):
#Record the account amount and store it in excel
public_address_amount2 = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[1]/ul/li/span'
).text
public_address_hideen_amount2 = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[2]/ul/li/span'
).text
private_address_amount2 = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[3]/ul/li/span'
).text
private_address_hidden_amount2 = driver.find_element_by_xpath(
'/html/body/div/div/main/div/div/div[2]/div[2]/div[4]/ul/li/span'
).text
flag = True
if (int(private_address_amount) +
int(private_address_hidden_amount) -
rand) != (int(private_address_amount2) +
int(private_address_hidden_amount2)):
flag = False
WriteExcel().write_excel_xls_append([
public_address_amount, public_address_hideen_amount,
private_address_amount, private_address_hidden_amount,
'send private adress hidden amount: ' + str(rand) + 'TRY',
public_address_amount2, public_address_hideen_amount2,
private_address_amount2, private_address_hidden_amount2,
flag
])
print('The account has been updated. Please check')
break
time.sleep(5)
driver.quit()
def aliba(url2,searched_word):
#%% ALIBABA: Set Driver and open webpage
print('\nAlibaba: Open URL...')
driver = webdriver.Chrome(chrome_driver, options=options)
driver.get(url2)
driver.maximize_window();
# # For Pop-up
# WebDriverWait(driver,5).until(
# EC.presence_of_all_elements_located((By.CLASS_NAME,'double-close')))[0].click()
#%% ALIBABA - Prepare for webscraping
WebDriverWait(driver,5).until(EC.presence_of_all_elements_located((By.CLASS_NAME,'ui-searchbar-keyword')))
driver.find_element_by_class_name('ui-searchbar-keyword').clear()
driver.find_element_by_class_name('ui-searchbar-keyword').send_keys(searched_word)
time.sleep(3)
WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'ui-searchbar-type-display')))[0].click()
# a[0].click()
sel_supp=WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'ui-searchbar-type-option')))#[0].click()
print('Alibaba: Select Supplier, Trade and Verified...')
sel_supp[1].click()
driver.find_element_by_class_name('ui-searchbar-button-icon').click()
trade_check = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.ID,'J-filter-ta')))
trade_check.click()
verified = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'supplier-icon.icbu-certificate-icon.icbu-certificate-icon-verified')))
verified.click()
pagination = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'ui2-pagination-show')))
pagination_tag=pagination.find_elements_by_tag_name('a')
pagination_tag[-1].click()
#%% ALIBABA: Webscraping
count = 0
page = 1
start_time = time.time()
print('Alibaba: Webscraping starts...')
company_container = WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'title.ellipsis')))
f1 = WebDriverWait(driver,5).until(
EC.presence_of_element_located((By.CLASS_NAME,'ui2-pagination-pages')))
f = f1.find_elements_by_tag_name('a')
total_page = int(f[-2].text)
while page <= 2:#total_page: #2:
try:
print(f'Scraping page {page}')
page = page + 1
articles = WebDriverWait(driver,3).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'title.ellipsis')))
except TimeoutException as e:
print(e)
continue
else:
for article in articles:
ActionChains(driver).move_to_element(article).perform()
company_names.append(article.text)
count = count + 1
print(count)
# print(count)
next_page = WebDriverWait(driver,5).until(
EC.presence_of_all_elements_located((By.CLASS_NAME,'next'))) #[0].click()
ActionChains(driver).move_to_element(next_page[0]).click().perform()
print(f'Alibaba: Webscraping completed. {count} companies found.')
print("Alibaba: Webscraping takes %s seconds ---" % round((time.time() - start_time),1))
#%% ALIBABA: Close website
driver.close()
print('Alibaba: URL closed')
def test_should_fail_to_find_elements_when_explicit_waiting(driver, pages):
pages.load("dynamic.html")
with pytest.raises(TimeoutException):
WebDriverWait(driver, 0.7).until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "redbox")))
# from zhulong.util.etl import gg_meta,gg_html
import sys
# sys.setrecursionlimit(2000)
driver = webdriver.Chrome()
url = 'http://www.aqzbcg.org:1102/jyxx/012001/012001002/20181107/8d01c015-f37d-432b-9043-88624601e362.html'
driver.get(url)
locator = (By.XPATH, '/html/body')
WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator))
title = driver.title
locator = (By.CLASS_NAME, 'ewb-con')
WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located(locator))
before = len(driver.page_source)
time.sleep(0.1)
after = len(driver.page_source)
i = 0
while before != after:
before = len(driver.page_source)
time.sleep(0.1)
after = len(driver.page_source)
i += 1
if i > 5: break
page = driver.page_source
soup = BeautifulSoup(page, 'lxml')
def get_image_title(url):
# 웹 드라이버 초기화
driver_path = "C:/SSAFY/Webdriver/chromedriver.exe"
driver = webdriver.Chrome(driver_path)
driver.implicitly_wait(30) # or bigger second
# 열고자 하는 채널 -> 동영상 목록으로 된 url 페이지를 엶
driver.get(url)
idx = 1
while True:
try:
# 영상 썸네일 : img 안에 있는 src 이용
img_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/ytd-thumbnail/a/yt-img-shadow/img'
# img_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-shelf-renderer[1]/div[1]/div[2]/ytd-vertical-list-renderer/div[1]/ytd-video-renderer[1]/div[1]/ytd-thumbnail/a/yt-img-shadow/img'
# 영상 제목 : a 태그 안에 있는 title 이용
title_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[1]/div/h3/a/yt-formatted-string'
# title_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[1]/div/h3/a'
# title_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-browse/ytd-two-column-browse-results-renderer/div[1]/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-grid-renderer/div[1]/ytd-grid-video-renderer['+str(idx)+']/div[1]/div[1]/div[1]/h3/a'
# 영상 조회수 : span 안에 있는 것 이용
viewcnt_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[1]/ytd-video-meta-block/div[1]/div[2]/span[1]'
# viewcnt_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-browse/ytd-two-column-browse-results-renderer/div[1]/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-grid-renderer/div[1]/ytd-grid-video-renderer['+str(idx)+']/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/span[1]'
# 영상 url : a 태그 안에 있는 href 이용
url_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/ytd-thumbnail/a'
# 채널명 : a 안에 글씨 이용
channel_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[2]/ytd-channel-name/div/div/yt-formatted-string/a'
# 채널 이미지 : img 안에 있는 src 이용
channel_img_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[2]/a/yt-img-shadow/img'
# 이미지가 곧바로 로드 되지 않을 때, 20초간 강제로 기다림
img = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.XPATH, img_xpath)))
if img is None:
print(idx, 'img is not loaded.')
# 동영상 목록을 추가 불러오기 위해 스크롤 내림
if idx % 3 == 0 :
driver.execute_script('window.scrollBy(0, 1080);')
time.sleep(2)
driver.execute_script('window.scrollBy(0, 1080);')
time.sleep(2)
driver.execute_script('window.scrollBy(0, 1080);')
time.sleep(2)
# 썸네일 주소를 리스트에 저장
image = driver.find_element_by_xpath(img_xpath)
img_url = image.get_attribute('src')
image_list.append(img_url)
# 타이틀을 리스트에 저장
title = driver.find_element_by_xpath(title_xpath)
title_list.append(title.text)
# 조회수를 리스트에 저장
view = driver.find_element_by_xpath(viewcnt_xpath)
view_list.append(view.text)
# 해당 url을 리스트에 저장
urls = driver.find_element_by_xpath(url_xpath)
real_url = urls.get_attribute('href')
url_list.append(real_url)
# 채널명을 리스트에 저장
channel = driver.find_element_by_xpath(channel_xpath)
channel_list.append(channel.text)
# 채널 이미지를 리스트에 저장
channel_img = driver.find_element_by_xpath(channel_img_xpath)
channel_img_url = channel_img.get_attribute('src')
channel_img_list.append(channel_img_url)
print(idx, title.text, view.text, real_url, channel.text, channel_img_url)
idx += 1
except Exception as e:
print()
print(e)
break
assert len(image_list) == len(title_list)
return image_list, title_list, view_list, url_list, channel_list, channel_img_list
from selenium.webdriver.chrome.options import Options
from time import sleep
# Definimos el User Agent en Selenium utilizando la clase Options
opts = Options()
opts.add_argument(
"user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/71.0.3578.80 Chrome/71.0.3578.80 Safari/537.36"
)
driver = webdriver.Chrome(
'./chromedriver',
chrome_options=opts) # REMPLAZA AQUI EL NOMBRE DE TU CHROME DRIVER
driver.get('https://www.jumbo.cl/despensa')
links_paginas = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located(
(By.XPATH, '//button[@class="page-number "]')))
links_paginacion = []
for tag_button in links_paginas:
links_paginacion.append('https://www.jumbo.cl/despensa?page=' +
tag_button.text)
for pagina in links_paginacion:
print(pagina)
driver.get(pagina)
links_productos = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located(
(By.XPATH, '//a[@class="shelf-product-title"]')))
links_de_la_pagina = []
import json
USERNAME = "******"
PASSWORD = "******"
hashTag = "#cat"
options = Options()
options.add_argument('--headless')
print("Initializing ...")
driver = webdriver.Firefox(options=options)
driver.get("https://www.instagram.com/accounts/login/")
try:
WebDriverWait(driver, 5).until(
EC.presence_of_all_elements_located((By.NAME, 'username')))
except:
print("Cant Load")
# LOGIN
print("Logging in as {0} ...".format(USERNAME))
driver.find_element_by_xpath("//input[@type='text']").send_keys(USERNAME)
driver.find_element_by_xpath("//input[@type='password']").send_keys(PASSWORD)
driver.find_element_by_xpath("//button[@type='submit']").click()
not_now = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable(
(By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
not_now = WebDriverWait(driver, 5).until(
EC.element_to_be_clickable(
(By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
# ブラウザを開き、URLにアクセスする
driver = webdriver.Firefox()
driver.get(URL)
# 検索ボックスに検索ワードを入力して検索する
search_box = driver.find_element_by_css_selector("input#id-search-field")
search_box.send_keys(SEARCH_WORD)
search_box.send_keys(Keys.RETURN)
# Nextボタンが無くなるまでループする
while True:
# ページのロードを待機する
try:
search_results = WebDriverWait(driver, 5).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR, "ul.list-recent-events li")))
except TimeoutException:
raise
# 検索結果を取得する
for search_result in search_results:
title = search_result.find_element_by_css_selector("h3 a").text
url = urljoin(
URL,
search_result.find_element_by_css_selector("h3 a").get_attribute(
"href"),
)
description_text = ""
descriptions = search_result.find_elements_by_css_selector("p")
for description in descriptions:
description_text += description.text + "\n"
def wait_dialog(self):
confirm_dialog = EC.presence_of_all_elements_located((By.CSS_SELECTOR, '.sweet-alert'))
joined = find_data(soup, CLASS_JOINED).replace("’", ' ')
location = find_data(soup, CLASS_LOCATION)
market = find_data(soup, CLASS_MARKET)
website = find_data(soup, CLASS_WEBSITE)
size = find_data(soup, CLASS_SIZE).replace("-", ' to ')
raised = find_data(soup, CLASS_RAISED)
except Exception as e:
print(f'Error: {e.__doc__}')
pass
else:
company_list.append(
(company_name, company_description, joined, location,
market, website, size, raised, tweeter_link))
print((company_name, company_description, joined, location,
market, website, size, raised, tweeter_link))
count += 1
df = pd.DataFrame(company_list, columns=COLOUMNS)
df.to_csv('startup.csv', index=False)
if __name__ == '__main__':
pprint("Extracting Start-UP Company details....")
driver, wait = start()
driver.get(URL)
results = wait.until(
EC.presence_of_all_elements_located((By.XPATH, XPATH_COMPANIES)))
main(results)
stop(driver)
pprint("Completed !!!!!!")
def like_latest_posts(**kwargs):
# Input arguments - instagram username + driver
username = kwargs.get('username')
num_posts = kwargs.get('num_posts')
driver = kwargs.get('driver')
wait = WebDriverWait(driver, 5)
# Go to webpage
base_url = 'https://www.instagram.com/'
full_url = base_url + username
driver.get(full_url)
time.sleep(1)
# Look through posts and like
for i in range(num_posts):
time.sleep(1)
# Get list of latest posts
posts = wait.until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "v1Nh3")))
time.sleep(random.randint(1, 3))
# Open post
post_url = posts[i].find_element_by_css_selector("a").get_attribute(
"href")
driver.get(post_url)
# Get like button
like_button = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, "button svg")))
time.sleep(random.randint(1, 3))
# Try button since videos don't have number of likes
try:
# Original number of likes
likes_tag_orig = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, "Nm9Fw")))
likes_num_orig = int(
reduce(lambda x, y: x + y,
re.findall(r'\d+', likes_tag_orig.text)))
# Click Like button
like_button.click()
time.sleep(random.randint(1, 3))
# Number of likes after clicking
likes_tag_new = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, "Nm9Fw")))
likes_num_new = int(
reduce(lambda x, y: x + y,
re.findall(r'\d+', likes_tag_new.text)))
time.sleep(random.randint(1, 3))
# If already liked, will be unliked & number of likes will go down - in that case click like button again
if (likes_num_new < likes_num_orig):
like_button = wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "button svg")))
time.sleep(random.randint(1, 3))
# Click Like button
like_button.click()
already_liked = True
else:
already_liked = False
except Exception as e:
# Click Like button
like_button.click()
time.sleep(random.randint(1, 3))
# Go back to user page
driver.get(full_url)
time.sleep(random.randint(1, 3))
# Check if blocked
blocked = check_blocked(driver=driver)
return (already_liked, blocked)
for x in tracks:
ActionChains(driver).move_by_offset(xoffset=x, yoffset=0).perform()
ActionChains(driver).release(element).perform()
time.sleep(3)
if __name__ == '__main__':
driver = webdriver.Chrome()
driver.maximize_window()
driver.get('http://www.cnbaowen.net/api/geetest/')
try:
count = 5
# waiting slidingVC loading
wait = WebDriverWait(driver, 10)
element = wait.until(
EC.element_to_be_clickable((By.CLASS_NAME, 'gt_slider_knob')))
while count > 0:
main(driver, element)
try:
succes = wait.until(
EC.presence_of_all_elements_located(
(By.XPATH, '//div[@class="gt_ajax_tip gt_success"]')))
if succes:
print('恭喜你！识别成功...')
break
except Exception as e:
print('识别错误，继续')
count -= 1
finally:
driver.quit()
search_btn = wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, 'button.btn-search.tb-bg')))
tb_input.send_keys('移动硬盘')
print('搜索...')
search_btn.click()
ls = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist')))
# 定位总的页数
total = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, 'div.total')))
while True:
# 定位商品的条目
ls = wait.until(
EC.presence_of_all_elements_located(
(By.XPATH, '//div[contains(@class,"item J_MouserOnverReq")]')))
browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
time.sleep(random.random() * 2)
print('len:', len(ls))
for item in ls:
title = item.find_elements_by_xpath(
'.//div[@class="row row-2 title"]/a')[0].text.strip()
print('title:', title)
price = item.find_elements_by_xpath(
'.//div[@class="price g_price g_price-highlight"]/strong'
)[0].text.strip()
print('price:', price)
print('=' * 600)
browser.execute_script(
'window.scrollTo(0,document.body.scrollHeight-500);')
def process_chunk(state, district, block):
try:
display = Display(visible=0, size=(1024, 768))
display.start()
driver = webdriver.Firefox()
driver.get("http://omms.nic.in/Home/")
main_menu = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "a.menuLnk")))
#main_menu = driver.find_element_by_css_selector("a.menuLnk")
main_click = main_menu.get_attribute('onclick')
time.sleep(3)
driver.execute_script("return " + main_click)
time.sleep(3)
for year in range(2000, 2017):
#State: Rajasthan District: Ajmer Block: Arain Year : 2000-2001 Batch: All Batches
print "Accessing data for state: ", state, " district: ", district, " block: ", block, " year: ", year, "..."
year = str(year)
access_string = str(
"http://omms.nic.in/MvcReportViewer.aspx?_r=%2fPMGSYCitizen%2fSanctionedProjects&Level=3&State="
+ state[0] + "&District=" + district[0] + "&Block=" +
block[0] + "&Year=" + year +
"&Batch=0&PMGSY=1&DisplayStateName=" + state[1] +
"&DisplayDistName=" + district[1] + "&DispBlockName=" +
block[1] + "&LocalizationValue=en&BatchName=All+Batches")
time.sleep(5)
#print access_string
driver.get(access_string)
#let it load
time.sleep(10)
#Find the right CSS web element using Chrome
elem = WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR, "div[id$='208iT0R0x0'] > a")))
time.sleep(3)
hov = ActionChains(driver).move_to_element(elem[1])
hov.perform()
time.sleep(3)
elem[1].click()
time.sleep(5)
link = driver.find_elements_by_css_selector(
'div#ReportViewer_ctl05_ctl04_ctl00_Menu > div > a')
button_name = link[1].get_attribute('onclick')
time.sleep(3)
try:
driver.execute_script("return " + button_name)
except:
print "Page is being updated"
#Allow to finish download
time.sleep(5)
#harvest(state,district,block,year)
driver.close()
display.stop()
except:
print "Unknown exception"
pass
def AutoPin(self, username, password, pinurl, board):
try:
self.driver.get(
'https://www.pinterest.com/login/?referrer=home_page')
self.driver.find_element_by_xpath(
'//form//input[@type="email"]').send_keys(username)
self.driver.find_element_by_xpath(
'//form//input[@type="password"]').send_keys(password)
self.driver.find_element_by_xpath('//form//button').click()
logger.info('Successfully log in pinterest!')
time.sleep(3)
except Exception:
logger.error('Failed log in pinterest!')
return
try:
self.driver.get(pinurl)
wait = WebDriverWait(self.driver, self.WAIT_TIME)
results = wait.until(
EC.presence_of_all_elements_located(
(By.XPATH, '//dd[@class="prodName"]/a')))
urlList = [result.get_attribute('href') for result in results]
if urlList:
for url in urlList:
logger.info('Auto pin url : %s' % url)
self.driver.get(url)
pinnedImages = []
while True:
try:
currentImage = self.driver.find_element_by_xpath(
'//div[@id="w-featurePics"]/a').get_attribute(
'href')
if currentImage in pinnedImages:
break
else:
pinnedImages.append(currentImage)
logger.info('Auto pin Image : %s' %
currentImage)
self.driver.find_element_by_xpath(
'//div[@id="w-featurePics"]/div').click()
if len(self.driver.window_handles) > 1:
self.driver.switch_to.window(
self.driver.window_handles[-1])
self.driver.find_element_by_xpath(
'//div[@role="button"]//p[contains(text(), "%s")]'
% board).click()
time.sleep(2)
self.driver.close()
self.driver.switch_to.window(
self.driver.window_handles[0])
self.driver.find_element_by_xpath(
'//div[@id="prodthumbnails"]/div/a[1]').click(
)
time.sleep(2)
except Exception:
logger.error(
'Failed auto pin! Continue another to pin')
break
except Exception:
logger.error('Failed auto pin!')
return
driver.find_elements_by_xpath(
'//*[@id="root"]/main/div/section[2]//img')
c_i = True
except:
c_i = False
cover_images = "Not Available"
pass
if c_i:
cover_images = []
s = 0
while True:
try:
elements = WebDriverWait(driver, 5).until(
EC.presence_of_all_elements_located(
(By.XPATH,
'//*[@id="root"]/main/div/section[2]//img')))
element = elements[s]
actions = ActionChains(driver)
actions.move_to_element(element).perform()
cover_image_url = str(element.get_attribute("src"))
img_filename = url.split(
'/')[4] + "_coverimage_" + f"{s}" + ".webp"
# for f in os.listdir('./images'):
# if fnmatch.fnmatch(f,img_filename): pass
# else:
cover_image = requests.get(
cover_image_url,
headers={
'user-agent':
'Mozilla/5.0 (Linux; U; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13'
def wait_for_ajs_dialog(self):
confirm_ajs_dialog = EC.presence_of_all_elements_located((By.CLASS_NAME, 'ajs-body'))
WebDriverWait(self.web_element, 10).until(confirm_ajs_dialog)
def hotel(self):
css_sel = By.CSS_SELECTOR, 'a.flex-link'
return WebDriverWait(self.driver, 3).until(
EC.presence_of_all_elements_located(css_sel))[self.hotel_num]
def startSteps(self):
pTxt = "\n-------- Step 'C007' started!!! --------------------------------------------------------------------"
print(pTxt)
self.driver = webdriver.Firefox()
self.driver.maximize_window()
''' 1. Navigate to staging.getkumbu.com '''
pTxt = "\n1. Navigate to staging.getkumbu.com\n"
print(pTxt)
try:
self.driver.get(self.url)
pTxt = "\t\t(Success)\tLoad webpage successfully"
print(pTxt)
except:
pTxt = "\t\t(Error)\tFailed to load webpage"
print(pTxt)
self.driver.quit()
return
''' 2. Input email adress: [email protected] '''
pTxt = "\n2. Input email adress: [email protected]\n"
print(pTxt)
try:
inputs = WebDriverWait(self.driver, 50).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR, "input.kumbu-input")))
except:
pTxt = "\t\t(Error)\tCan't find 'Email' and 'Password' Inputs"
print(pTxt)
self.driver.quit()
return
try:
email_input = inputs[0]
email_input.send_keys(self.email)
pTxt = "\t\t(Success)\tInput email successfully"
print(pTxt)
except:
pTxt = "\t\t(Error)\tFailed to input 'email'"
print(pTxt)
self.driver.quit()
return
''' 3. Input password: “kumbu is cool” '''
pTxt = "\n3. Input password: '******'\n"
print(pTxt)
try:
pwd_input = inputs[1]
pwd_input.send_keys(self.password)
pTxt = "\t\t(Success)\tInputted 'Password' successfully"
print(pTxt)
except:
pTxt = "\t\t(Error)\tFailed to input 'Password'"
print(pTxt)
self.driver.quit()
return
''' 4. Click Sign in '''
pTxt = "\n4. Click Sign in\n"
print(pTxt)
try:
submit_btn = WebDriverWait(self.driver, 50).until(
EC.element_to_be_clickable(
(By.CSS_SELECTOR, "input#login-submit")))
submit_btn.click()
pTxt = "\t\t(Success)\tClicked 'Sign in' button. Logged in successfully"
print(pTxt)
except:
pTxt = "\t\t(Error)\tFailed to click 'Sign in' button"
print(pTxt)
self.driver.quit()
return
''' 5. Click on 'Collection for Test $TEST_NUMBER' '''
pTxt = "\n5. Click on 'Collection for Test $TEST_NUMBER'\n"
print(pTxt)
try:
collections = WebDriverWait(self.driver, 50).until(
EC.visibility_of_all_elements_located(
(By.CSS_SELECTOR,
"div.collection.columns.small-12.medium-3.text-center")))
flag = False
for collection in collections:
if self.collection_txt in collection.text.strip():
flag = True
break
if flag:
try:
collection.click()
pTxt = "\t\t(Success)\tClicked Successfully"
print(pTxt)
except:
pTxt = "\t\t(Error)\tFailed to click"
print(pTxt)
self.driver.quit()
return
else:
pTxt = "\t\t(Failure)\tFailed to click"
print(pTxt)
self.driver.quit()
return
except:
pTxt = "\t\t(Error)\tFailed to click"
print(pTxt)
self.driver.quit()
return
''' 6. Click on the image whose src starts with /item/thumbnail '''
pTxt = "\n6. Click on the image whose src starts with /item/thumbnail\n"
print(pTxt)
try:
images = WebDriverWait(self.driver, 50).until(
EC.presence_of_all_elements_located(
(By.CSS_SELECTOR,
"div.item.columns.small-6.medium-2.upload")))
if len(images) > 0:
#images[0].find_element_by_tag_name["a"].click()
images[0].click()
pTxt = "\t\t(Success)\tImage is clicked"
print(pTxt)
else:
pTxt = "\t\t(Failure)\tImage can not be clickable"
print(pTxt)
except:
pTxt = "\t\t(Error)\tImage can not be clickable"
print(pTxt)
self.driver.quit()
return
''' 7. Verify that 'Appears in 1 collection' is visible '''
pTxt = "\n7. Verify that 'Appears in 1 collection' is visible\n"
print(pTxt)
try:
txt1 = WebDriverWait(self.driver, 50).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "div.add-collection-action > h4")))
if "Appears in 1 collection" in txt1.text.strip():
pTxt = "\t\t(Success)\t'Appears in 1 collection' is visible"
print(pTxt)
else:
pTxt = "\t\t(Failure\t'Appears in 1 collection' is not visible"
except:
pTxt = "\t\t(Error)\t'Appears in 1 collection' is not visible"
print(pTxt)
self.driver.quit()
return
''' 8. Verify that an image is visible on the right side '''
pTxt = "\n8. Verify that an image is visible on the right side\n"
print(pTxt)
try:
img1 = WebDriverWait(self.driver, 50).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR,
"div.item-content > div.picture-item > img")))
pTxt = "\t\t(Success)\tAn image is visible on the right side"
print(pTxt)
except:
pTxt = "\t\t(Error)\tNo image is visible on the right side"
print(pTxt)
self.driver.quit()
return
''' 9. Click on item-title and replace with 'Test Image $TEST_NUMBER' '''
pTxt = "\n9. Click on item-title and replace with 'Test Image $TEST_NUMBER'\n"
print(pTxt)
try:
item_title = WebDriverWait(self.driver, 50).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "div.meta-info > h2")))
actionChains = ActionChains(self.driver)
actionChains.click(item_title).key_down(Keys.CONTROL).send_keys('a').key_up(Keys.CONTROL) \
.send_keys(Keys.DELETE).send_keys(self.img_text).perform()
pTxt = "\t\t(Success)\t"
print(pTxt)
except:
pTxt = "\t\t(Error)\t"
print(pTxt)
self.driver.quit()
return
''' 10. Click on the little checkmark '''
pTxt = "\n10. Click on the little checkmark\n"
print(pTxt)
try:
check_btn = WebDriverWait(self.driver, 50).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR,
"div.meta-info > a.confirm-item-title-edition")))
check_btn.click()
pTxt = "\t\t(Success)\t"
print(pTxt)
except:
pTxt = "\t\t(Error)\t"
print(pTxt)
self.driver.quit()
return
''' 11. Verify that 'Test Image $TEST_NUMBER' is visible '''
pTxt = "\n11. Verify that 'Test Image $TEST_NUMBER' is visible\n"
print(pTxt)
try:
item_title = WebDriverWait(self.driver, 50).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "div.meta-info > h2")))
if self.img_text in item_title.text.strip():
pTxt = "\t\t(Success)\t'{}' is visible".format(self.img_text)
print(pTxt)
else:
pTxt = "\t\t(Failure)\t'{}' is not visible".format(
self.img_text)
print(pTxt)
except:
pTxt = "\t\t(Error)\t'{}' is not visible".format(self.img_text)
print(pTxt)
self.driver.quit()
return
self.driver.quit()
return
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
# Load photo-sharing site
geckodriver_path = '/home/alex/Downloads/geckodriver'
driver = webdriver.Firefox(executable_path=geckodriver_path)
driver.get('https://www.flickr.com')
# Search category of photos
el_search_field = driver.find_element_by_id('search-field')
el_search_field.send_keys('cats' + Keys.RETURN)
# Click photo first photo
el_photo_list = WebDriverWait(driver, 20). \
until(expected_conditions.presence_of_all_elements_located((By.CSS_SELECTOR, "div.view.photo-list-view > div")))
el_photo = el_photo_list[0]
WebDriverWait(driver, 20). \
until(expected_conditions.visibility_of(el_photo))
el_photo.click()
# Find the URL of the photo.
for i in range(20):
el_main_photo = WebDriverWait(driver, 20). \
until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, "img.main-photo")))
print(el_main_photo.get_attribute('id'))
