def test_solution_cell(browser): _load_notebook(browser) _activate_toolbar(browser) # make sure the toolbar appeared _wait(browser).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".celltoolbar select"))) # does the nbgrader metadata exist? assert _get_metadata(browser) is None # make it a solution cell _select_solution(browser) assert _get_metadata(browser)['solution'] assert not _get_metadata(browser)['grade'] assert not _get_metadata(browser)['locked'] # wait for the id field to appear _wait(browser).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id"))) # set the id _set_id(browser) assert "foo" == _get_metadata(browser)['grade_id'] # make it nothing _select_none(browser) assert not _get_metadata(browser)['solution'] assert not _get_metadata(browser)['grade'] assert not _get_metadata(browser)['locked']
def test_tests_cell(browser, port): _load_notebook(browser, port) _activate_toolbar(browser) # does the nbgrader metadata exist? assert _get_metadata(browser) is None # make it autograder tests _select_tests(browser) assert not _get_metadata(browser)['solution'] assert _get_metadata(browser)['grade'] assert _get_metadata(browser)['locked'] # wait for the points and id fields to appear _wait(browser).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-points"))) _wait(browser).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id"))) WebDriverWait(browser, 30).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".lock-button"))) # set the points _set_points(browser) assert 2 == _get_metadata(browser)['points'] # set the id _set_id(browser) assert "foo" == _get_metadata(browser)['grade_id'] # make it nothing _select_none(browser) assert not _get_metadata(browser)['solution'] assert not _get_metadata(browser)['grade'] assert not _get_metadata(browser)['locked']
def findLineOfCertainStatus(driver, status): try: WebDriverWait(driver, 10).until( EC.presence_of_element_located((By.XPATH, "//div[@class='table']/div/div/table/tbody"))) except TimeoutException as e: logCommon.error('None unacked records in the table.') tbody = driver.find_element_by_xpath("//div[@class='table']/div/div/table/tbody") trs = WebDriverWait(tbody, 10).until(EC.presence_of_all_elements_located((By.XPATH, ".//tr"))) # trs = tbody.find_elements_by_xpath(".//tr") for tr in trs: # tds = tr.find_elements_by_xpath(".//td") tds = WebDriverWait(tr, 10).until(EC.presence_of_all_elements_located((By.XPATH, ".//td"))) if (0 == status): if ("未清除" == tds[5].get_attribute("innerHTML").encode('utf-8').strip()): print "告警代码:" + tds[1].get_attribute("innerHTML").encode('utf-8') + " 未清除" return tr elif (2 == status): if ("未确认" == tds[6].get_attribute("innerHTML").encode('utf-8').strip()): # print "告警代码:" + tds[1].get_attribute("innerHTML").encode('utf-8') + " 未确认" logCommon.info("ID: " + str(tds[1].get_attribute("innerHTML").encode('utf-8')) + " not acked") return tr elif (4 == status): if ("未确认" == tds[6].get_attribute("innerHTML").encode('utf-8').strip() and "未清除" == tds[5].get_attribute( "innerHTML").encode('utf-8').strip()): print "告警代码:" + tds[1].get_attribute("innerHTML").encode('utf-8') + " 未确认并且未清除" return tr return False
def test_check_countries(self): wait = WebDriverWait(self.driver, 5) self.sign_in() self.driver.get(COUNTRIES_PAGE) current_countries = [] not_null_zones_country_links = [] country_rows = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".row"))) for row in country_rows: country = row.find_element_by_css_selector("td:nth-child(5) a") current_countries.append(country.text) zones = row.find_element_by_css_selector("td:nth-child(6)") if int(zones.text) > 0: not_null_zones_country_links.append(country.get_attribute("href")) expected_countries = sorted(current_countries) self.assertListEqual(current_countries, expected_countries) for link in not_null_zones_country_links: self.driver.get(link) current_country_zones = [x.text for x in wait.until(EC.presence_of_all_elements_located( (By.XPATH, "(.//*[@id=\"table-zones\"]//td[3])[position() < last()]") ))] expected_country_zones = sorted(current_country_zones) self.assertListEqual(current_country_zones, expected_country_zones)
def test_locked_cell(browser, port): _load_notebook(browser, port) _activate_toolbar(browser) # does the nbgrader metadata exist? assert _get_metadata(browser) is None # make it locked _select_locked(browser) assert not _get_metadata(browser)['solution'] assert not _get_metadata(browser)['grade'] assert _get_metadata(browser)['locked'] # wait for the id and lock button to appear WebDriverWait(browser, 30).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id"))) WebDriverWait(browser, 30).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".lock-button"))) # set the id _set_id(browser) assert "foo" == _get_metadata(browser)['grade_id'] # make sure the metadata is valid _save_and_validate(browser) # make it nothing _select_none(browser) assert not _get_metadata(browser) _save_and_validate(browser)
def test_manual_cell(browser, port): _load_notebook(browser, port) _activate_toolbar(browser) # does the nbgrader metadata exist? assert _get_metadata(browser) is None # make it manually graded _select_manual(browser) assert _get_metadata(browser)['solution'] assert _get_metadata(browser)['grade'] assert not _get_metadata(browser)['locked'] # wait for the points and id fields to appear _wait(browser).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-points"))) _wait(browser).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".nbgrader-id"))) # set the points _set_points(browser) assert 2 == _get_metadata(browser)['points'] # set the id _set_id(browser) assert "foo" == _get_metadata(browser)['grade_id'] # make sure the metadata is valid _save_and_validate(browser) # make it nothing _select_none(browser) assert not _get_metadata(browser) _save_and_validate(browser)
def test_correct_tabs_are_displayed(self): """Tabs with the class admin-only should not be displayed, and tabs with the class teacher-only should be displayed """ try: admin_only_elements = WebDriverWait(self.browser, 10).until( expected_conditions.presence_of_all_elements_located((By.CLASS_NAME, "admin-only")) ) except TimeoutException: admin_only_elements = [] teacher_only_elements = WebDriverWait(self.browser, 10).until( expected_conditions.presence_of_all_elements_located((By.CLASS_NAME, "teacher-only")) ) # Make sure nav bar is expanded e.g. in a small screen try: navbar_expand = self.browser.find_element_by_class_name("navbar-toggle") self.browser_activate_element(elem=navbar_expand) # Wait for the animation to finish WebDriverWait(self.browser, 3).until( expected_conditions.visibility_of_element_located((By.CLASS_NAME, "nav")) ) except ElementNotVisibleException: # browser_activate_element could throw this, meaning nav bar is already visible pass for el in admin_only_elements: self.assertFalse(el.is_displayed(), "Elements with `admin-only` class should not be displayed!") for el in teacher_only_elements: self.assertTrue(el.is_displayed(), "Elements with `teacher-only` class should be displayed!")
def login(self): url = 'https://mp.weixin.qq.com/' self._browser.get(url) self._browser.implicitly_wait(3) try: input_username = self._wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_input_panel > div:nth-child(1) > div > span > input"))) input_username[0].clear() input_username[0].send_keys(self.username) input_password = self._wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_input_panel > div:nth-child(2) > div > span > input"))) input_password[0].clear() input_password[0].send_keys(self.password) ##记住我 remember_me = self._wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_help_panel > label > i"))) remember_me[0].click() submit = self._wait.until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#header > div.banner > div > div > form > div.login_btn_panel > a"))) submit[0].click() time.sleep(10) ##存储cookies到本地 self._browser.get(url) for cookie_item in self._browser.get_cookies(): self._cookie[cookie_item['name']] = cookie_item['value'] cookie_str = json.dumps(self._cookie) with open(path.join(path.abspath('.'),'cookie.txt'), 'w+', encoding='utf-8') as f: f.write(cookie_str) except TimeoutException: print("Time Out")
def data_extractor(input_driver): """ HELPER FUNCTION takes the driver object and extracts the page content data and cleans it if present and loaded correctly then returns the processed data and state : True/False (driver object) --> (boolean, dictionary) """ try: final_dict = {} # retrieve data by class Name for the name fields name_list = [' '] namefields = waiter(input_driver, 5).until( EC.presence_of_all_elements_located( (By.CLASS_NAME, "RightAlignBlack"))) for name in namefields: name_list.append(str(name.text)) #data corresponding to name fields data_list = [] datafields = waiter(input_driver, 5).until( EC.presence_of_all_elements_located( (By.CLASS_NAME, "LeftAlignGreen"))) for data in datafields: data_str = str(data.text) if data_str != ' : ': data_list.append(data_str) #handling the special case of the name field: Standard Seating Capacity capacity_namexpath = """ //*[@id="ctl00_UpdatePanel1"]/table/tbody/tr[5]/td/table/tbody/tr[6] /td[2]/fieldset/table/tbody/tr[3]/td[5]""" capacity_namefield = waiter(input_driver, 5).until( EC.presence_of_element_located((By.XPATH, capacity_namexpath))) cs_namefield = str(capacity_namefield.text) #getting special case of the data field for Standard Seating Capacity capacity_dataxpath = """ //*[@id="ctl00_UpdatePanel1"]/table/tbody/tr[5]/td/table/tbody/tr[6] /td[2]/fieldset/table/tbody/tr[3]/td[7]""" capacity_datafield = waiter(input_driver, 5).until( EC.presence_of_element_located((By.XPATH, capacity_dataxpath))) cs_datafield = str(capacity_datafield.text) #creating the clean data and enter in dictionary pos = 0 offset = 0 for nameval in name_list: if pos == 0 or pos == 1: pos += 1 continue if cs_datafield == data_list[pos + offset]: offset += 1 final_dict[nameval] = data_list[pos + offset] pos += 1 #finally add the exception case data to dictionary final_dict[cs_namefield] = cs_datafield return(True, final_dict) except TimeoutException: return(False, None)
def find_elements(self,by,locator,timeout=configfile.timeout_find_element): if by == 'id': elements = WebDriverWait(self.driver, timeout).until( EC.presence_of_all_elements_located((By.ID,locator))) elif by == 'xpath': elements = WebDriverWait(self.driver, timeout).until( EC.presence_of_all_elements_located((By.XPATH,locator))) elif by == 'name': elements = WebDriverWait(self.driver, timeout).until( EC.presence_of_all_elements_located((By.NAME,locator))) elif by == 'class': elements = WebDriverWait(self.driver, timeout).until( EC.presence_of_all_elements_located((By.CLASS_NAME,locator))) return elements
def clico_menu_dashboards(context): #esperando elementos do menu wait = WebDriverWait(context.driver, 10) wait.until(EC.presence_of_all_elements_located((By.XPATH, "//div[contains(@class, 'x-btn x-box-item x-toolbar-item x-btn-custom-toolbar-small x-noicon x-btn-noicon x-btn-custom-toolbar-small-noicon')]"))) #selencionado o menu admnistração menu_lista = context.driver.find_elements_by_xpath("//div[contains(@class, 'x-btn x-box-item x-toolbar-item x-btn-custom-toolbar-small x-noicon x-btn-noicon x-btn-custom-toolbar-small-noicon')]") admin = list(menu_lista) menu_admin = admin[1] menu_admin.click() wait = WebDriverWait(context.driver, 20) wait.until(EC.presence_of_all_elements_located((By.XPATH, "//span[contains(@class, 'x-menu-item-text')]"))) menu_admin_lista = list(context.driver.find_elements_by_xpath("//span[contains(@class, 'x-menu-item-text')]")) menu_admin_dashboards = menu_admin_lista[9] menu_admin_dashboards.click()
def Click_Element_By_Link_Text(_name,parent=False,uppercase=False): ''' Use this function only if you are sure that there wont be any conflicting Name. If possible use Click_Element_By_ID ''' sModuleInfo = inspect.stack()[0][3] + " : " + inspect.getmoduleinfo(__file__).name try: CommonUtil.TakeScreenShot(sModuleInfo, local_run) #Find all elements containing the name CommonUtil.ExecLog(sModuleInfo, "Trying to find element by Link Text: %s"%_name, 1,local_run) if isinstance(parent, (bool)) == True: if uppercase: allElements = WebDriverWait(sBrowser, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT, _name.upper()))) else: allElements = WebDriverWait(sBrowser, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT,_name))) else: if uppercase: allElements = WebDriverWait(parent, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT, _name.upper()))) else: allElements = WebDriverWait(parent, WebDriver_Wait).until(EC.presence_of_all_elements_located((By.LINK_TEXT, _name))) if allElements == []: CommonUtil.ExecLog(sModuleInfo, "Could not find your element by Link Text: %s"%_name, 3,local_run) return "failed" else: if len(allElements) > 1: CommonUtil.ExecLog(sModuleInfo, "Found more than one element and will use the first one. ** if fails, try providing parent element or try by ID** ", 2, local_run) for each in allElements: if uppercase: if (WebDriverWait(each, WebDriver_Wait).until(lambda driver : each.text==_name.upper())) == True: Element = each CommonUtil.ExecLog(sModuleInfo, "Found your element by Link Text: %s. Using the first element found to click"%_name, 1,local_run) break #Now we simply click it else: if (WebDriverWait(each, WebDriver_Wait).until(lambda driver : each.text==_name)) == True: Element = each CommonUtil.ExecLog(sModuleInfo, "Found your element by Link Text: %s. Using the first element found to click"%_name, 1,local_run) break Element.click() CommonUtil.TakeScreenShot(sModuleInfo, local_run) CommonUtil.ExecLog(sModuleInfo, "Successfully clicked your element: %s"%_name, 1,local_run) return "passed" except Exception, e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] Error_Detail = ((str(exc_type).replace("type ", "Error Type: ")) + ";" + "Error Message: " + str(exc_obj) +";" + "File Name: " + fname + ";" + "Line: "+ str(exc_tb.tb_lineno)) CommonUtil.ExecLog(sModuleInfo, "Unable to expand menu: %s. Error: %s"%(_name,Error_Detail), 3,local_run) return "failed"
def get_img_code(self): global driver if driver: driver.close() firefox_path = self.find_firefox_path() driver = webdriver.Firefox(executable_path=firefox_path) driver.maximize_window() actions = ActionChains(driver) url = "https://inv-veri.chinatax.gov.cn/index.html" driver.get(url) #打开网页 WebDriverWait(driver, 5, 0.5).until(ec.presence_of_all_elements_located((By.ID, "kprq"))) fpdm = driver.find_element_by_id("fpdm") fpdm.send_keys(self.invoice_code) time.sleep(1) actions.click(fpdm)#提交发票代码 fphm = driver.find_element_by_id("fphm") fphm.send_keys(self.name) #提交发票号码 date = self.invoice_date.replace('-', '') # 转换日期格式 date_in = driver.find_element_by_id("kprq")#提交发票日期 date_in.clear() date_in.send_keys(date) if self.invoice_type == 'zy': driver.find_element_by_id("kjje").send_keys(str(self.invoice_amount))#提交发票金额 if self.invoice_type == 'pt': driver.find_element_by_id("kjje").send_keys(str(self.invoice_heck_code)[-6:]) # 提交校验码后六位 if self.invoice_type == 'dz': driver.find_element_by_id("kjje").send_keys(str(self.invoice_heck_code)[-6:]) # 提交校验码后六位 time.sleep(1) WebDriverWait(driver, 5, 0.5).until(ec.presence_of_all_elements_located((By.ID, "yzminfo"))) # 取图片 try: imgelement = driver.find_element_by_id("yzm_img") except: actions.click(fpdm) actions.click(fphm) imgelement = driver.find_element_by_id("yzm_img") finally: img = imgelement.get_attribute('src').split(",")[-1] # 取图片说明 img_note = driver.find_element_by_id("yzminfo").text print img self.write({'img': img, 'img_note': img_note}) # 处理base64代码不足报错 missing_padding = 4 - len(img) % 4 if missing_padding: img += b'=' * missing_padding imgdata = base64.b64decode(img) file = open('./log/out.png', 'wb') file.write(imgdata)
def parseItem(self, url): ''' Parsing item with url Args: url of item url: url of item Returns: item, in dictionary ''' # take care of videos when choosing image self.web_driver.get(url) try: # so after all this is probably the right way to find elements elem = WebDriverWait(self.web_driver, 10).until( EC.presence_of_all_elements_located((By.XPATH, '//input[contains(@class, "thumbs")]')) ) except Exception as e: print e item = {} item["detail_image_path"] = "" images = [] for i in elem: images.append(i.get_attribute('src')) item["detail_image_path"] = "|".join(images) item["suitable_images_index"] = 1 item['white_suitable_images_index'] = 0 item['white_suitable_images'] = "" try: # so after all this is probably the right way to find elements elem = WebDriverWait(self.web_driver, 10).until( EC.presence_of_all_elements_located((By.XPATH, '//div[contains(@id, "tabWindow")]')) ) except Exception as e: print e if len(elem) >= 1: item["info"] = elem[0].text try: # so after all this is probably the right way to find elements elem = WebDriverWait(self.web_driver, 10).until( EC.presence_of_all_elements_located((By.XPATH, '//p[contains(@class, "description")]')) ) except Exception as e: print e if len(elem)>=1: item["description"] = elem[0].text return item
def add_to_cart(self, index): self.wait.until(EC.presence_of_all_elements_located( (By.CLASS_NAME, 'js-product-to-cart') ))[index].click() self.wait.until(EC.visibility_of_element_located( (By.CLASS_NAME, 'js-cart-wrapper') ))
def getupc(data, sleeptime): display = Display(visible=0, size=(800, 600)) display.start() a = webdriver.Firefox() a.get('https://www.google.com/ncr') time.sleep(sleeptime) search = WebDriverWait(a, 5).until(EC.element_to_be_clickable((By.XPATH, "//input[@type='text']"))) for i in data: ActionChains(a).move_to_element(search).click(search).send_keys(i['name'] + ' upc', Keys.ENTER).perform() time.sleep(sleeptime) contents = WebDriverWait(a, 5).until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='g']"))) try: upc = next( (re.split(r'/', href.find_element_by_tag_name('a').get_attribute('href'))[-1] for href in contents if href.find_element_by_tag_name('a').get_attribute('href').startswith( 'http://www.upcitemdb.com/upc'))) i['upc'] = upc except StopIteration: pass search = WebDriverWait(a, 5).until(EC.element_to_be_clickable((By.XPATH, "//input[@type='text']"))) search.clear() a.close() display.stop() return data
def getHtml(word,driver): driver.get('http://www.shufazidian.com') elem = driver.find_element_by_name('wd') select = Select(driver.find_element_by_name('sort')) select.select_by_value('9') # print('1, ', ctime()) elem.send_keys(word) # wait = WebDriverWait(driver,5) elem.send_keys(Keys.RETURN) # driver.implicitly_wait(5) # try: # element = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.mbpho'))) # print('3, ',ctime()) # except Exception,e: # print(e) # print('4, ', ctime()) delay = 7 # seconds try: WebDriverWait(driver, delay).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.mbpho'))) except TimeoutException: # print >>f , "TimeoutException Loading took too much time!" pass time.sleep(5) return driver.page_source
def find_paper_in_scholar(cnx, title, id, key): driver.get(SCHOLAR) try: search_box = WebDriverWait(driver, WAIT_TIME).until(EC.presence_of_element_located((By.ID, "gs_hp_tsi"))) search_box.send_keys(title + Keys.RETURN) if in_captcha_page(): print "redirected to captcha" try: #wait scholar_hits = WebDriverWait(driver, WAIT_TIME).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "gs_ri"))) time.sleep(2) for hit in scholar_hits: info_hit = hit.find_element_by_class_name("gs_rt") try: hit_link = info_hit.find_element_by_tag_name("a") title_hit = hit_link.text leveh_distance = nltk.metrics.edit_distance(re.sub(r'\s+', '', title_hit.lower()), re.sub(r'\s+', '', title.lower())) if leveh_distance <= 6: if leveh_distance > 1: logging.warning("match: " + title_hit + " ******* " + title + " ******* " + str(leveh_distance)) add_abstract_to_paper(hit_link, cnx, id, key) break except NoSuchElementException: continue except TimeoutException: logging.warning("abstract not found for " + title) except TimeoutException: logging.info("Google Scholar not loaded")
def FindCssElements(self, css, obj=None): """Find web elements using CSS elements. Args: css: string, css elements to search for. obj: web element object to search within. Returns: list of web element objects that contain css. """ try: self.GetWait().until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, css))) except TimeoutException: self.logger.error('Timed out looking for css: %s', css) return None else: try: if obj: elements = obj.find_elements_by_css_selector(css) else: elements = self.driver.find_elements_by_css_selector(css) except NoSuchElementException: self.logger.error('Element css %s not found', css) return None return elements
def next_level(self): self.assert_success() self.browser.find_element_by_id("next_level_button").click() WebDriverWait(self.browser, 10).until( presence_of_all_elements_located((By.ID, "blockly_tab")) ) return self
def run_alarms(self, driver, test_log): gui_lib = Utils(driver) self.login_manager.login() test_log.start('Alarms') test_helper = TestHelper(test_log, driver) driver.switch_to_default_content() gui_lib.click_element('menu_node_status') gui_lib.click_element('menu_node_alarms') driver.switch_to_frame('frame_content') # t = driver.find_element(By.XPATH, "//body/fieldset/legend").text # print('test', t) buttons = driver.find_elements_by_tag_name('button') # try test for visibility on the function below as it finds the buttons # before they are onscreen # (EC.visibility_of_element_located( WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.TAG_NAME, 'button'))) # check that there are 3 buttons test_helper.assert_true(len(buttons) != 3, 'Not all buttons displayed', 'Check that all buttons displayed') # test that the buttons have text on them for btn in buttons: test_helper.assert_true(len(btn.text.strip()) == 0, 'No text on button', 'Check buttons have text') print(len(btn.text.strip())) # test_log.close() self.login_manager.logout()
def browser_check_django_message(self, message_type=None, contains=None, exact=None, num_messages=1): """Both central and distributed servers use the Django messaging system. This code will verify that a message with the given type contains the specified text.""" # Get messages (and limit by type) if num_messages > 0: messages = WebDriverWait(self.browser, 5).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "alert"))) else: messages = [] if isinstance(contains, str): # make sure we only look at messages we're looking for. messages = [message for message in messages if contains in message.text] # Check that we got as many as expected if num_messages is not None: msg = "Make sure there are %d message(s), type='%s'." % \ (num_messages, message_type if message_type else "(any)") self.assertEqual(num_messages, len(messages), msg) for i, message in enumerate(messages): if type(contains) == list: contain = contains[i] else: contain = contains if type(exact) == list: exac = exact[i] else: exac = exact if contain is not None: self.assertIn(contain, message.text, "Make sure message contains '%s'" % contain) if exac is not None: self.assertEqual(exac, message.text, "Make sure message = '%s'" % exac)
def FindClasses(self, classname, obj=None): """Find all web elements with classname. Args: classname: string, class to search for. obj: web element object to search within. Returns: list of web element objects containing classname. """ try: self.GetWait().until(EC.presence_of_all_elements_located((By.CLASS_NAME, classname))) except TimeoutException: self.logger.error('Timed out looking for class: %s', classname) return None else: try: if obj: elements = obj.find_elements_by_class_name(classname) else: elements = self.driver.find_elements_by_class_name(classname) except NoSuchElementException: self.logger.error('Error finding %s class name.', classname) return None return elements
def FindTags(self, tagname, obj=None): """Find web elements using tagname. Args: tagname: string, tag to search for. obj: web element object to search within. Returns: list of web element objects containing tagname. """ try: self.GetWait().until(EC.presence_of_all_elements_located((By.TAG_NAME, tagname))) except TimeoutException: self.logger.error('Timed out finding names: %s', tagname) return None else: try: if obj: elements = obj.find_elements_by_tag_name(tagname) else: elements = self.driver.find_elements_by_tag_name(tagname) except NoSuchElementException: self.logger.error('Error finding tag: %s', tagname) return None return elements
def action_on_richdata(driver): """ Method to perform actions on rich data""" try: WebDriverWait(driver, 30).until(EC.presence_of_all_elements_located(driver.find_elements_by_xpath("//*"))) except: wait_to_load_webelement(driver,"xpath","//*") ##wait_to_load_webelement(driver,"tagname","a") all_elements_list = driver.find_elements_by_tag_name("a") ## time.sleep(8) for item_all_elements_list in all_elements_list: if str(item_all_elements_list.get_attribute("class")).split("-")[0] == 'colorbox': try: element = item_all_elements_list.find_element_by_tag_name("img") element.click() ## close_button = self.driver.find_element_by_class_name("fancybox-close") ## close_button.click() time.sleep(5) except : item_all_elements_list.click() time.sleep(5) wait_to_load_webelement(driver, "id", "fancybox-close") ## time.sleep(2) close_button = driver.find_elements_by_class_name("fancybox-close") for btn in close_button: btn.click() break time.sleep(5)
def size_options(self): try: return self.wait.until(EC.presence_of_all_elements_located( self.SIZE_OPTIONS )) except TimeoutException: return []
def test_check_all_admin_pages(self): wait = WebDriverWait(self.driver, 5) self.driver.get(SIGN_IN_PAGE) self.driver.find_element_by_name("username").send_keys(LOGIN) self.driver.find_element_by_name("password").send_keys(PASSWORD) self.driver.find_element_by_name("login").click() first_menu_elements = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#app->a"))) first_menu_links = [x.get_attribute("href") for x in first_menu_elements] for link in first_menu_links: self.driver.get(link) first_menu_item_header = self.driver.find_elements_by_css_selector("#content>h1") self.assertTrue(first_menu_item_header) second_menu_elements = self.driver.find_elements_by_css_selector(".docs a") second_menu_links = [x.get_attribute("href") for x in second_menu_elements] if second_menu_elements: for link in second_menu_links: self.driver.get(link) second_menu_item_header = self.driver.find_elements_by_css_selector("#content>h1") self.assertTrue(second_menu_item_header)
def test_question_page(self): # login self.client.get('http://*****:*****@example.com') self.client.find_element_by_name('password').send_keys('cat') self.client.find_element_by_id('submit').click() # navigate to a question page time.sleep(2) self.client.find_element_by_id('explore').click() question_links = WebDriverWait(self.client, 10).until( EC.presence_of_all_elements_located(( By.CLASS_NAME, 'question_link')) ) random.choice(question_links).click() # follow the question WebDriverWait(self.client, 10).until( lambda c: len(c.window_handles) == 2) self.client.switch_to_window(self.client.window_handles[-1]) time.sleep(5) follow_button = WebDriverWait(self.client, 10).until( EC.presence_of_element_located(( By.XPATH, '//div[@class="side-section"]/button')) ) follow_button.click() time.sleep(5) self.assertTrue('btn-white' in follow_button.get_attribute('class'))
def sell_orders(): scroll_down() #xpath = '//tr[contains(@class,"sell_tr")]' while True: logging.debug("Getting sell rows") xpath = '//table[@id="md-sell"]/tbody/tr[position() <= 5]' trs = WebDriverWait(driver, 20).until( EC.presence_of_all_elements_located((By.XPATH, xpath))) trs = reversed(trs) logging.debug("Looping through rows") try: tr = trs.next() _tds = tr.find_elements_by_tag_name('td') tds = [] print "tds = {0}".format(pp.pformat(_tds)) for td in _tds: tds.append(float(td.text)) so = SellOrder(*tds[0:2]) logging.debug("yielding sell order: {0}".format(so)) yield so except StaleElementReferenceException: logging.debug("StaleElementReferenceException!") continue
def parseC(self, response): chemical_item = Chemical() ###Create Item ###Name name_url = response.url + "#section=Top" self.sel.get(name_url) time.sleep(4) chemical_item['name']= WebDriverWait(self.sel, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/main/div/div/div[1]/div[2]/div/h1/span'))).text ###CAS cas_url = response.url + "#section=CAS" self.sel.get(cas_url) time.sleep(1) chemical_item['cas'] = WebDriverWait(self.sel, 5).until(EC.presence_of_element_located((By.XPATH,'//ol/li[3]/ol/li[3]/ol/li[1]/div[2]/div[1]'))).text ###GHS CLASSIFICATION hazard_url = response.url + "#section=GHS-Classification" self.sel.get(hazard_url) time.sleep(1) try: thumbnail_container = WebDriverWait(self.sel, 5).until(EC.presence_of_all_elements_located((By.XPATH,'//ol/li[1]/ol/li[1]/div[2]/div[1]/div/img'))) urls = [] for element in thumbnail_container: urls.append(element.get_attribute('src')) chemical_item['image_urls']=urls except TimeoutException: chemical_item['image_urls']= [] self.chemical_info.append(chemical_item) time.sleep(3) return chemical_item
def presence_of_all_elements_located(self, selector): '''判断是否至少有1个元素存在于dom树中,如果定位到就返回列表''' locator = self.handle_locator(selector) return WebDriverWait(self.driver, 10).until( EC.presence_of_all_elements_located(locator))
def test_edit_promotion_method_(self): driver = self.driver self.authorizatuion(driver) WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//a[@href='/superadmin/promotions']"))).click() time.sleep(2) list_promotions = WebDriverWait(driver, 10).until(ec.presence_of_all_elements_located((By.XPATH, "//mat-row[@class='mat-row']"))) list_promotions[randint(0, len(list_promotions)-1)].click() time.sleep(2) #WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//*[@id='portal-classic-content']/app-super-promotion-view/app-promotion-view/div/mat-card/mat-card-content/div[1]/form/div[1]/app-gallery-picker/div/div[1]/button"))).click() #time.sleep(2) file_dicitionary = {0: "/Users/rufina/Desktop/dishs/BjJ6inaYiWam0GGViLFHLQ-double.jpg", 1: "/Users/rufina/Desktop/dishs/4Rve51WmWfk.jpg", 2: "/Users/rufina/Desktop/dishs/2531.jpg", 3: "/Users/rufina/Desktop/dishs/salat_kinoa.jpg__1499258543__50030.jpg", 4: "/Users/rufina/Desktop/dishs/4703.jpg", 5: "/Users/rufina/Desktop/dishs/caption (1).jpg"} #WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//input[@type='file']"))).send_keys(file_dicitionary[randint(0, len(file_dicitionary) - 1)]) #time.sleep(2) name_promotion = WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//input[@placeholder='Название акции']"))) name_promotion.clear() name_promotion.send_keys(self.my_metho_randem_stroka(randint(4, 7), 3)) time.sleep(2) text_promotion = WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//textarea[@placeholder ='Описание акции']"))) text_promotion.clear() text_promotion.send_keys(self.my_metho_randem_stroka(randint(5,10), randint(3, 5))) time.sleep(2) WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//input[@placeholder='От']"))).click() time.sleep(2) list_dates_from = WebDriverWait(driver, 10).until(ec.presence_of_all_elements_located((By.XPATH, "//td[@class='mat-calendar-body-cell ng-star-inserted']"))) list_dates_from[randint(0, len(list_dates_from )-1)].click() time.sleep(2) WebDriverWait(driver, 10).until( ec.presence_of_element_located((By.XPATH, "//input[@placeholder='До']"))).click() time.sleep(2) list_dates_to = WebDriverWait(driver, 10).until( ec.presence_of_all_elements_located((By.XPATH, "//td[@class='mat-calendar-body-cell ng-star-inserted']"))) list_dates_to[randint(0, len(list_dates_to) - 1)].click() time.sleep(2) WebDriverWait(driver, 10).until(ec.presence_of_element_located((By.XPATH, "//button[@class='mat-button mat-flat-button']"))).click() time.sleep(5)
def testShouldFailToFindElementsWhenExplicitWaiting(driver, pages): pages.load("dynamic.html") with pytest.raises(TimeoutException): WebDriverWait(driver, 0.7).until( EC.presence_of_all_elements_located((By.CLASS_NAME, "redbox")))
) dcap = dict(DesiredCapabilities.PHANTOMJS) dcap["phantomjs.page.settings.userAgent"] = USER_AGENT print('Setting up Phantom JS webdriver...') ########################################### browser = webdriver.PhantomJS(desired_capabilities=dcap) print('Fetching - ', TARGET) ############################ browser.get('https://orgsync.com/412/community/calendar?view=upcoming') print('Waiting for JS to load...') ################################## wait = WebDriverWait(browser, 10) wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'osw-events-list-date'))) events_module = browser.find_element_by_class_name('osw-events-index-main') print('Found events module') # Constants specific to Orgsync DOM DATE_CLASS = 'osw-events-list-date' EVENT_CLASS = 'osw-events-list-item-content' TITLE_CLASS = 'osw-events-list-date-header' NAME_CLASS = 'osw-events-list-item-title' TIME_CLASS = 'osw-events-list-item-time' ORG_CLASS = 'osw-events-list-item-portal-name' # Parse remaining with BS4 html = events_module.get_attribute('outerHTML') soup = BeautifulSoup(html, 'html.parser') dates = soup.find_all('div', attrs={'class': DATE_CLASS})
def linkedin(url3,searched_word,job_fcn,sen_level): print('\nLinkedin: Open URL...') driver = webdriver.Chrome(chrome_driver, options=options) driver.get(url3) # driver.maximize_window() # driver.set_window_size(1470, 1033) #%% Linkedin: Login print('Linkedin: Logging in...') idd = '*****@*****.**' pw = 'awesomeMe2020' WebDriverWait(driver,5).until( EC.presence_of_element_located((By.ID,'username'))) driver.find_element_by_id('username').clear() driver.find_element_by_id('username').send_keys(idd) driver.find_element_by_id('password').clear() driver.find_element_by_id('password').send_keys(pw) driver.find_element_by_class_name('btn__primary--large.from__button--floating').click() #%% Linkedin: Click sales Navigator print('Linkedin: Open Sales Navigator...') main_page = driver.window_handles[0] sales_navi = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'global-nav__primary-item.global-nav__spotlight'))) sales_navi.click() sales_navi_windows_handle = driver.window_handles[1] driver.switch_to.window(sales_navi_windows_handle) #%% Linkedin: Click Advanced Search print('Linkedin: Filling in Advanced search...') # adv_search = WebDriverWait(driver,5).until( # EC.presence_of_element_located((By.CLASS_NAME,'artdeco-dropdown__trigger.color-white.t-14.t-bold.artdeco-dropdown__trigger--non-button.flex.align-items-flex-end.artdeco-dropdown__trigger--placement-bottom.ember-view'))) adv_search = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'artdeco-dropdown.artdeco-dropdown--placement-bottom.artdeco-dropdown--justification-left.ember-view'))) adv_search.click() time.sleep(0.2) leads = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'artdeco-dropdown__item.artdeco-dropdown__item--is-dropdown.ember-view'))) leads.click() #%% Linkedin: Obtaining all filter handles filt_list = [ 'custom lists', 'past lead', 'geography', 'relationship', 'industry', 'school', 'profile language', 'first name', 'last name', 'seniority level', 'years in current position', 'years at current company', 'function', 'title current', 'years of experience', 'company current', 'company headcount', 'company type', 'past company', 'became a member', 'groups find people', 'tags', 'posted content keywords', ] filt_containers = WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'flex.pt4.ph4.pb3.flex-wrap.cursor-pointer'))) geography_handle = filt_containers[2] # # custom_list = filt_containers[3] # industry = filt_containers[4] senioritylevel_handle = filt_containers[9] function_handle = filt_containers[12] companycurrent_handle = filt_containers[15] # for item in filt_containers: # a =' '.join(item.text.replace('\n', ' ').lower().split()[:2]) # exec(a.replace(' ','')+'=item',globals(),locals()) # exec() # for selection in filt_list: # for i in range(0,len(filt_containers)): # if selection.lower() in filt_containers[i].text.replace('\n', ' ').lower(): # # print(filt_containers[i].text.replace('\n', ' ').lower()) # print(filt_list[i].replace(' ','')+f'_handle=filt_containers[{i}]') # # exec(filt_list[i].replace(' ','')+f'_handle=filt_containers[{i}]') # break #%% Linkedin: Filter by Geography def geo(): geography_handle.click() geo_searched_region1 = 'Malaysia' geo_searched_region2 = 'Singapore' geo_enter = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'text-input--no-border.mb4.ember-text-field.ember-view'))) geo_enter.clear() geo_enter.send_keys(geo_searched_region1) time.sleep(1) geo1 = driver.find_elements_by_class_name("button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left") geo1[0].click() # Entering second location geo_enter.clear() geo_enter.send_keys(geo_searched_region2) time.sleep(0.5) geo1 = driver.find_elements_by_class_name("button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left") geo1[0].click() time.sleep(0.5) # geo() #%% Linkedin: Filter by seniority level print('Linkedin: Filter Seniority level') senioritylevel_handle.click() # senior_containers = WebDriverWait(driver,5).until( # EC.presence_of_all_elements_located((By.CLASS_NAME,'ph4.pb4.ember-view'))) # senior_containers = WebDriverWait(driver,5).until( # EC.presence_of_all_elements_located((By.CLASS_NAME,'button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left'))) for selection in sen_level: # print(selection.lower()) senior_containers = WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left'))) for i in range(0,len(senior_containers)): # print(i) if selection.lower() in senior_containers[i].text.lower(): senior_containers[i].click() break #%% Linkedin: Filter by current company print('Linkedin: Filter Company') driver.find_element_by_class_name('advanced-search-filter__list').click() companycurrent_handle.click() company_enter = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'text-input--no-border.ember-text-field.ember-view'))) for company in company_names[0:15]: # print(company) company_enter.clear() company_enter.send_keys(company) company_enter.send_keys(Keys.ENTER) #%% Linkedin: Filter by function print('Linkedin: Filter Job function') function_handle.click() function_enter = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'text-input--no-border.mb4.ember-text-field.ember-view'))) function_enter.clear() function_enter.send_keys(job_fcn) function_click = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'button--unstyled.link-without-visited-state.t-14.font-weight-400.cursor-pointer.search-filter-typeahead__suggestion-item-value.text-align-left'))) function_click.click() #%% Linkedin: Start search print('Linkedin: Start search for suspects') start_search = driver.find_element_by_class_name('button-primary-medium') start_search.click() driver.set_window_size(1657, 923) #just to load all data time.sleep(2) #%% Linkedin: Initialisation name = [] position = [] company = [] location = [] searched_keyword = [] email = [] found_email = "*****@*****.**" page = 1 #%% Linkedin: Scraping prospects prospect_container = WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'pv5.ph2.search-results__result-item'))) f = driver.find_element_by_class_name('search-results__pagination-list').find_elements_by_tag_name('li') print('Linkedin: Webscraping starts') count= 0 try: total_page = int(f[-1].find_element_by_tag_name('button').text) except Exception as e: print(e) total_page = 1 finally: while page <= total_page: articles = WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'pv5.ph2.search-results__result-item'))) for article in articles: count = count + 1 ActionChains(driver).move_to_element(article).perform() """Name""" try: article.find_element_by_class_name('result-lockup__name') # Name article.find_element_by_class_name('result-lockup__highlight-keyword') # Position article.find_element_by_class_name('result-lockup__misc-item') # Location article.find_element_by_class_name('result-lockup__position-company') # Company except NoSuchElementException: # print('var not defined') # name.append("NA") # position.append("NA") # location.append("NA") # company.append("NA") continue else: name.append(article.find_element_by_class_name('result-lockup__name').text) position.append(article.find_element_by_class_name('result-lockup__highlight-keyword').find_element_by_class_name('t-14.t-bold').text) location.append(article.find_element_by_class_name('result-lockup__misc-item').text) company.append(article.find_element_by_class_name('result-lockup__position-company').find_element_by_tag_name('span').text) searched_keyword.append(searched_word) email.append(found_email) print(f'Suspect {count}') next_page = driver.find_element_by_class_name('search-results__pagination-next-button') next_page.find_element_by_class_name('v-align-middle').click() page = page + 1 #%% Linkedin: Close website driver.close() driver.switch_to.window(main_page) driver.close() return name,position,location,company,searched_keyword,email
def find_elements(self, locator, time=10): return WebDriverWait(self.driver, time).until(EC.presence_of_all_elements_located(locator), message=f"Can't find elements by locator {locator}")
def parse_list(self, url, search_company): """ 解析列表页,获取部分信息 :param url: search url :return: item(含部分信息)以及详情url """ while True: self.driver.get(url) current_url = self.driver.current_url if 'login' in current_url: self.login() time.sleep(2) else: break time.sleep(2) try: self.detail_yzm() except Exception: pass try: nodes = self.wait.until(EC.presence_of_all_elements_located((By.XPATH, '//*[@id="searchlist"]//tbody/tr'))) if len(nodes) == 0: detail_url = None item = None return detail_url, item except Exception as E: detail_url = None item = None return detail_url, item if not self.accurate: item = {} if self.type != '3': get_company = self.driver.find_element_by_xpath( '//*[@id="searchlist"]//tr[1]/td[2]/a').text.replace('(', '(').replace(')', ')') item['search_company'] = search_company try: item['company_name'] = get_company except: item['company_name'] = ' ' try: item['another_name'] = self.driver.find_element_by_xpath( '//*[@id="searchlist"]//tbody/tr[1]//i/..').text except: item['another_name'] = ' ' try: item['company_url'] = self.driver.find_element_by_xpath( '//*[@id="searchlist"]//tbody/tr[1]/td[2]/a').get_attribute('href') except: item['company_url'] = ' ' try: item['telphone'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]//span[contains(text(),"电话:")]').text.replace('电话:', '') except: item['telphone'] = ' ' try: nodes = self.driver.find_elements_by_xpath('//*[@id="phoneModal"]/div/div/div[2]/div/div[1]') item['tel_more'] = ';'.join([node.text for node in nodes]) except: item['tel_more'] = ' ' try: item['email'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]//p[contains(text(),"邮箱:")]').text.strip().split(' ')[0].replace('邮箱:', '') except: item['email'] = ' ' try: item['法定代表人'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[1]').text.split('注册资本')[0].replace('法定代表人:', '').strip() except: item['法定代表人'] = ' ' try: capital = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[1]/span[1]').text.replace('注册资本:', '') pattern = re.compile(r'(\d+\.?\d*)') num = pattern.findall(capital)[0] if '人民币' in capital: item['注册资本'] = capital elif '美元' in capital: item['注册资本'] = capital.replace(num, str(int(num)*6.8)).replace('美元', '人民币') elif '日元' in capital: item['注册资本'] = capital.replace(num, str(int(num)*0.06)).replace('日元', '人民币') elif '欧元' in capital: item['注册资本'] = capital.replace(num, str(int(num)*7.9)).replace('欧元', '人民币') elif '港元' in capital: item['注册资本'] = capital.replace(num, str(int(num)*0.87)).replace('港元', '人民币') elif '韩元' in capital: item['注册资本'] = capital.replace(num, str(int(num)*0.006)).replace('韩元', '人民币') elif '新加坡元' in capital: item['注册资本'] = capital.replace(num, str(int(num)*5)).replace('新加坡元', '人民币') elif '瑞士法郎' in capital: item['注册资本'] = capital.replace(num, str(int(num)*6.9)).replace('瑞士法郎', '人民币') elif '新台币' in capital: item['注册资本'] = capital.replace(num, str(int(num)*0.22)).replace('新台币', '人民币') else: item['注册资本'] = capital except: item['注册资本'] = ' ' pattern = re.compile(r'(\d+\.?\d*)') if '股本' not in item['注册资本']: n = pattern.findall(item['注册资本']) if n: item['注册资本(万)'] = n[0] else: item['注册资本(万)'] = ' ' else: item['注册资本(万)'] = ' ' try: item['成立时间'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[1]/span[2]').text.replace('成立时间:', '').split('-')[0] except: item['成立时间'] = ' ' item['成立时间(new)'] = item['成立时间'] + '年' try: item['公司地址'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/p[3]').text.replace('地址:', '') except: item['公司地址'] = ' ' try: item['状态'] = self.driver.find_element_by_xpath('//*[@id="searchlist"]/table/tbody/tr[1]/td[3]/span').text except: item['状态'] = ' ' detail_url = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="searchlist"]/table/tbody/tr[1]/td[2]/a'))).get_attribute('href') return detail_url, item # 精准匹配 if self.accurate == 'on': for node in nodes: company = node.find_element_by_xpath('./td[2]//a').text if search_company.replace('(', '(').replace(')', ')') == company.replace('(', '(').replace(')', ')'): item = {} if self.type != '3': item['search_company'] = search_company item['company_name'] = company.replace('(', '(').replace(')', ')') try: item['company_url'] = node.find_element_by_xpath( './td[2]/a').get_attribute('href') except: item['company_url'] = ' ' try: item['another_name'] = node.find_element_by_xpath( './/i/..').text except: item['another_name'] = ' ' try: item['telphone'] = node.find_element_by_xpath( './/span[contains(text(),"电话:")]').text.strip().replace('电话:', '') except: item['telphone'] = ' ' try: nodes = self.driver.find_elements_by_xpath('//*[@id="phoneModal"]/div/div/div[2]/div/div[1]') item['tel_more'] = ';'.join([node.text for node in nodes]) except: item['tel_more'] = ' ' try: item['email'] = node.find_element_by_xpath( './/p[contains(text(),"邮箱:")]').text.strip().replace('邮箱:', '') except: item['email'] = ' ' try: item['法定代表人'] = node.find_element_by_xpath( './td[2]/p[1]').text.split('注册资本')[0].replace('法定代表人:', '').strip() except: item['法定代表人'] = ' ' try: item['注册资本'] = node.find_element_by_xpath( './td[2]/p[1]/span[1]').text except: item['注册资本'] = ' ' try: item['成立时间'] = node.find_element_by_xpath( './td[2]/p[1]/span[2]').text except: item['成立时间'] = ' ' try: item['公司地址'] = node.find_element_by_xpath( './td[2]/p[3]').text.replace('地址:', '') except: item['公司地址'] = ' ' try: item['状态'] = node.find_element_by_xpath( './td[3]/span').text except: item['状态'] = ' ' detail_url = node.find_element_by_xpath('./td[2]/a').get_attribute('href') return detail_url, item return None, None
def america(): options = webdriver.ChromeOptions() options.add_argument("headless") options.add_argument("disable-gpu") driver = webdriver.Chrome("C:/selenium/chromedriver", options=options) url = "https://kr.investing.com/rates-bonds/u.s.-10-year-bond-yield-historical-data" # 인베스팅_10년물 채권 수익률 driver.get(url) price_list = [] # 일주일치 price 데이터 얻기 - ok for i in range(1, 8): price_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[2]" price_element = WebDriverWait(driver, 25). \ until(EC.presence_of_all_elements_located((By.XPATH, price_xpath))) price_list.append(price_element[0].text) change_list = [] # 일주일치 change 데이터 얻기 - ok for i in range(1, 8): change_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[6]" change_element = WebDriverWait(driver, 25). \ until(EC.presence_of_all_elements_located((By.XPATH, change_xpath))) change_list.append(change_element[0].text) change_integrate = 0 # 14일치 change 데이터 얻기 - ok for i in range(1, 15): change_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[6]" change_element = WebDriverWait(driver, 25). \ until(EC.presence_of_all_elements_located((By.XPATH, change_xpath))) change_element = change_element[0].text change_element = change_element[:-1] # 마지막에 %를 제거하기 change_integrate = change_integrate + float(change_element) # 일주일치 Date 데이터 얻기 - ok date_list = [] for i in range(1, 8): date_xpath = "//*[@id='curr_table']/tbody/tr[" + str(i) + "]/td[1]" date_element = WebDriverWait(driver, 25). \ until(EC.presence_of_all_elements_located((By.XPATH, date_xpath))) date_list.append(date_element[0].text) # 출력할 데이터 - ok show_list = [] show_list.append("미국 10년 채권: ") show_list.append("14일치 누적: " + str(round(change_integrate, 2)) + "%") show_list.append(change_list) show_list.append(driver.current_url) print(show_list) driver.close() # 몇 일 연속으로 오르고 or 내렸는지 화인 plus = 0 # 플러스일 경우 +1 minus = 0 # 마이너스일 경우 +1 plus_plus = 0 # 연속 + minus_minus = 0 # 연속 - stoper = 0 for i in range(0, 7): singleData = change_list[i] # 수급의 플러스, 마이너스 확인하기 if singleData[0] == "-": minus += 1 else: plus += 1 # 순매도의 경우에는 부호 -가 붙지만 # 순매수의 경우에는 부호가 붙지않아서 붙여줌 singleData = "+" + str(singleData) # 연속 + or - 확인하기 if i == 0 and plus == 1: plus_plus = 1 elif i == 0 and minus == 1: minus_minus = 1 # 부호의 경우 마이너스는 -가 붙지만 플러스는 바로 숫자로 표기됨 if i >= 1 and singleData[ 0] == "+" and plus_plus >= 1 and stoper == 0: plus_plus += 1 elif i >= 1 and singleData[ 0] == "-" and plus_plus >= 1 and stoper == 0: stoper = 1 elif i >= 1 and singleData[ 0] == "-" and minus_minus >= 1 and stoper == 0: minus_minus += 1 elif i >= 1 and singleData[ 0] == "+" and minus_minus >= 1 and stoper == 0: stoper = 1 return change_integrate
driver = webdriver.Chrome(path + "\chromedriver.exe") driver.get("https://play.typeracer.com") driver.maximize_window() while True: target = input("What is your target Words Per Minute? ") try: target = int(target) break except ValueError: print("Please type an integer") while True: try: stuff = WebDriverWait(driver, 10).until( EC.presence_of_all_elements_located( (By.XPATH, "//span[@unselectable= 'on']"))) words = [] for i in stuff: words.append(i.text) if len(words) == 3: words[0] = words[0] + words[1] words.pop(1) words = ' '.join(words) words = words.split() if words[1] == ",": words[0] = words[0] + words[1] words.pop(1) speed = (((len(words) * 60) / target) / len(words)) - 0.01 if not words:
def find_elements(self, locator, timeout=10): '''定位一组元素''' elements = WebDriverWait(self.driver, timeout, 1).until(EC.presence_of_all_elements_located(locator)) return elements
def send_private_address_hidden_amount(self): #Send TRY login process driver = Browser().open_browser() #Enter the page driver.get('https://wallet.trias.one/') driver.find_element_by_xpath( '/html/body/div/div/main/div/a[2]/div[1]/h2').click() #Gets the file name under the downloadfile folder dir = os.path.abspath('..') url = dir + "\\downloadfile\\" files = GetFileName().getfilename(url) #Calculate folder length len1 = len(os.listdir(url)) v1 = url + files[len1 - 2] wait = WebDriverWait(driver, 60) wait.until( EC.presence_of_all_elements_located( (By.XPATH, '//*[@id="fselector"]'))) #Upload a file driver.find_element_by_xpath('//*[@id="fselector"]').send_keys(v1) driver.implicitly_wait(5) #Enter the password driver.find_element_by_xpath( '//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input' ).send_keys(123456789) wait.until( EC.element_to_be_clickable( (By.XPATH, '//*[contains(text(),"UNLOCK")]'))) driver.find_element_by_xpath( '//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input' ).send_keys(Keys.TAB) #login time.sleep(9) try: driver.find_element_by_xpath( '/html/body/div/div/main/div/div/article[1]/div[2]/article/div/div[3]/a' ).click() except: print('This element was not found') #Transaction process # Get account amount information time.sleep(2) for i in range(50): time.sleep(1) account_total_amount = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[1]/div[2]/ul/li/span' ).text public_address_amount = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[1]/ul/li/span' ).text public_address_hideen_amount = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[2]/ul/li/span' ).text private_address_amount = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[3]/ul/li/span' ).text private_address_hidden_amount = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[4]/ul/li/span' ).text if int(account_total_amount) != 0: break #Get the name and address of the transfer target account splits = files[len1 - 1].split('--') driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[3]/article/article[1]/div[1]/div[1]/input' ).send_keys(splits[2]) #Random transfer amount rand = random.randint(1, 150) driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[3]/article/article[1]/section/div[2]/div/input' ).send_keys(rand) #Click the hide address button driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[3]/article/article[1]/a[1]' ).click() #Click the hide amount button driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[3]/article/article[1]/a[2]' ).click() #Click the trade button driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[3]/article/article[1]/div[2]/div/a' ).click() time.sleep(3) driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[3]/article/article[2]/section/section/div[2]/button[2]' ).click() #Wait for the trade page to change try: wait.until( EC.presence_of_all_elements_located(( By.XPATH, '/html/body/div/div/main/div/div[1]/article[2]/section/a[2]/span' ))) except: print('Transaction timeout') time.sleep(2) for i in range(100): time.sleep(2) #enter the homepage driver.find_element_by_xpath( '/html/body/div/div/div[2]/div[2]/div/ul/li[1]/a/span').click( ) #Check the view account info driver.find_element_by_xpath( '/html/body/div/div/div[2]/div[2]/div/ul/li[4]/a/span').click( ) #Upload a file driver.find_element_by_xpath('//*[@id="fselector"]').send_keys(v1) driver.implicitly_wait(5) #Enter the password driver.find_element_by_xpath( '//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input' ).send_keys(123456789) wait.until( EC.element_to_be_clickable( (By.XPATH, '//*[contains(text(),"UNLOCK")]'))) driver.find_element_by_xpath( '//*[@id="app"]/div/main/div/div/article[1]/div[2]/article/div/div[2]/input' ).send_keys(Keys.TAB) #login time.sleep(8) try: driver.find_element_by_xpath( '/html/body/div/div/main/div/div/article[1]/div[2]/article/div/div[3]/a' ).click() except: print('This element was not found') #Get account_total_amount again time.sleep(2) for i in range(50): account_total_amount_second = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[1]/div[2]/ul/li/span' ).text time.sleep(2) if int(account_total_amount_second) != 0: break if int(account_total_amount_second) != int(account_total_amount): #Record the account amount and store it in excel public_address_amount2 = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[1]/ul/li/span' ).text public_address_hideen_amount2 = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[2]/ul/li/span' ).text private_address_amount2 = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[3]/ul/li/span' ).text private_address_hidden_amount2 = driver.find_element_by_xpath( '/html/body/div/div/main/div/div/div[2]/div[2]/div[4]/ul/li/span' ).text flag = True if (int(private_address_amount) + int(private_address_hidden_amount) - rand) != (int(private_address_amount2) + int(private_address_hidden_amount2)): flag = False WriteExcel().write_excel_xls_append([ public_address_amount, public_address_hideen_amount, private_address_amount, private_address_hidden_amount, 'send private adress hidden amount: ' + str(rand) + 'TRY', public_address_amount2, public_address_hideen_amount2, private_address_amount2, private_address_hidden_amount2, flag ]) print('The account has been updated. Please check') break time.sleep(5) driver.quit()
def aliba(url2,searched_word): #%% ALIBABA: Set Driver and open webpage print('\nAlibaba: Open URL...') driver = webdriver.Chrome(chrome_driver, options=options) driver.get(url2) driver.maximize_window(); # # For Pop-up # WebDriverWait(driver,5).until( # EC.presence_of_all_elements_located((By.CLASS_NAME,'double-close')))[0].click() #%% ALIBABA - Prepare for webscraping WebDriverWait(driver,5).until(EC.presence_of_all_elements_located((By.CLASS_NAME,'ui-searchbar-keyword'))) driver.find_element_by_class_name('ui-searchbar-keyword').clear() driver.find_element_by_class_name('ui-searchbar-keyword').send_keys(searched_word) time.sleep(3) WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'ui-searchbar-type-display')))[0].click() # a[0].click() sel_supp=WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'ui-searchbar-type-option')))#[0].click() print('Alibaba: Select Supplier, Trade and Verified...') sel_supp[1].click() driver.find_element_by_class_name('ui-searchbar-button-icon').click() trade_check = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.ID,'J-filter-ta'))) trade_check.click() verified = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'supplier-icon.icbu-certificate-icon.icbu-certificate-icon-verified'))) verified.click() pagination = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'ui2-pagination-show'))) pagination_tag=pagination.find_elements_by_tag_name('a') pagination_tag[-1].click() #%% ALIBABA: Webscraping count = 0 page = 1 start_time = time.time() print('Alibaba: Webscraping starts...') company_container = WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'title.ellipsis'))) f1 = WebDriverWait(driver,5).until( EC.presence_of_element_located((By.CLASS_NAME,'ui2-pagination-pages'))) f = f1.find_elements_by_tag_name('a') total_page = int(f[-2].text) while page <= 2:#total_page: #2: try: print(f'Scraping page {page}') page = page + 1 articles = WebDriverWait(driver,3).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'title.ellipsis'))) except TimeoutException as e: print(e) continue else: for article in articles: ActionChains(driver).move_to_element(article).perform() company_names.append(article.text) count = count + 1 print(count) # print(count) next_page = WebDriverWait(driver,5).until( EC.presence_of_all_elements_located((By.CLASS_NAME,'next'))) #[0].click() ActionChains(driver).move_to_element(next_page[0]).click().perform() print(f'Alibaba: Webscraping completed. {count} companies found.') print("Alibaba: Webscraping takes %s seconds ---" % round((time.time() - start_time),1)) #%% ALIBABA: Close website driver.close() print('Alibaba: URL closed')
def test_should_fail_to_find_elements_when_explicit_waiting(driver, pages): pages.load("dynamic.html") with pytest.raises(TimeoutException): WebDriverWait(driver, 0.7).until( EC.presence_of_all_elements_located((By.CLASS_NAME, "redbox")))
# from zhulong.util.etl import gg_meta,gg_html import sys # sys.setrecursionlimit(2000) driver = webdriver.Chrome() url = 'http://www.aqzbcg.org:1102/jyxx/012001/012001002/20181107/8d01c015-f37d-432b-9043-88624601e362.html' driver.get(url) locator = (By.XPATH, '/html/body') WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator)) title = driver.title locator = (By.CLASS_NAME, 'ewb-con') WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located(locator)) before = len(driver.page_source) time.sleep(0.1) after = len(driver.page_source) i = 0 while before != after: before = len(driver.page_source) time.sleep(0.1) after = len(driver.page_source) i += 1 if i > 5: break page = driver.page_source soup = BeautifulSoup(page, 'lxml')
def get_image_title(url): # 웹 드라이버 초기화 driver_path = "C:/SSAFY/Webdriver/chromedriver.exe" driver = webdriver.Chrome(driver_path) driver.implicitly_wait(30) # or bigger second # 열고자 하는 채널 -> 동영상 목록으로 된 url 페이지를 엶 driver.get(url) idx = 1 while True: try: # 영상 썸네일 : img 안에 있는 src 이용 img_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/ytd-thumbnail/a/yt-img-shadow/img' # img_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-shelf-renderer[1]/div[1]/div[2]/ytd-vertical-list-renderer/div[1]/ytd-video-renderer[1]/div[1]/ytd-thumbnail/a/yt-img-shadow/img' # 영상 제목 : a 태그 안에 있는 title 이용 title_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[1]/div/h3/a/yt-formatted-string' # title_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[1]/div/h3/a' # title_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-browse/ytd-two-column-browse-results-renderer/div[1]/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-grid-renderer/div[1]/ytd-grid-video-renderer['+str(idx)+']/div[1]/div[1]/div[1]/h3/a' # 영상 조회수 : span 안에 있는 것 이용 viewcnt_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[1]/ytd-video-meta-block/div[1]/div[2]/span[1]' # viewcnt_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-browse/ytd-two-column-browse-results-renderer/div[1]/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-grid-renderer/div[1]/ytd-grid-video-renderer['+str(idx)+']/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/span[1]' # 영상 url : a 태그 안에 있는 href 이용 url_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/ytd-thumbnail/a' # 채널명 : a 안에 글씨 이용 channel_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[2]/ytd-channel-name/div/div/yt-formatted-string/a' # 채널 이미지 : img 안에 있는 src 이용 channel_img_xpath = '/html/body/ytd-app/div/ytd-page-manager/ytd-search/div[1]/ytd-two-column-search-results-renderer/div/ytd-section-list-renderer/div[2]/ytd-item-section-renderer/div[3]/ytd-video-renderer['+str(idx)+']/div[1]/div/div[2]/a/yt-img-shadow/img' # 이미지가 곧바로 로드 되지 않을 때, 20초간 강제로 기다림 img = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.XPATH, img_xpath))) if img is None: print(idx, 'img is not loaded.') # 동영상 목록을 추가 불러오기 위해 스크롤 내림 if idx % 3 == 0 : driver.execute_script('window.scrollBy(0, 1080);') time.sleep(2) driver.execute_script('window.scrollBy(0, 1080);') time.sleep(2) driver.execute_script('window.scrollBy(0, 1080);') time.sleep(2) # 썸네일 주소를 리스트에 저장 image = driver.find_element_by_xpath(img_xpath) img_url = image.get_attribute('src') image_list.append(img_url) # 타이틀을 리스트에 저장 title = driver.find_element_by_xpath(title_xpath) title_list.append(title.text) # 조회수를 리스트에 저장 view = driver.find_element_by_xpath(viewcnt_xpath) view_list.append(view.text) # 해당 url을 리스트에 저장 urls = driver.find_element_by_xpath(url_xpath) real_url = urls.get_attribute('href') url_list.append(real_url) # 채널명을 리스트에 저장 channel = driver.find_element_by_xpath(channel_xpath) channel_list.append(channel.text) # 채널 이미지를 리스트에 저장 channel_img = driver.find_element_by_xpath(channel_img_xpath) channel_img_url = channel_img.get_attribute('src') channel_img_list.append(channel_img_url) print(idx, title.text, view.text, real_url, channel.text, channel_img_url) idx += 1 except Exception as e: print() print(e) break assert len(image_list) == len(title_list) return image_list, title_list, view_list, url_list, channel_list, channel_img_list
from selenium.webdriver.chrome.options import Options from time import sleep # Definimos el User Agent en Selenium utilizando la clase Options opts = Options() opts.add_argument( "user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/71.0.3578.80 Chrome/71.0.3578.80 Safari/537.36" ) driver = webdriver.Chrome( './chromedriver', chrome_options=opts) # REMPLAZA AQUI EL NOMBRE DE TU CHROME DRIVER driver.get('https://www.jumbo.cl/despensa') links_paginas = WebDriverWait(driver, 10).until( EC.presence_of_all_elements_located( (By.XPATH, '//button[@class="page-number "]'))) links_paginacion = [] for tag_button in links_paginas: links_paginacion.append('https://www.jumbo.cl/despensa?page=' + tag_button.text) for pagina in links_paginacion: print(pagina) driver.get(pagina) links_productos = WebDriverWait(driver, 10).until( EC.presence_of_all_elements_located( (By.XPATH, '//a[@class="shelf-product-title"]'))) links_de_la_pagina = []
import json USERNAME = "******" PASSWORD = "******" hashTag = "#cat" options = Options() options.add_argument('--headless') print("Initializing ...") driver = webdriver.Firefox(options=options) driver.get("https://www.instagram.com/accounts/login/") try: WebDriverWait(driver, 5).until( EC.presence_of_all_elements_located((By.NAME, 'username'))) except: print("Cant Load") # LOGIN print("Logging in as {0} ...".format(USERNAME)) driver.find_element_by_xpath("//input[@type='text']").send_keys(USERNAME) driver.find_element_by_xpath("//input[@type='password']").send_keys(PASSWORD) driver.find_element_by_xpath("//button[@type='submit']").click() not_now = WebDriverWait(driver, 5).until( EC.element_to_be_clickable( (By.XPATH, '//button[contains(text(), "Not Now")]'))).click() not_now = WebDriverWait(driver, 5).until( EC.element_to_be_clickable( (By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
# ブラウザを開き、URLにアクセスする driver = webdriver.Firefox() driver.get(URL) # 検索ボックスに検索ワードを入力して検索する search_box = driver.find_element_by_css_selector("input#id-search-field") search_box.send_keys(SEARCH_WORD) search_box.send_keys(Keys.RETURN) # Nextボタンが無くなるまでループする while True: # ページのロードを待機する try: search_results = WebDriverWait(driver, 5).until( EC.presence_of_all_elements_located( (By.CSS_SELECTOR, "ul.list-recent-events li"))) except TimeoutException: raise # 検索結果を取得する for search_result in search_results: title = search_result.find_element_by_css_selector("h3 a").text url = urljoin( URL, search_result.find_element_by_css_selector("h3 a").get_attribute( "href"), ) description_text = "" descriptions = search_result.find_elements_by_css_selector("p") for description in descriptions: description_text += description.text + "\n"
def wait_dialog(self): confirm_dialog = EC.presence_of_all_elements_located((By.CSS_SELECTOR, '.sweet-alert'))
joined = find_data(soup, CLASS_JOINED).replace("’", ' ') location = find_data(soup, CLASS_LOCATION) market = find_data(soup, CLASS_MARKET) website = find_data(soup, CLASS_WEBSITE) size = find_data(soup, CLASS_SIZE).replace("-", ' to ') raised = find_data(soup, CLASS_RAISED) except Exception as e: print(f'Error: {e.__doc__}') pass else: company_list.append( (company_name, company_description, joined, location, market, website, size, raised, tweeter_link)) print((company_name, company_description, joined, location, market, website, size, raised, tweeter_link)) count += 1 df = pd.DataFrame(company_list, columns=COLOUMNS) df.to_csv('startup.csv', index=False) if __name__ == '__main__': pprint("Extracting Start-UP Company details....") driver, wait = start() driver.get(URL) results = wait.until( EC.presence_of_all_elements_located((By.XPATH, XPATH_COMPANIES))) main(results) stop(driver) pprint("Completed !!!!!!")
def like_latest_posts(**kwargs): # Input arguments - instagram username + driver username = kwargs.get('username') num_posts = kwargs.get('num_posts') driver = kwargs.get('driver') wait = WebDriverWait(driver, 5) # Go to webpage base_url = 'https://www.instagram.com/' full_url = base_url + username driver.get(full_url) time.sleep(1) # Look through posts and like for i in range(num_posts): time.sleep(1) # Get list of latest posts posts = wait.until( EC.presence_of_all_elements_located((By.CLASS_NAME, "v1Nh3"))) time.sleep(random.randint(1, 3)) # Open post post_url = posts[i].find_element_by_css_selector("a").get_attribute( "href") driver.get(post_url) # Get like button like_button = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, "button svg"))) time.sleep(random.randint(1, 3)) # Try button since videos don't have number of likes try: # Original number of likes likes_tag_orig = wait.until( EC.presence_of_element_located((By.CLASS_NAME, "Nm9Fw"))) likes_num_orig = int( reduce(lambda x, y: x + y, re.findall(r'\d+', likes_tag_orig.text))) # Click Like button like_button.click() time.sleep(random.randint(1, 3)) # Number of likes after clicking likes_tag_new = wait.until( EC.presence_of_element_located((By.CLASS_NAME, "Nm9Fw"))) likes_num_new = int( reduce(lambda x, y: x + y, re.findall(r'\d+', likes_tag_new.text))) time.sleep(random.randint(1, 3)) # If already liked, will be unliked & number of likes will go down - in that case click like button again if (likes_num_new < likes_num_orig): like_button = wait.until( EC.presence_of_element_located( (By.CSS_SELECTOR, "button svg"))) time.sleep(random.randint(1, 3)) # Click Like button like_button.click() already_liked = True else: already_liked = False except Exception as e: # Click Like button like_button.click() time.sleep(random.randint(1, 3)) # Go back to user page driver.get(full_url) time.sleep(random.randint(1, 3)) # Check if blocked blocked = check_blocked(driver=driver) return (already_liked, blocked)
for x in tracks: ActionChains(driver).move_by_offset(xoffset=x, yoffset=0).perform() ActionChains(driver).release(element).perform() time.sleep(3) if __name__ == '__main__': driver = webdriver.Chrome() driver.maximize_window() driver.get('http://www.cnbaowen.net/api/geetest/') try: count = 5 # waiting slidingVC loading wait = WebDriverWait(driver, 10) element = wait.until( EC.element_to_be_clickable((By.CLASS_NAME, 'gt_slider_knob'))) while count > 0: main(driver, element) try: succes = wait.until( EC.presence_of_all_elements_located( (By.XPATH, '//div[@class="gt_ajax_tip gt_success"]'))) if succes: print('恭喜你!识别成功...') break except Exception as e: print('识别错误,继续') count -= 1 finally: driver.quit()
search_btn = wait.until( EC.presence_of_element_located( (By.CSS_SELECTOR, 'button.btn-search.tb-bg'))) tb_input.send_keys('移动硬盘') print('搜索...') search_btn.click() ls = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist'))) # 定位总的页数 total = wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, 'div.total'))) while True: # 定位商品的条目 ls = wait.until( EC.presence_of_all_elements_located( (By.XPATH, '//div[contains(@class,"item J_MouserOnverReq")]'))) browser.execute_script('window.scrollTo(0,document.body.scrollHeight)') time.sleep(random.random() * 2) print('len:', len(ls)) for item in ls: title = item.find_elements_by_xpath( './/div[@class="row row-2 title"]/a')[0].text.strip() print('title:', title) price = item.find_elements_by_xpath( './/div[@class="price g_price g_price-highlight"]/strong' )[0].text.strip() print('price:', price) print('=' * 600) browser.execute_script( 'window.scrollTo(0,document.body.scrollHeight-500);')
def process_chunk(state, district, block): try: display = Display(visible=0, size=(1024, 768)) display.start() driver = webdriver.Firefox() driver.get("http://omms.nic.in/Home/") main_menu = WebDriverWait(driver, 10).until( EC.presence_of_element_located((By.CSS_SELECTOR, "a.menuLnk"))) #main_menu = driver.find_element_by_css_selector("a.menuLnk") main_click = main_menu.get_attribute('onclick') time.sleep(3) driver.execute_script("return " + main_click) time.sleep(3) for year in range(2000, 2017): #State: Rajasthan District: Ajmer Block: Arain Year : 2000-2001 Batch: All Batches print "Accessing data for state: ", state, " district: ", district, " block: ", block, " year: ", year, "..." year = str(year) access_string = str( "http://omms.nic.in/MvcReportViewer.aspx?_r=%2fPMGSYCitizen%2fSanctionedProjects&Level=3&State=" + state[0] + "&District=" + district[0] + "&Block=" + block[0] + "&Year=" + year + "&Batch=0&PMGSY=1&DisplayStateName=" + state[1] + "&DisplayDistName=" + district[1] + "&DispBlockName=" + block[1] + "&LocalizationValue=en&BatchName=All+Batches") time.sleep(5) #print access_string driver.get(access_string) #let it load time.sleep(10) #Find the right CSS web element using Chrome elem = WebDriverWait(driver, 10).until( EC.presence_of_all_elements_located( (By.CSS_SELECTOR, "div[id$='208iT0R0x0'] > a"))) time.sleep(3) hov = ActionChains(driver).move_to_element(elem[1]) hov.perform() time.sleep(3) elem[1].click() time.sleep(5) link = driver.find_elements_by_css_selector( 'div#ReportViewer_ctl05_ctl04_ctl00_Menu > div > a') button_name = link[1].get_attribute('onclick') time.sleep(3) try: driver.execute_script("return " + button_name) except: print "Page is being updated" #Allow to finish download time.sleep(5) #harvest(state,district,block,year) driver.close() display.stop() except: print "Unknown exception" pass
def AutoPin(self, username, password, pinurl, board): try: self.driver.get( 'https://www.pinterest.com/login/?referrer=home_page') self.driver.find_element_by_xpath( '//form//input[@type="email"]').send_keys(username) self.driver.find_element_by_xpath( '//form//input[@type="password"]').send_keys(password) self.driver.find_element_by_xpath('//form//button').click() logger.info('Successfully log in pinterest!') time.sleep(3) except Exception: logger.error('Failed log in pinterest!') return try: self.driver.get(pinurl) wait = WebDriverWait(self.driver, self.WAIT_TIME) results = wait.until( EC.presence_of_all_elements_located( (By.XPATH, '//dd[@class="prodName"]/a'))) urlList = [result.get_attribute('href') for result in results] if urlList: for url in urlList: logger.info('Auto pin url : %s' % url) self.driver.get(url) pinnedImages = [] while True: try: currentImage = self.driver.find_element_by_xpath( '//div[@id="w-featurePics"]/a').get_attribute( 'href') if currentImage in pinnedImages: break else: pinnedImages.append(currentImage) logger.info('Auto pin Image : %s' % currentImage) self.driver.find_element_by_xpath( '//div[@id="w-featurePics"]/div').click() if len(self.driver.window_handles) > 1: self.driver.switch_to.window( self.driver.window_handles[-1]) self.driver.find_element_by_xpath( '//div[@role="button"]//p[contains(text(), "%s")]' % board).click() time.sleep(2) self.driver.close() self.driver.switch_to.window( self.driver.window_handles[0]) self.driver.find_element_by_xpath( '//div[@id="prodthumbnails"]/div/a[1]').click( ) time.sleep(2) except Exception: logger.error( 'Failed auto pin! Continue another to pin') break except Exception: logger.error('Failed auto pin!') return
driver.find_elements_by_xpath( '//*[@id="root"]/main/div/section[2]//img') c_i = True except: c_i = False cover_images = "Not Available" pass if c_i: cover_images = [] s = 0 while True: try: elements = WebDriverWait(driver, 5).until( EC.presence_of_all_elements_located( (By.XPATH, '//*[@id="root"]/main/div/section[2]//img'))) element = elements[s] actions = ActionChains(driver) actions.move_to_element(element).perform() cover_image_url = str(element.get_attribute("src")) img_filename = url.split( '/')[4] + "_coverimage_" + f"{s}" + ".webp" # for f in os.listdir('./images'): # if fnmatch.fnmatch(f,img_filename): pass # else: cover_image = requests.get( cover_image_url, headers={ 'user-agent': 'Mozilla/5.0 (Linux; U; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13'
def wait_for_ajs_dialog(self): confirm_ajs_dialog = EC.presence_of_all_elements_located((By.CLASS_NAME, 'ajs-body')) WebDriverWait(self.web_element, 10).until(confirm_ajs_dialog)
def hotel(self): css_sel = By.CSS_SELECTOR, 'a.flex-link' return WebDriverWait(self.driver, 3).until( EC.presence_of_all_elements_located(css_sel))[self.hotel_num]
def startSteps(self): pTxt = "\n-------- Step 'C007' started!!! --------------------------------------------------------------------" print(pTxt) self.driver = webdriver.Firefox() self.driver.maximize_window() ''' 1. Navigate to staging.getkumbu.com ''' pTxt = "\n1. Navigate to staging.getkumbu.com\n" print(pTxt) try: self.driver.get(self.url) pTxt = "\t\t(Success)\tLoad webpage successfully" print(pTxt) except: pTxt = "\t\t(Error)\tFailed to load webpage" print(pTxt) self.driver.quit() return ''' 2. Input email adress: [email protected] ''' pTxt = "\n2. Input email adress: [email protected]\n" print(pTxt) try: inputs = WebDriverWait(self.driver, 50).until( EC.presence_of_all_elements_located( (By.CSS_SELECTOR, "input.kumbu-input"))) except: pTxt = "\t\t(Error)\tCan't find 'Email' and 'Password' Inputs" print(pTxt) self.driver.quit() return try: email_input = inputs[0] email_input.send_keys(self.email) pTxt = "\t\t(Success)\tInput email successfully" print(pTxt) except: pTxt = "\t\t(Error)\tFailed to input 'email'" print(pTxt) self.driver.quit() return ''' 3. Input password: “kumbu is cool” ''' pTxt = "\n3. Input password: '******'\n" print(pTxt) try: pwd_input = inputs[1] pwd_input.send_keys(self.password) pTxt = "\t\t(Success)\tInputted 'Password' successfully" print(pTxt) except: pTxt = "\t\t(Error)\tFailed to input 'Password'" print(pTxt) self.driver.quit() return ''' 4. Click Sign in ''' pTxt = "\n4. Click Sign in\n" print(pTxt) try: submit_btn = WebDriverWait(self.driver, 50).until( EC.element_to_be_clickable( (By.CSS_SELECTOR, "input#login-submit"))) submit_btn.click() pTxt = "\t\t(Success)\tClicked 'Sign in' button. Logged in successfully" print(pTxt) except: pTxt = "\t\t(Error)\tFailed to click 'Sign in' button" print(pTxt) self.driver.quit() return ''' 5. Click on 'Collection for Test $TEST_NUMBER' ''' pTxt = "\n5. Click on 'Collection for Test $TEST_NUMBER'\n" print(pTxt) try: collections = WebDriverWait(self.driver, 50).until( EC.visibility_of_all_elements_located( (By.CSS_SELECTOR, "div.collection.columns.small-12.medium-3.text-center"))) flag = False for collection in collections: if self.collection_txt in collection.text.strip(): flag = True break if flag: try: collection.click() pTxt = "\t\t(Success)\tClicked Successfully" print(pTxt) except: pTxt = "\t\t(Error)\tFailed to click" print(pTxt) self.driver.quit() return else: pTxt = "\t\t(Failure)\tFailed to click" print(pTxt) self.driver.quit() return except: pTxt = "\t\t(Error)\tFailed to click" print(pTxt) self.driver.quit() return ''' 6. Click on the image whose src starts with /item/thumbnail ''' pTxt = "\n6. Click on the image whose src starts with /item/thumbnail\n" print(pTxt) try: images = WebDriverWait(self.driver, 50).until( EC.presence_of_all_elements_located( (By.CSS_SELECTOR, "div.item.columns.small-6.medium-2.upload"))) if len(images) > 0: #images[0].find_element_by_tag_name["a"].click() images[0].click() pTxt = "\t\t(Success)\tImage is clicked" print(pTxt) else: pTxt = "\t\t(Failure)\tImage can not be clickable" print(pTxt) except: pTxt = "\t\t(Error)\tImage can not be clickable" print(pTxt) self.driver.quit() return ''' 7. Verify that 'Appears in 1 collection' is visible ''' pTxt = "\n7. Verify that 'Appears in 1 collection' is visible\n" print(pTxt) try: txt1 = WebDriverWait(self.driver, 50).until( EC.presence_of_element_located( (By.CSS_SELECTOR, "div.add-collection-action > h4"))) if "Appears in 1 collection" in txt1.text.strip(): pTxt = "\t\t(Success)\t'Appears in 1 collection' is visible" print(pTxt) else: pTxt = "\t\t(Failure\t'Appears in 1 collection' is not visible" except: pTxt = "\t\t(Error)\t'Appears in 1 collection' is not visible" print(pTxt) self.driver.quit() return ''' 8. Verify that an image is visible on the right side ''' pTxt = "\n8. Verify that an image is visible on the right side\n" print(pTxt) try: img1 = WebDriverWait(self.driver, 50).until( EC.presence_of_element_located( (By.CSS_SELECTOR, "div.item-content > div.picture-item > img"))) pTxt = "\t\t(Success)\tAn image is visible on the right side" print(pTxt) except: pTxt = "\t\t(Error)\tNo image is visible on the right side" print(pTxt) self.driver.quit() return ''' 9. Click on item-title and replace with 'Test Image $TEST_NUMBER' ''' pTxt = "\n9. Click on item-title and replace with 'Test Image $TEST_NUMBER'\n" print(pTxt) try: item_title = WebDriverWait(self.driver, 50).until( EC.presence_of_element_located( (By.CSS_SELECTOR, "div.meta-info > h2"))) actionChains = ActionChains(self.driver) actionChains.click(item_title).key_down(Keys.CONTROL).send_keys('a').key_up(Keys.CONTROL) \ .send_keys(Keys.DELETE).send_keys(self.img_text).perform() pTxt = "\t\t(Success)\t" print(pTxt) except: pTxt = "\t\t(Error)\t" print(pTxt) self.driver.quit() return ''' 10. Click on the little checkmark ''' pTxt = "\n10. Click on the little checkmark\n" print(pTxt) try: check_btn = WebDriverWait(self.driver, 50).until( EC.presence_of_element_located( (By.CSS_SELECTOR, "div.meta-info > a.confirm-item-title-edition"))) check_btn.click() pTxt = "\t\t(Success)\t" print(pTxt) except: pTxt = "\t\t(Error)\t" print(pTxt) self.driver.quit() return ''' 11. Verify that 'Test Image $TEST_NUMBER' is visible ''' pTxt = "\n11. Verify that 'Test Image $TEST_NUMBER' is visible\n" print(pTxt) try: item_title = WebDriverWait(self.driver, 50).until( EC.presence_of_element_located( (By.CSS_SELECTOR, "div.meta-info > h2"))) if self.img_text in item_title.text.strip(): pTxt = "\t\t(Success)\t'{}' is visible".format(self.img_text) print(pTxt) else: pTxt = "\t\t(Failure)\t'{}' is not visible".format( self.img_text) print(pTxt) except: pTxt = "\t\t(Error)\t'{}' is not visible".format(self.img_text) print(pTxt) self.driver.quit() return self.driver.quit() return
from selenium.webdriver.support import expected_conditions from selenium.webdriver.common.by import By # Load photo-sharing site geckodriver_path = '/home/alex/Downloads/geckodriver' driver = webdriver.Firefox(executable_path=geckodriver_path) driver.get('https://www.flickr.com') # Search category of photos el_search_field = driver.find_element_by_id('search-field') el_search_field.send_keys('cats' + Keys.RETURN) # Click photo first photo el_photo_list = WebDriverWait(driver, 20). \ until(expected_conditions.presence_of_all_elements_located((By.CSS_SELECTOR, "div.view.photo-list-view > div"))) el_photo = el_photo_list[0] WebDriverWait(driver, 20). \ until(expected_conditions.visibility_of(el_photo)) el_photo.click() # Find the URL of the photo. for i in range(20): el_main_photo = WebDriverWait(driver, 20). \ until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, "img.main-photo"))) print(el_main_photo.get_attribute('id'))