def get_post(target_url): while 1: try: option.add_argument(f"--proxy-server={random.choice([i['ip'] for i in client.ips.ips.find()])}") break except:time.sleep(0.5) option.add_argument(random.choice(user_agent_list)) option.add_experimental_option('w3c', False) driver = webdriver.Chrome(chrome_options=option) # driver.maximize_window() try: driver.get(target_url) print(target_url) ui.WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, '//a[@class="block"]'))) except TimeoutException as e: driver.quit() print(e) return button = driver.find_element_by_xpath('//a[@class="block"]') Action = TouchActions(driver) Action.scroll_from_element(button, 0, 5000).perform() datas = driver.find_elements_by_tag_name('a') urls = [i for i in [i.get_attribute('href') for i in datas if i.get_attribute('href')] if 'detail' in i] for url in urls: try: client.all_post.mogujie_detail_url.insert({'_id': url, 'status': 0}) print('over:', url) except Exception as e: print(e) client.all_post.mogujie_user.update({'_id': target_url}, {'$set': {'status': 1}}) driver.quit()
def get_comment(self): try: self.fast_click_first_item_same_page_by_partial_link_text(link_text='查看全部') time.sleep(5) button = self.driver.find_element_by_css_selector( css_selector='ul.commit-wrapper-0 > li:nth-child(1)') Action = TouchActions(self.driver) Action.scroll_from_element(on_element=button, xoffset=0, yoffset=int(10000)).perform() Action.scroll_from_element(on_element=button, xoffset=0, yoffset=int(0)).perform() time.sleep(5) self.from_page_get_data_list(page=page_comment_1) except Exception: print("没有评论") time.sleep(5) try: self.fast_click_page_by_css_selector(click_css_selector='li.card.status-return.fold') except Exception as e: print("无元素")
def get_user(): driver = webdriver.Chrome(chrome_options=option) driver.get('https://www.mogu.com/') # ui.WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, '//div[@class="column-wrap"]'))) time.sleep(5) button = driver.find_element_by_xpath('//a') Action = TouchActions(driver) Action.scroll_from_element(button, 0, 10000).perform()
def down_refresh_data(self): """下拉刷新""" Action = TouchActions(self.driver) clickHoldElement = self.by_xpath_name( '//*[@id="191936765"]/div[1]/span') Action.scroll_from_element(clickHoldElement, 0, -200).perform() # 下拉刷新是负数 time.sleep(3)
def async_loading(self): """上拉分页""" time.sleep(3) Action = TouchActions(self.driver) clickHoldElement = self.by_xpath_name( '//*[@id="191936765"]/div[1]/span') Action.scroll_from_element(clickHoldElement, 0, 2500).perform() time.sleep(3)
def get_comment_info_list(self): shop_collcetion = Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection, host='localhost').get_collection() shop_name_url_list = list() for i in shop_collcetion.find(self.get_data_key()): if i.get('shop_comment_url'): shop_name_url_list.append( (i.get('shop_name'), i.get('shop_comment_url'))) for i in range(len(shop_name_url_list)): # 可能会有反爬 self.info_log(data='第%s个,%s' % (i + 1, shop_name_url_list[i][0])) self.shop_name = shop_name_url_list[i][0] self.fast_new_page(url=shop_name_url_list[i][1]) time.sleep(5) # main-page > header > h2 > div:nth-child(2) try: #查看是否有顶部按钮 有就点击 dianping = self.driver.find_element_by_css_selector( css_selector='#main-page > header > h2 > div:nth-child(2)') self.fast_click_same_page_by_css_selector( click_css_selector= '#main-page > header > h2 > div:nth-child(2)') time.sleep(6) except Exception as e: print(111) #点击最新的 try: new = self.driver.find_element_by_xpath( '//li[@data-tagtype="44"]') ActionChains(self.driver).click(new).perform() time.sleep(5) except Exception as e: print(222) #向下进行滚动 try: button = self.driver.find_element_by_css_selector( css_selector= '#main-page > div.mp-comment-mpcon > div.mp-addcomment.mp-border-top > a > div' ) Action = TouchActions(self.driver) Action.scroll_from_element(on_element=button, xoffset=0, yoffset=int(8000)).perform() time.sleep(5) except Exception as e: print(333) self.fast_click_same_page_by_css_selector( click_css_selector='#main-page > div.mp-gotop > div') time.sleep(6) comment_data_list = self.from_page_get_data_list( page=page_comment_1)
def test_touchaction_scrollbotton(self): self.driver.get("https://www.baidu.com/") ele = self.driver.find_element_by_id("kw") ele_search = self.driver.find_element_by_id("su") ele.send_keys("selenium测试") action = TouchActions(self.driver) action.tap(ele_search).perform() action.scroll_from_element(ele, 0, 10000).perform() sleep(3)
def scroll_from_element(self, locator, x_offset, y_offset): """ Scrolls starting from ``locator``. ``x_offset`` is the X offset to scroll to. ``y_offset`` is the Y offset to scroll to.""" element = self._element_find(locator, True, True) action = TouchActions(self._current_application()) action.scroll_from_element(element, x_offset, y_offset).perform()
def test_touchaction(self): self.driver.get("https://www.baidu.com") touchaction = TouchActions(self.driver) el = self.driver.find_element_by_id("kw") el_search = self.driver.find_element_by_id("su") el.send_keys('健身') el_search.click() touchaction.scroll_from_element(el, 0, 10000).perform() time.sleep(5)
def test_touchaction_scroll(self): self.driver.get("http://www.baidu.com") element = self.driver.find_element(By.CSS_SELECTOR, '#kw') element.send_keys("selenium测试") search_element = self.driver.find_element_by_id('su') action = TouchActions(self.driver).tap(search_element) # action.perform() action.scroll_from_element(search_element, 0, 10000).perform() time.sleep(3)
def get(url): try: driver = webdriver.Chrome(chrome_options=option) driver.get(url) WebDriverWait(driver, 5) button = driver.find_element_by_tag_name('a') Action = TouchActions(driver) Action.scroll_from_element(button, 0, 30000).perform() client.all_post.wao_url.update({'_id': url}, {'$set': {'status': 1}}) driver.quit() except Exception as e: print(e)
def tryScrolling(driver): Action = TouchActions(driver) """从button元素像下滑动200元素,以50的速度向下滑动""" for i in range(2): time.sleep(5) # WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.CSS_SELECTOR,))) Action.scroll_from_element( driver.find_element_by_css_selector( "[class='mint-loadmore-bottom']"), 0, 200).perform() sections, sectionsText = [], [] sections, sectionsText = refreshSections(startPoint=listLen) return sections, sectionsText
def auto_data_preview(driver): wait = WebDriverWait(driver, 30) # 1. 起一个action实例 actions = ActionChains(driver) # 2. 点击'数据预览' wait.until(EC.visibility_of_element_located((By.LINK_TEXT, '数据预览'))) data_presee = driver.find_element_by_link_text('数据预览') actions.move_to_element(data_presee).perform() wait_for(2) data_presee.click() wait_for(2) # 3. 输入'csv文件路径'和'读入行数' 并点击提交 wait.until(EC.visibility_of_element_located((By.ID, 'file_list'))) wait.until(EC.visibility_of_element_located((By.ID, 'nrows_list'))) wait.until( EC.visibility_of_element_located((By.XPATH, '//*[@id="csv-sumbit"]'))) file_list = driver.find_element_by_id('file_list') nrows_list = driver.find_element_by_id('nrows_list') csv_submit = driver.find_element_by_xpath('//*[@id="csv-sumbit"]') file_list.clear() wait_for(1) file_list.send_keys('heart.csv') wait_for(1) nrows_list.clear() wait_for(1) nrows_list.send_keys('1000') wait_for(1) csv_submit.click() wait_for(2) # 4. 点开df_heart 并展现读入的数据 再关闭 wait.until( EC.visibility_of_element_located( (By.XPATH, '//*[@id="accordion"]/div/div[1]/a'))) df_heart_view = driver.find_element_by_xpath( '//*[@id="accordion"]/div/div[1]/a') actions = ActionChains(driver) actions.move_to_element(df_heart_view).perform() wait_for(1) df_heart_view.click() wait_for(2) touch_actions = TouchActions(driver) touch_actions.scroll_from_element(df_heart_view, 0, 400).perform() wait_for(2) df_heart_view.click() wait_for(2) # 5. 展示已经读入HDFStore的文件 wait.until( EC.visibility_of_element_located((By.XPATH, '//*[@id="df_sumbit"]'))) wait.until( EC.visibility_of_element_located((By.XPATH, '//*[@id="store_path"]'))) df_submit = driver.find_element_by_xpath('//*[@id="df_sumbit"]') store_path = driver.find_element_by_xpath('//*[@id="store_path"]') store_path.clear() wait_for(1) store_path.send_keys('data_curation.h5') wait_for(1) df_submit.click() wait_for(1) wait.until( EC.visibility_of_element_located( (By.XPATH, '//*[@id="accordion"]/div/div[1]/a'))) df_show = driver.find_element_by_xpath('//*[@id="accordion"]/div/div[1]/a') df_show.click() wait_for(1) touch_actions = TouchActions(driver) touch_actions.scroll_from_element(df_show, 0, 800).perform() wait_for(1) df_show.click() wait_for(2)
def set_font_big(self): """设置大字体""" Action = TouchActions(self.driver) clickHoldElement = self.by_xpath_name('/html/body/div[3]/ul/li[2]/div/input') Action.scroll_from_element(clickHoldElement, -100, 0).perform() # 左加右减
time.sleep(2) # 选择单位证件类型为 其他证件 com_id_other_type = chrome_001.find_element_by_xpath( '//*[@id="proveReturn"]') com_id_other_type.click() time.sleep(1) com_id_other_type_chose = chrome_001.find_element_by_xpath( '//*[@id="proveTypeName"]') com_id_other_type_chose.click() time.sleep(1) # 定义触控,下拉框选择 action_001 = TouchActions(chrome_001) com_id_other_type_00 = chrome_001.find_element_by_xpath( '//*[@id="oneLevelContain"]/ul/li[1]') action_001.scroll_from_element(com_id_other_type_00, 0, 200) action_001.perform() time.sleep(2) # 单位其他证件类型选择OK com_id_other_type_ok = chrome_001.find_element_by_xpath( '/html/body/div[9]/div/header/a[2]') com_id_other_type_ok.click() com_id_number = data_plate_file_xl.worksheets[0].cell( data_row + 1, 2).value # excel中行列是从1开始数 com_id_number_input = chrome_001.find_element_by_xpath( '//*[@id="proveNoNew"]') com_id_number_input.send_keys(com_id_number) try: # 是否法人亲临
#coding=utf-8 import time from selenium import webdriver from selenium.webdriver.common.touch_actions import TouchActions """设置手机的大小""" driver = webdriver.Chrome() driver.get('http://m.test.90dichan.com') driver.maximize_window() """定位操作元素""" time.sleep(2) button = driver.find_element_by_xpath( '//*[@id="pullrefresh"]/div[2]/ul/li[2]/a/div[2]/span') time.sleep(3) """从button元素像下滑动200元素""" Action = TouchActions(driver) Action.scroll_from_element(button, 0, -200).perform() time.sleep(3) driver.close()
# It's a bit of an annoying process -- in order for the driver to # click to expand a menu, the clickable item must be on the screen. # I figured out the best way to do this is to expand the list of states # starting at the bottom and working up. us_elem = driver.find_elements_by_css_selector('div.' + class_name + '.row')[0] elements = driver.find_elements_by_css_selector('div.' + class_name + '.stat.row') elements = [us_elem] + elements driver.execute_script("arguments[0].scrollIntoView(true);", elements[1]) elements[1].click() len(elements) # Iterate through the elements (states) and click to expand each for i in range(len(elements)-3,-1,-1): driver.execute_script("arguments[0].scrollIntoView(true);", elements[i]); touchAction = TouchActions(driver) touchAction.scroll_from_element(on_element=elements[i],xoffset=0,yoffset=-1) print(i+2) elements[i+2].click() touchAction = TouchActions(driver) # Pull elements which contain the state and county data state_elements = driver.find_elements_by_class_name(class_name + '.stat.row.expand') county_elements = driver.find_elements_by_class_name(class_name + '.counties') # Define dictionary where # keys are state names # items are dataframes which have state + county data master_dict = {} # Iterate through each state - county group for state,counties in zip(state_elements,county_elements):
"[class=mint-loadmore-bottom]") # 去除广告阻挡 # 方案一 点击关闭,在设置不加载图片后 失效 # driver.find_element_by_css_selector("[class='unit-0 X flex-middle']").click() # 方案二 隐藏元素 js = "document.querySelector(\"[class='popInfo']\").style.display='none';" driver.execute_script(js) # 疯狂上拉 Action = TouchActions(driver) """从button元素像下滑动200元素,以50的速度向下滑动""" for i in range(6): time.sleep(5) # WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.CSS_SELECTOR,))) Action.scroll_from_element(bottoms[3], 0, 200).perform() # time.sleep(5) sections, sectionsText = refreshSections(sections, sectionsText, 1) listLen = len(sections) #tempFlag = 21 for sec in range(listLen): # if sec = tempFlag: # continue # 点击进入二层节目列表 try: sections[sec].click() except StaleElementReferenceException: