def get_post(target_url):
while 1:
try:
option.add_argument(f"--proxy-server={random.choice([i['ip'] for i in client.ips.ips.find()])}")
break
except:time.sleep(0.5)
option.add_argument(random.choice(user_agent_list))
option.add_experimental_option('w3c', False)
driver = webdriver.Chrome(chrome_options=option)
# driver.maximize_window()
try:
driver.get(target_url)
print(target_url)
ui.WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, '//a[@class="block"]')))
except TimeoutException as e:
driver.quit()
print(e)
return
button = driver.find_element_by_xpath('//a[@class="block"]')
Action = TouchActions(driver)
Action.scroll_from_element(button, 0, 5000).perform()
datas = driver.find_elements_by_tag_name('a')
urls = [i for i in [i.get_attribute('href') for i in datas if i.get_attribute('href')] if 'detail' in i]
for url in urls:
try:
client.all_post.mogujie_detail_url.insert({'_id': url, 'status': 0})
print('over:', url)
except Exception as e:
print(e)
client.all_post.mogujie_user.update({'_id': target_url}, {'$set': {'status': 1}})
driver.quit()
def get_comment(self):
try:
self.fast_click_first_item_same_page_by_partial_link_text(link_text='查看全部')
time.sleep(5)
button = self.driver.find_element_by_css_selector(
css_selector='ul.commit-wrapper-0 > li:nth-child(1)')
Action = TouchActions(self.driver)
Action.scroll_from_element(on_element=button, xoffset=0, yoffset=int(10000)).perform()
Action.scroll_from_element(on_element=button, xoffset=0, yoffset=int(0)).perform()
time.sleep(5)
self.from_page_get_data_list(page=page_comment_1)
except Exception:
print("没有评论")
time.sleep(5)
try:
self.fast_click_page_by_css_selector(click_css_selector='li.card.status-return.fold')
except Exception as e:
print("无元素")
def get_user():
driver = webdriver.Chrome(chrome_options=option)
driver.get('https://www.mogu.com/')
# ui.WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, '//div[@class="column-wrap"]')))
time.sleep(5)
button = driver.find_element_by_xpath('//a')
Action = TouchActions(driver)
Action.scroll_from_element(button, 0, 10000).perform()
def down_refresh_data(self):
"""下拉刷新"""
Action = TouchActions(self.driver)
clickHoldElement = self.by_xpath_name(
'//*[@id="191936765"]/div[1]/span')
Action.scroll_from_element(clickHoldElement, 0,
-200).perform() # 下拉刷新是负数
time.sleep(3)
def async_loading(self):
"""上拉分页"""
time.sleep(3)
Action = TouchActions(self.driver)
clickHoldElement = self.by_xpath_name(
'//*[@id="191936765"]/div[1]/span')
Action.scroll_from_element(clickHoldElement, 0, 2500).perform()
time.sleep(3)
def get_comment_info_list(self):
shop_collcetion = Mongodb(db=TravelDriver.db,
collection=TravelDriver.shop_collection,
host='localhost').get_collection()
shop_name_url_list = list()
for i in shop_collcetion.find(self.get_data_key()):
if i.get('shop_comment_url'):
shop_name_url_list.append(
(i.get('shop_name'), i.get('shop_comment_url')))
for i in range(len(shop_name_url_list)):
# 可能会有反爬
self.info_log(data='第%s个,%s' % (i + 1, shop_name_url_list[i][0]))
self.shop_name = shop_name_url_list[i][0]
self.fast_new_page(url=shop_name_url_list[i][1])
time.sleep(5)
# main-page > header > h2 > div:nth-child(2)
try:
#查看是否有顶部按钮 有就点击
dianping = self.driver.find_element_by_css_selector(
css_selector='#main-page > header > h2 > div:nth-child(2)')
self.fast_click_same_page_by_css_selector(
click_css_selector=
'#main-page > header > h2 > div:nth-child(2)')
time.sleep(6)
except Exception as e:
print(111)
#点击最新的
try:
new = self.driver.find_element_by_xpath(
'//li[@data-tagtype="44"]')
ActionChains(self.driver).click(new).perform()
time.sleep(5)
except Exception as e:
print(222)
#向下进行滚动
try:
button = self.driver.find_element_by_css_selector(
css_selector=
'#main-page > div.mp-comment-mpcon > div.mp-addcomment.mp-border-top > a > div'
)
Action = TouchActions(self.driver)
Action.scroll_from_element(on_element=button,
xoffset=0,
yoffset=int(8000)).perform()
time.sleep(5)
except Exception as e:
print(333)
self.fast_click_same_page_by_css_selector(
click_css_selector='#main-page > div.mp-gotop > div')
time.sleep(6)
comment_data_list = self.from_page_get_data_list(
page=page_comment_1)
def test_touchaction_scrollbotton(self):
self.driver.get("https://www.baidu.com/")
ele = self.driver.find_element_by_id("kw")
ele_search = self.driver.find_element_by_id("su")
ele.send_keys("selenium测试")
action = TouchActions(self.driver)
action.tap(ele_search).perform()
action.scroll_from_element(ele, 0, 10000).perform()
sleep(3)
def scroll_from_element(self, locator, x_offset, y_offset):
""" Scrolls starting from ``locator``.
``x_offset`` is the X offset to scroll to.
``y_offset`` is the Y offset to scroll to."""
element = self._element_find(locator, True, True)
action = TouchActions(self._current_application())
action.scroll_from_element(element, x_offset, y_offset).perform()
def test_touchaction(self):
self.driver.get("https://www.baidu.com")
touchaction = TouchActions(self.driver)
el = self.driver.find_element_by_id("kw")
el_search = self.driver.find_element_by_id("su")
el.send_keys('健身')
el_search.click()
touchaction.scroll_from_element(el, 0, 10000).perform()
time.sleep(5)
def test_touchaction_scroll(self):
self.driver.get("http://www.baidu.com")
element = self.driver.find_element(By.CSS_SELECTOR, '#kw')
element.send_keys("selenium测试")
search_element = self.driver.find_element_by_id('su')
action = TouchActions(self.driver).tap(search_element)
# action.perform()
action.scroll_from_element(search_element, 0, 10000).perform()
time.sleep(3)
def get(url):
try:
driver = webdriver.Chrome(chrome_options=option)
driver.get(url)
WebDriverWait(driver, 5)
button = driver.find_element_by_tag_name('a')
Action = TouchActions(driver)
Action.scroll_from_element(button, 0, 30000).perform()
client.all_post.wao_url.update({'_id': url}, {'$set': {'status': 1}})
driver.quit()
except Exception as e:
print(e)
def tryScrolling(driver):
Action = TouchActions(driver)
"""从button元素像下滑动200元素，以50的速度向下滑动"""
for i in range(2):
time.sleep(5)
# WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.CSS_SELECTOR,)))
Action.scroll_from_element(
driver.find_element_by_css_selector(
"[class='mint-loadmore-bottom']"), 0, 200).perform()
sections, sectionsText = [], []
sections, sectionsText = refreshSections(startPoint=listLen)
return sections, sectionsText
def auto_data_preview(driver):
wait = WebDriverWait(driver, 30)
# 1. 起一个action实例
actions = ActionChains(driver)
# 2. 点击'数据预览'
wait.until(EC.visibility_of_element_located((By.LINK_TEXT, '数据预览')))
data_presee = driver.find_element_by_link_text('数据预览')
actions.move_to_element(data_presee).perform()
wait_for(2)
data_presee.click()
wait_for(2)
# 3. 输入'csv文件路径'和'读入行数' 并点击提交
wait.until(EC.visibility_of_element_located((By.ID, 'file_list')))
wait.until(EC.visibility_of_element_located((By.ID, 'nrows_list')))
wait.until(
EC.visibility_of_element_located((By.XPATH, '//*[@id="csv-sumbit"]')))
file_list = driver.find_element_by_id('file_list')
nrows_list = driver.find_element_by_id('nrows_list')
csv_submit = driver.find_element_by_xpath('//*[@id="csv-sumbit"]')
file_list.clear()
wait_for(1)
file_list.send_keys('heart.csv')
wait_for(1)
nrows_list.clear()
wait_for(1)
nrows_list.send_keys('1000')
wait_for(1)
csv_submit.click()
wait_for(2)
# 4. 点开df_heart 并展现读入的数据 再关闭
wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//*[@id="accordion"]/div/div[1]/a')))
df_heart_view = driver.find_element_by_xpath(
'//*[@id="accordion"]/div/div[1]/a')
actions = ActionChains(driver)
actions.move_to_element(df_heart_view).perform()
wait_for(1)
df_heart_view.click()
wait_for(2)
touch_actions = TouchActions(driver)
touch_actions.scroll_from_element(df_heart_view, 0, 400).perform()
wait_for(2)
df_heart_view.click()
wait_for(2)
# 5. 展示已经读入HDFStore的文件
wait.until(
EC.visibility_of_element_located((By.XPATH, '//*[@id="df_sumbit"]')))
wait.until(
EC.visibility_of_element_located((By.XPATH, '//*[@id="store_path"]')))
df_submit = driver.find_element_by_xpath('//*[@id="df_sumbit"]')
store_path = driver.find_element_by_xpath('//*[@id="store_path"]')
store_path.clear()
wait_for(1)
store_path.send_keys('data_curation.h5')
wait_for(1)
df_submit.click()
wait_for(1)
wait.until(
EC.visibility_of_element_located(
(By.XPATH, '//*[@id="accordion"]/div/div[1]/a')))
df_show = driver.find_element_by_xpath('//*[@id="accordion"]/div/div[1]/a')
df_show.click()
wait_for(1)
touch_actions = TouchActions(driver)
touch_actions.scroll_from_element(df_show, 0, 800).perform()
wait_for(1)
df_show.click()
wait_for(2)
def set_font_big(self):
"""设置大字体"""
Action = TouchActions(self.driver)
clickHoldElement = self.by_xpath_name('/html/body/div[3]/ul/li[2]/div/input')
Action.scroll_from_element(clickHoldElement, -100, 0).perform() # 左加右减
time.sleep(2)
# 选择单位证件类型为 其他证件
com_id_other_type = chrome_001.find_element_by_xpath(
'//*[@id="proveReturn"]')
com_id_other_type.click()
time.sleep(1)
com_id_other_type_chose = chrome_001.find_element_by_xpath(
'//*[@id="proveTypeName"]')
com_id_other_type_chose.click()
time.sleep(1)
# 定义触控，下拉框选择
action_001 = TouchActions(chrome_001)
com_id_other_type_00 = chrome_001.find_element_by_xpath(
'//*[@id="oneLevelContain"]/ul/li[1]')
action_001.scroll_from_element(com_id_other_type_00, 0, 200)
action_001.perform()
time.sleep(2)
# 单位其他证件类型选择OK
com_id_other_type_ok = chrome_001.find_element_by_xpath(
'/html/body/div[9]/div/header/a[2]')
com_id_other_type_ok.click()
com_id_number = data_plate_file_xl.worksheets[0].cell(
data_row + 1, 2).value # excel中行列是从1开始数
com_id_number_input = chrome_001.find_element_by_xpath(
'//*[@id="proveNoNew"]')
com_id_number_input.send_keys(com_id_number)
try:
# 是否法人亲临
#coding=utf-8
import time
from selenium import webdriver
from selenium.webdriver.common.touch_actions import TouchActions
"""设置手机的大小"""
driver = webdriver.Chrome()
driver.get('http://m.test.90dichan.com')
driver.maximize_window()
"""定位操作元素"""
time.sleep(2)
button = driver.find_element_by_xpath(
'//*[@id="pullrefresh"]/div[2]/ul/li[2]/a/div[2]/span')
time.sleep(3)
"""从button元素像下滑动200元素"""
Action = TouchActions(driver)
Action.scroll_from_element(button, 0, -200).perform()
time.sleep(3)
driver.close()
# It's a bit of an annoying process -- in order for the driver to
# click to expand a menu, the clickable item must be on the screen.
# I figured out the best way to do this is to expand the list of states
# starting at the bottom and working up.
us_elem = driver.find_elements_by_css_selector('div.' + class_name + '.row')[0]
elements = driver.find_elements_by_css_selector('div.' + class_name + '.stat.row')
elements = [us_elem] + elements
driver.execute_script("arguments[0].scrollIntoView(true);", elements[1])
elements[1].click()
len(elements)
# Iterate through the elements (states) and click to expand each
for i in range(len(elements)-3,-1,-1):
driver.execute_script("arguments[0].scrollIntoView(true);", elements[i]);
touchAction = TouchActions(driver)
touchAction.scroll_from_element(on_element=elements[i],xoffset=0,yoffset=-1)
print(i+2)
elements[i+2].click()
touchAction = TouchActions(driver)
# Pull elements which contain the state and county data
state_elements = driver.find_elements_by_class_name(class_name + '.stat.row.expand')
county_elements = driver.find_elements_by_class_name(class_name + '.counties')
# Define dictionary where
# keys are state names
# items are dataframes which have state + county data
master_dict = {}
# Iterate through each state - county group
for state,counties in zip(state_elements,county_elements):
"[class=mint-loadmore-bottom]")
# 去除广告阻挡
# 方案一 点击关闭,在设置不加载图片后 失效
# driver.find_element_by_css_selector("[class='unit-0 X flex-middle']").click()
# 方案二 隐藏元素
js = "document.querySelector(\"[class='popInfo']\").style.display='none';"
driver.execute_script(js)
# 疯狂上拉
Action = TouchActions(driver)
"""从button元素像下滑动200元素，以50的速度向下滑动"""
for i in range(6):
time.sleep(5)
# WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.CSS_SELECTOR,)))
Action.scroll_from_element(bottoms[3], 0, 200).perform()
# time.sleep(5)
sections, sectionsText = refreshSections(sections, sectionsText, 1)
listLen = len(sections)
#tempFlag = 21
for sec in range(listLen):
# if sec = tempFlag:
# continue
# 点击进入二层节目列表
try:
sections[sec].click()
except StaleElementReferenceException:
