def monitor(website_name, merchant_name, batch_num):
"""
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=chromedriver_path)
"""
try:
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://www.chinaft.com.cn/news/search/_1.shtml?key=" + urllib.parse.quote(
website_name)
driver.get(url)
source = driver.page_source
senti_util.snapshot_home("交易中国", merchant_name, url, batch_num,
driver)
soup = BeautifulSoup(source, 'html.parser')
news = soup.find_all("div",
attrs={'class': 'xixi_ChinaFT_left_news_box'})
if news.__len__() > 0:
for new in news:
if not gl.check_by_batch_num(batch_num):
break
href = new.find_all('a')[1].get("href")
content = new.find_all('a')[1].get_text()
if content.find(website_name) != -1:
senti_util.senti_process_text(
"交易中国", merchant_name, content,
"http://www.chinaft.com.cn" + href, batch_num)
else:
logger.info("交易中国没有搜索到数据: %s", merchant_name)
except Exception as e:
logger.error(e)
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "https://tousu.sina.com.cn/index/search/?keywords=" + urllib.parse.quote(keyword) + "&t=0"
if driver is None:
senti_util.log_error("黑猫投诉", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
source = driver.page_source
senti_util.snapshot_home("黑猫投诉", url, batch_num, website, driver)
soup = BeautifulSoup(source, 'html.parser')
items = soup.find_all(attrs={'class': 'blackcat-con'})
if items.__len__() > 0:
for item in items:
href = item.find_all('a')[0].get("href")
content = item.find_all('h1')[0].get_text()
if content.find(keyword) != -1:
senti_util.senti_process_text("黑猫投诉", content, href,
batch_num, website)
else:
logger.info("黑猫投诉没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://paynews.net/search.php?mod=forum"
if driver is None:
senti_util.log_error("支付产业网", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
search_text_blank = driver.find_element_by_id("scform_srchtxt")
search_text_blank.send_keys(keyword)
search_text_blank.send_keys(Keys.RETURN)
senti_util.snapshot_home("支付产业网", url, batch_num, website, driver)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
div_list = soup.find(attrs={'class': 'slst mtw'})
if div_list is not None and div_list.__len__() > 0:
news = div_list.find_all('li')
for new in news:
href = new.find_all('a')[0].get("href")
content = new.find_all('a')[0].get_text()
if content.find(keyword) != -1:
senti_util.senti_process_text(
"支付产业网", content, "http://paynews.net/" + href,
batch_num, website)
else:
logger.info("支付产业网没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = 'https://baike.baidu.com/item/%s' % urllib.parse.quote(keyword)
if driver is None:
senti_util.log_error("百度百科", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
check_exist = soup.find_all(
name='p', attrs={'class': re.compile('sorryCont')})
if check_exist.__len__() == 0:
description = soup.find(
attrs={"name": "description"})['content']
senti_util.senti_process_text("百度百科", description, url,
batch_num, website)
else:
senti_util.snapshot_home("百度百科", url, batch_num, website,
driver)
logger.info("百度百科没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, website_name, batch_num, merchant_name, merchant_num):
"""
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=chromedriver_path)
"""
driver = WebDriver.get_chrome()
try:
senti_util = SentiUtil()
url = "http://tieba.baidu.com/f?fr=wwwt&kw=" + urllib.parse.quote(
keyword)
driver.get(url)
senti_util.snapshot_home("百度贴吧", website_name, url, batch_num,
merchant_name, merchant_num, driver)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
news = soup.find_all(
"div", attrs={'class': 'threadlist_title pull_left j_th_tit '})
if news.__len__() > 0:
for new in news:
href = new.find_all('a')[0].get("href")
content = new.find_all('a')[0].get_text()
if content.find(keyword) != -1:
senti_util.senti_process_text(
"百度贴吧", website_name, content,
"http://tieba.baidu.com" + href, batch_num,
merchant_name, merchant_num)
else:
logger.info("百度贴吧没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def get_access_res(url):
driver = WebDriver.get_chrome_for_access()
try:
if str(url).startswith("http"):
http_url = str(url)
else:
http_url = "http://" + str(url)
logger.info("http_url: %s", http_url)
driver.get(http_url)
title = driver.title
source = driver.page_source
if title.__contains__('404') or source.__contains__(
'ERR_NAME_NOT_RESOLVED') or source.__contains__(
'ERR_CONNECTION_REFUSED') or source.__contains__(
'ERR_CONNECTION_TIMED_OUT') or source.__contains__(
'ERR_NAME_NOT_RESOLVED') or source.__contains__(
'ERR_NAME_RESOLUTION_FAILED') or source.__contains__(
'DNS_PROBE_FINISHED_NXDOMAIN') or source.__contains__(
'ERR_EMPTY_RESPONSE') or source.__contains__(
'主机开设成功') or source.__contains__(
'非法阻断') or source.__contains__(
'Bad Request') or source.__contains__(
'404 page not found') or source.__contains__('https://wanwang.aliyun.com/domain/parking'):
return None, http_url
else:
return http_url, driver.current_url
except Exception as e:
logger.error(e)
return None, None
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://ts.21cn.com/home/search?keyword=" + urllib.parse.quote(
keyword)
if driver is None:
senti_util.log_error("聚投诉", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
driver.implicitly_wait(3)
source = driver.page_source
senti_util.snapshot_home("聚投诉", url, batch_num, website, driver)
soup = BeautifulSoup(source, 'html.parser')
items = soup.find_all(attrs={'class': 'complain-item'})
if items.__len__() > 0:
for item in items:
href = item.find_all('a')[1].get("href")
content = item.find_all('a')[1].get_text()
if content.find(keyword) != -1:
senti_util.senti_process_text(
"聚投诉", content,
"http://www.paycircle.cn" + href[1:], batch_num,
website)
else:
logger.info("聚投诉没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://wenshu.court.gov.cn/"
if driver is None:
senti_util.log_error("黑猫投诉", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
search_text_blank = driver.find_element_by_xpath(
"//*[@class='searchKey search-inp']")
search_text_blank.send_keys(keyword)
search_text_blank.send_keys(Keys.RETURN)
time.sleep(3)
source = driver.page_source
senti_util.snapshot_home("裁判文书网", url, batch_num, website, driver)
soup = BeautifulSoup(source, 'html.parser')
for a_tag in soup.find_all('a', class_='caseName'):
href = a_tag.get("href")
title = a_tag.get_text()
if title.find(keyword) != -1:
senti_util.senti_process_text(
"裁判文书网", title,
"http://wenshu.court.gov.cn/website/wenshu" + href[2:],
batch_num, website)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "https://www.baidu.com/"
if driver is None:
senti_util.log_error("百度搜索", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
search_text_blank = driver.find_element_by_id("kw")
search_text_blank.send_keys(keyword)
search_text_blank.send_keys(Keys.RETURN)
time.sleep(5)
# driver.find_element_by_xpath('//input[@name="wd"]').send_keys(website_name)
senti_util.snapshot_home("百度搜索", url, batch_num, website, driver)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
for result_table in soup.find_all('h3', class_='t'):
a_click = result_table.find("a")
title = a_click.get_text()
if title.find(keyword) != -1:
senti_util.senti_process_text("百度搜索", title, str(a_click.get("href")),
batch_num, website)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://www.paycircle.cn/company/search.php?kw=" + urllib.parse.quote(
keyword) + "&c=SearchList&"
if driver is None:
senti_util.log_error("支付圈", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
source = driver.page_source
senti_util.snapshot_home("支付圈", url, batch_num, website, driver)
soup = BeautifulSoup(source, 'html.parser')
div_list = soup.find_all(attrs={'class': 'list'})
if div_list.__len__() > 0:
news = div_list[0].find_all('tr')
for new in news:
href = new.find_all('td')[2].find_all('a')[0].get("href")
content = new.find_all('td')[2].find_all(
'li')[1].get_text()
if content.find(keyword) != -1:
senti_util.senti_process_text("支付圈", content, href,
batch_num, website)
else:
logger.info("支付圈没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(website_name, merchant_name, batch_num):
try:
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "https://www.p2peye.com/search.php?mod=zonghe&srchtxt=" + urllib.parse.quote(website_name)
driver.get(url)
source = driver.page_source
senti_util.snapshot_home("网贷天眼", merchant_name, url,
batch_num, driver)
soup = BeautifulSoup(source, 'html.parser')
news = soup.find_all(attrs={'class': 'result-t'})
if news.__len__() > 0:
for new in news:
href = new.find_all('a')[0].get("href")
content = new.get_text()
if content.find(website_name) != -1:
senti_util.senti_process_text("网贷天眼", merchant_name,content, "http://" + href[2:],
batch_num)
else:
logger.info("网贷天眼没有搜索到数据: %s", merchant_name)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def senti_process_text(platform, text, href, batch_num, website):
driver = WebDriver.get_chrome()
keyword_dao = KeywordDao()
monitor_third_dao = MonitorThirdDao()
# 截图
try:
driver.get(href)
snapshot = SnapshotService.create_snapshot(driver, batch_num,
website, '舆情')
is_normal = "正常"
keywords = keyword_dao.get_all()
for keyword in keywords:
index = text.find(keyword.name)
monitor_third = MonitorThird()
monitor_third.website_name = website.website_name
monitor_third.merchant_num = website.merchant_num
monitor_third.merchant_name = website.merchant_name
monitor_third.domain_name = website.domain_name
monitor_third.saler = website.saler
monitor_third.batch_num = batch_num
monitor_third.url = href
monitor_third.type = platform
if index != -1:
is_normal = "异常"
monitor_third.is_normal = is_normal
monitor_third.level = '高'
monitor_third.outline = '检测到敏感词：' + str(keyword.name)
monitor_third.snapshot = snapshot
monitor_third_dao.add(monitor_third)
else:
pass
if is_normal == "正常":
if platform == "百度百科":
monitor_third.level = '-'
monitor_third.outline = '-'
monitor_third.is_normal = is_normal
monitor_third.snapshot = snapshot
monitor_third_dao.add(monitor_third)
pass
except ConnectionError as conn_error:
logger.error(conn_error)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
class TestMysql(object):
if __name__ == "__main__":
url = "http://www.zfzj.cn/search.php"
driver = webdriver.Remote(
command_executor='http://172.17.161.230:8911/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.maximize_window()
try:
driver.get(url)
search_text_blank = driver.find_element_by_id("scform_srchtxt")
search_text_blank.send_keys('京东')
search_text_blank.send_keys(Keys.RETURN)
time.sleep(5)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
items = soup.find_all(attrs={'class': 'blackcat-con'})
if items.__len__() > 0:
for item in items:
href = item.find_all('a')[0].get("href")
content = item.find_all('h1')[0].get_text()
else:
logger.info("黑猫投诉没有搜索到数据:")
driver.quit()
except Exception as e:
logger.error(e)
driver.quit()
class TestMysql(object):
if __name__ == "__main__":
url = "http://ts.21cn.com/home/search?keyword=%E4%BA%AC%E4%B8%9C"
driver = webdriver.Remote(
command_executor='http://172.17.161.230:8912/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.maximize_window()
try:
driver.get(url)
time.sleep(5)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
items = soup.find_all(attrs={'class': 'blackcat-con'})
if items.__len__() > 0:
for item in items:
href = item.find_all('a')[0].get("href")
content = item.find_all('h1')[0].get_text()
else:
logger.info("黑猫投诉没有搜索到数据:")
driver.quit()
except Exception as e:
logger.error(e)
driver.quit()
def get_chrome():
chrome_options = Options()
# 禁止图片和css加载
prefs = {"profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2}
chrome_options.add_experimental_option("prefs", prefs)
try:
driver = webdriver.Chrome(executable_path='/usr/bin/chromedriver',
desired_capabilities=DesiredCapabilities.CHROME,
options=chrome_options)
driver.set_page_load_timeout(60)
driver.set_script_timeout(60)
driver.maximize_window()
return driver
except Exception as e:
logger.error(e)
return None
class TestMysql(object):
if __name__ == "__main__":
url = "http://www.paycircle.cn/company/search.php?kw=" + urllib.parse.quote(
'京东') + "&c=SearchList&"
driver = webdriver.Remote(
command_executor='http://172.17.161.230:8911/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.maximize_window()
try:
driver.get(url)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
div_list = soup.find_all(attrs={'class': 'list'})
if div_list.__len__() > 0:
news = div_list[0].find_all('tr')
for new in news:
href = new.find_all('td')[2].find_all('a')[0].get("href")
content = new.find_all('td')[2].find_all('li')[1].get_text()
else:
logger.info("支付圈没有搜索到数据")
driver.quit()
except Exception as e:
logger.error(e)
class TestWangdaitianyan(object):
if __name__ == "__main__":
"""
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path="C:/chromedriver_2.38/chromedriver.exe")
driver.get("http://www.wangdaibus.com/search.php?mod=forum")
driver.find_element_by_id("scform_srchtxt").send_keys(u"京东")
driver.find_element_by_id("scform_submit").click()
"""
driver = WebDriver.get_chrome()
try:
driver.get(
"http://www.wangdaibus.com/search/list?subject=%E4%BA%AC%E4%B8%9C"
)
aaa = "京东"
#lement_by_xpath('//input[@name="subject"]').send_keys(aaa)
#driver.find_element_by_xpath('//input[@name="subject"]').send_keys(Keys.ENTER)
time.sleep(10)
except Exception as e: # 异常处理
logger.error(e)
pass
SnapshotService.create_snapshot(driver)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
news = soup.find_all("h3", attrs={'class': 'xs3'})
if news.__len__() > 0:
for new in news:
href = new.find_all('a')[0].get("href")
logger.info("http://www.wangdaibus.com/" + href)
logger.info(new.get_text())
'''
class TestMysql(object):
if __name__ == "__main__":
url = "http://paynews.net/search.php?mod=forum"
driver = webdriver.Remote(
command_executor='http://172.17.161.230:8911/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.maximize_window()
try:
driver.get(url)
search_text_blank = driver.find_element_by_id("scform_srchtxt")
search_text_blank.send_keys('京东')
search_text_blank.send_keys(Keys.RETURN)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
driver.save_screenshot("D:/a.png")
div_list = soup.find(attrs={'class': 'slst mtw'})
if div_list.__len__() > 0:
news = div_list.find_all('li')
for new in news:
href = new.find_all('a')[0].get("href")
content = new.find_all('a')[0].get_text()
print(content)
else:
logger.info("支付产业网没有搜索到数据: %s")
driver.quit()
except Exception as e:
logger.error(e)
driver.quit()
class TestMysql(object):
if __name__ == "__main__":
url = "https://www.trackingmore.com/choose-cn-70634105326416.html?"
# 实例化一个启动参数对象
chrome_options = Options()
# 禁止图片和css加载
prefs = {"profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2}
chrome_options.add_experimental_option("prefs", prefs)
driver = webdriver.Remote(command_executor='http://172.17.161.230:8911/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME,
options=chrome_options)
driver.maximize_window()
driver.set_page_load_timeout(60)
try:
driver.get(url)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
logis_list = soup.find_all(attrs={'class': 'ulliselect'})
start = datetime.datetime.now()
if logis_list is not None and logis_list.__len__() > 0:
for logis in logis_list:
href = logis.get("href")
name = logis.get_text()
print(name)
try:
print(href)
url = "https:" + href
driver.get(url)
driver.save_screenshot("D:/" + name + ".png")
end = datetime.datetime.now()
print(end - start)
except Exception as e:
logger.error(e)
print('time out after 30 seconds when loading page')
driver.execute_script('window.stop()')
print('111')
driver.save_screenshot("D:/" + href + ".png")
print('超时后截图完毕')
end = datetime.datetime.now()
print(end - start)
except Exception as e:
print('3333')
logger.error(e)
print('2222')
driver.quit()
def log_error(platform, href, batch_num, website):
monitor_third_dao = MonitorThirdDao()
# 截图
try:
is_normal = "异常"
monitor_third = MonitorThird()
monitor_third.merchant_num = website.merchant_num
monitor_third.merchant_name = website.merchant_name
monitor_third.website_name = website.website_name
monitor_third.domain_name = website.domain_name
monitor_third.saler = website.saler
monitor_third.batch_num = batch_num
monitor_third.url = href
monitor_third.type = platform
monitor_third.level = '-'
monitor_third.outline = '异常截图失败'
monitor_third.is_normal = is_normal
monitor_third.snapshot = ""
monitor_third_dao.add(monitor_third)
except Exception as e:
logger.error(e)
return
def tracking_execute():
job = os.environ['job']
if job == "tracking":
# 重启selenium
stop_selenium()
stop_chrome()
gl.set_value('STATUS', True)
gl.set_value('TRACKING_STATUS', True)
ims_api.heartbeat()
try:
task_id = request.form['taskId']
status = request.form['status']
logger.info("tracking begin task_id: %s,status: %s" % (str(task_id), str(status)))
t = threading.Thread(target=inspect_tracking, args=(task_id, status))
t.setDaemon(True)
t.start()
return 'OK'
except Exception as e:
logger.error(e)
else:
logger.info("Tracking is not my job!")
return 'OK'
def monitor(website_name, merchant_name, batch_num):
"""
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=chromedriver_path)
"""
try:
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "https://www.wdzj.com/front/search/index?key=" + urllib.parse.quote(
website_name)
driver.get(url)
source = driver.page_source
senti_util.snapshot_home("网贷之家", merchant_name, url, batch_num,
driver)
soup = BeautifulSoup(source, 'html.parser')
tzbox = soup.find_all("ul", attrs={'class': 'so-tzbox'})
if tzbox.__len__() == 0:
return
news = tzbox[0].find_all("li")
if news.__len__() > 0:
for new in news:
if not gl.check_by_batch_num(batch_num):
break
href = new.find_all('a')[0].get("href")
content = new.get_text()
if content.find(website_name) != -1:
senti_util.senti_process_text("网贷之家", merchant_name,
content,
"http://" + href[2:],
batch_num)
else:
logger.info("网贷之家没有搜索到数据: %s", merchant_name)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(website_name, merchant_name, batch_num):
"""
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=chromedriver_path)
"""
try:
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://www.wangdaibus.com/search/list?subject=" + urllib.parse.quote(
website_name)
driver.get(url)
time.sleep(10)
senti_util.snapshot_home("网贷巴士", merchant_name, url, batch_num,
driver)
# driver.find_element_by_xpath('//input[@name="srchtxt"]').send_keys(website_name)
# driver.find_element_by_xpath('//input[@name="srchtxt"]').send_keys(Keys.ENTER)
source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
news = soup.find_all("h3", attrs={'class': 'xs3'})
if news.__len__() > 0:
for new in news:
if not gl.check_by_batch_num(batch_num):
break
href = new.find_all('a')[0].get("href")
content = new.get_text()
if content.find(website_name) != -1:
senti_util.senti_process_text(
"网贷巴士", merchant_name, content,
"http://www.wangdaibus.com/" + href, batch_num)
else:
logger.info("网贷巴士没有搜索到数据: %s", merchant_name)
except Exception as e:
logger.error(e)
return
finally:
driver.quit()
def monitor(keyword, batch_num, website):
driver = WebDriver.get_chrome()
senti_util = SentiUtil()
url = "http://www.zhifujie.com/search/search"
if driver is None:
senti_util.log_error("支付界", url, batch_num, website)
return
else:
pass
try:
driver.get(url)
search_text_blank = driver.find_element_by_id("searchbox")
search_text_blank.send_keys(keyword)
driver.find_element_by_xpath(
'//button[contains(text(), "搜索")]').click()
time.sleep(5)
source = driver.page_source
senti_util.snapshot_home("支付界", url, batch_num, website, driver)
soup = BeautifulSoup(source, 'html.parser')
items = soup.find_all(attrs={'class': 'main-news-content-item'})
if items.__len__() > 0:
for item in items:
href = item.find_all('a')[1].get("href")
content = item.find_all('a')[1].get_text()
if content.find(keyword) != -1:
senti_util.senti_process_text(
"支付界", content,
"http://www.paycircle.cn" + href[1:], batch_num,
website)
else:
logger.info("支付界没有搜索到数据: %s", keyword)
except Exception as e:
logger.error(e)
senti_util.snapshot_home("支付界", url, batch_num, website, driver)
return
finally:
driver.quit()
def snapshot_home(platform, href, batch_num, website, driver):
monitor_third_dao = MonitorThirdDao()
# 截图
try:
snapshot = SnapshotService.create_snapshot(driver, batch_num,
website, '舆情')
is_normal = "正常"
monitor_third = MonitorThird()
monitor_third.merchant_num = website.merchant_num
monitor_third.merchant_name = website.merchant_name
monitor_third.website_name = website.website_name
monitor_third.domain_name = website.domain_name
monitor_third.saler = website.saler
monitor_third.batch_num = batch_num
monitor_third.url = href
monitor_third.type = platform
monitor_third.level = '-'
monitor_third.outline = '首页截图'
monitor_third.is_normal = is_normal
monitor_third.snapshot = snapshot
monitor_third_dao.add(monitor_third)
except Exception as e:
logger.error(e)
return
def execute():
logger.info("receive execute req !")
job = os.environ['job']
if job == "bc" or job == "other":
# 重启selenium
logger.info("restart selenium...")
stop_selenium()
start_selenium()
logger.info("update status...")
gl.set_value('STATUS', True)
logger.info("heartbeat...")
ims_api.heartbeat()
try:
batch_num = request.form['batchNum']
logger.info("spider begin batchNum: %s" % str(batch_num))
t = threading.Thread(target=inspect, args=(batch_num,))
t.setDaemon(True)
t.start()
return 'OK'
except Exception as e:
logger.error(e)
else:
logger.info("spider is not my job!")
return 'OK'
class TestMysql(object):
if __name__ == "__main__":
driver = webdriver.Remote(command_executor='http://172.17.161.230:8911/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.maximize_window()
try:
driver.get("http://en.ly.com")
driver.save_screenshot("D:/bb.jpg")
print(driver.current_url)
driver.quit()
except Exception as e:
logger.error(e)
def get_proxy_access_res(url):
if str(url).startswith("http"):
http_url = str(url)
else:
http_url = "http://" + str(url)
driver = WebDriver.get_proxy_chrome()
if driver is None:
return None, None
else:
try:
logger.info("http_url: %s", http_url)
driver.get(http_url)
title = driver.title
if title.__contains__('404') or driver.page_source.__contains__(
'ERR_NAME_NOT_RESOLVED') or driver.page_source.__contains__(
'ERR_CONNECTION_REFUSED') or driver.page_source.__contains__('ERR_CONNECTION_TIMED_OUT'):
return None, http_url
else:
return http_url, driver.current_url
except Exception as e:
logger.error(e)
return None, None
finally:
driver.quit()
class TestMysql(object):
if __name__ == "__main__":
url = "https://www.tianyancha.com/search?key=%E4%BA%AC%E4%B8%9C"
driver = webdriver.Remote(
command_executor='http://172.17.161.230:8912/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME)
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.maximize_window()
try:
driver.get(url)
time.sleep(5)
driver.save_screenshot("D:/bb.jpg")
driver.get("https://www.tianyancha.com/company/12562796")
time.sleep(5)
driver.save_screenshot("D:/cc.jpg")
driver.quit()
except Exception as e:
logger.error(e)
def tran2text(url):
# TODO
logger.error("TODO %s" % url)
return None
try:
if str(url).endswith(".jpg") or str(url).endswith(".png") or str(url).endswith(".bmp") or str(url).endswith(
".jpeg"):
logger.info("src path: %s" % url)
text = pytesseract.image_to_string(Image.open(url), lang='chi_sim')
logger.info("text: %s", str(text))
if text == "":
return None
return text
else:
logger.error("url is not pic! url:%s" % url)
return None
except Exception as e:
logger.error(e)
return None
