def l1lll1l1111_wcplus_():
settings = l1l1111ll_wcplus_.get()
print(settings)
if 'proxy' in settings:
l1lll1l111l_wcplus_ = settings['proxy']
else:
logger.logger('请先设置代理IP')
popUps('没有设置代理IP请先设置 设置之后请验证 确保返回的只有一个代理IP 例如 123.234.345.12:9808',
'代理设置', 'error')
return '127.0.0.1:1080'
try:
l1ll11ll11_wcplus_ = requests.get(l1lll1l111l_wcplus_).text
if '白名单' in l1ll11ll11_wcplus_:
popUps('即将使用真实IP' + l1ll11ll11_wcplus_, '获取代理IP出错', 'error')
time.sleep(1)
return '127.0.0.1:1080'
except:
popUps('请设置正确的代理IP 设置后请验证保证可返回一个代理IP 如果真实IP可用将使用真实IP进行采集', '代理设置',
'error')
return '127.0.0.1:1080'
else:
while '请求' in l1ll11ll11_wcplus_:
time.sleep(1)
print('等待返回代理IP...')
l1ll11ll11_wcplus_ = requests.get(l1lll1l111l_wcplus_).text
return l1ll11ll11_wcplus_
def l1l11lll11_wcplus_():
"""
:return: 是否直接使用代理
"""
from instance import l1l1111ll_wcplus_
settings = l1l1111ll_wcplus_.get()
if 'use_proxy' in settings:
if settings['use_proxy'] == 'true':
return False
return True
else:
return False
def l1ll1l111_wcplus_(self, process):
"""
:return: 根据请求参数发起一次文章内容采集
"""
from instance import l1l1111ll_wcplus_
l1l11l11l_wcplus_ = l1l1111ll_wcplus_.get()['use_proxy']
if l1l11l11l_wcplus_ == 'false':
l1ll11lll_wcplus_ = 64
else:
l1ll11lll_wcplus_ = 8
l1lll1111_wcplus_(l1ll11lll_wcplus_)
from app.l1ll11ll1_wcplus_.article import l1l1ll1ll_wcplus_
l1l1ll1ll_wcplus_(l1ll11lll_wcplus_=l1ll11lll_wcplus_, process=process)
1. 数据库有文章的永久链接
2. 代理IP地址有效
"""
import threading, os, codecs
from lxml.etree import tostring
from queue import Queue
import requests
from requests.exceptions import SSLError, Timeout, ProxyError, ConnectionError
import queue, time
from cmp.proxy import l1ll1l1111_wcplus_
from copy import copy
from cmp.db.mongodb import DB
from utils.base import logger
from utils.string_handle import encryptString
from instance import l1l1111ll_wcplus_
if 'save_html' in l1l1111ll_wcplus_.get():
l1ll11l1ll_wcplus_ = l1l1111ll_wcplus_.get()['save_html']
else:
l1ll11l1ll_wcplus_ = 'false'
l1l1l1111l_wcplus_ = []
nickname = None
l1ll1l11ll_wcplus_ = None
l11ll111l_wcplus_ = None
class l1ll111lll_wcplus_(threading.Thread):
"""
继承threading.Thread 定义任务处理类
1. 从任务队列中取出一个任务
2. 从代理IP队列中取出一个IP 发起请求 如果请求成功将代理IP放回队列 否者重新申请一个代理IP再次发起请求直到成功
"""
def get(self):
return l1l1111ll_wcplus_.get()