class DoFetchProxy(ProxyManager):
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('fetch_proxy')
def main(self):
self.log.info("***** start fetch proxy *****")
self.fetch()
self.log.info("***** finish fetch proxy *****")
def testLogHandler():
"""
test function LogHandler in Util/LogHandler
:return:
"""
log = LogHandler('test')
log.error('this is a log from test')
log.resetName(name='test1')
log.warning('this is a log from test1')
log.resetName(name='test2')
log.info('this is a log from test2')
class Check(ProxyManager, Thread):
def __init__(self, queue, thread_name):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.queue = queue
self.log = LogHandler('init_proxy_check')
def run(self):
self.log.info('Init Proxy Check - {} : start'.format(self.name))
while True:
try:
proxy_key = self.queue.get(block=False)
except Empty:
self.log.info('Init Proxy Check - {} : end'.format(self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_key)
proxy_obj, status = check_proxy_useful(proxy_obj)
if status:
self.log.info(
'Init Proxy Check - {}: {} validation pass'.format(
self.name, proxy_obj.proxy))
self.client.put(proxy_obj)
else:
self.log.info(
'Init Proxy Check - {}: {} validation fail'.format(
self.name, proxy_obj.proxy))
self.client.delete(proxy_obj.proxy)
self.queue.task_done()
class ProxyValidater:
"""
验证useful_proxy_queue中的代理,将不可用的移出
"""
def __init__(self):
self._pm = ProxyManager()
self.queue = Queue()
self.proxy_list = None
self.proxy_dict = dict()
self.log = LogHandler('proxy_validater')
def _valid_proxy(self, threads=50):
"""
验证useful_proxy代理
:param threads: 线程数
:return:
"""
thread_list = list()
for index in range(threads):
thread_list.append(ValidateProxy(self.queue, self.proxy_dict))
for thread in thread_list:
thread.daemon = True
thread.start()
for thread in thread_list:
thread.join()
def put_queue(self):
self._pm.db.change_table(self._pm.useful_proxy_queue)
self.proxy_list = self._pm.db.get_all()
for proxy in self.proxy_list:
self.queue.put(proxy)
self.proxy_dict[proxy] = 0
def main(self):
self.put_queue()
while True:
if not self.queue.empty():
self.log.info("Start valid useful proxy")
self._valid_proxy()
else:
self.log.info('Valid Complete! sleep 600 sec.')
time.sleep(600)
self.put_queue()
class ProxyManager(object):
def __init__(self):
self.client = db.DBclient()
self.log = LogHandler('proxy_manager')
def fetch(self):
proxy_set = set()
self.log.info(u'代理抓取: start')
get_function = GetFunctions()
for proxy_get in get_function.proxy_get_functions:
self.log.info('Get Proxy - {}: start'.format(proxy_get))
try:
for proxy in getattr(GetFreeProxy, proxy_get.strip())():
proxy = proxy.strip()
if not proxy or not verifyProxyFormat(proxy):
self.log.error('Get Proxy - {}: {} error'.format(
proxy_get, proxy))
continue
elif proxy in proxy_set:
self.log.info('Get Proxy - {}: {} is exist'.format(
proxy_get, proxy))
continue
else:
self.log.info('Get Proxy - {}: {} success'.format(
proxy_get, proxy))
self.client.put(Proxy(proxy, source=proxy_get))
proxy_set.add(proxy)
except Exception as e:
self.log.error('Get Proxy - {}: error'.format(proxy_get))
self.log.error(str(e))
def get(self):
proxy_list = self.client.getAll()
if proxy_list:
proxy = random.choice(proxy_list)
return Proxy.newProxyFromJson(proxy)
else:
return None
def getAll(self):
proxy_list = self.client.getAll()
return [Proxy.newProxyFromJson(_) for _ in proxy_list]
def getCount(self):
proxy_counts = self.client.getCount()
return proxy_counts
def delete(self, proxy_key):
self.client.delete(proxy_key)
class ValidateProxy(Thread):
"""
多线程验证useful_proxy
"""
def __init__(self, queue, item_dict):
self._pm = ProxyManager()
super().__init__()
self.log = LogHandler('validate_proxy', file=False) # 多线程同时写一个日志文件会有问题
self.queue = queue
self.item_dict = item_dict
def run(self):
self._pm.db.change_table(self._pm.useful_proxy_queue)
while self.queue.qsize():
proxy = self.queue.get()
if valid_useful_proxy(proxy):
# 验证通过，从计数字典删除
self.log.info('ProxyCheck: {} validation pass'.format(proxy))
del self.item_dict[proxy]
else:
# 验证失败，计数加1
self.item_dict[proxy] += 1
self.log.info('ProxyCheck: {} validation fail'.format(proxy))
if self.item_dict[proxy] >= FAIL_COUNT:
# 超过最大失败次数，从计数字典和数据库删除
self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy))
del self.item_dict[proxy]
self._pm.db.delete(proxy)
else:
# 未超过最大失败次数，放回队列
self.queue.put(proxy)
self.queue.task_done()
# print(data)
# print(proxy_ip)
CheckUidHtml(data, uid)
log.warning("访问页面失败*2,我也没办法了", e)
# 自动获取需要爬取用户id
def GetUid(data):
try:
r = api.get_uid(data)
return (json.loads(r)["data"])
except Exception as e:
log.error("api request fail: %s", format(e))
while True:
# 定义需要爬取的文件个数及爬虫名字
data = {'need': req_nums, 'remark': app}
uid = GetUid(data)
log.info(uid)
s_id = uid[0]
e_id = uid[1]
o_id = 0
for i in range(s_id, e_id):
CheckUidPage(i)
from util.log_handler import LogHandler
from util.config import GetConfig
configs = GetConfig()
host = 'ws://' + str(configs.host_ip) + ':' + str(configs.host_port) + "/cable"
try:
import thread
except ImportError:
import _thread as thread
import time
logger = LogHandler('web_socket')
logger.info('this is a log from web_socket')
def on_message(ws, message):
data = json.loads(message)
print(data['type'])
if data['type'] == 'ping':
print(data['type'])
else:
logger.info(data)
def on_error(ws, error):
class ProxyRefresher:
"""
代理定时刷新
"""
def __init__(self):
self._pm = ProxyManager()
self.log = LogHandler('proxy_refresher')
def fetch_all_proxy(self):
"""
fetch proxy into Db by ProxyGetter/get_free_proxy.py
:return:
"""
for proxyGetter in config.proxy_getter_functions:
# fetch
try:
self.log.info(
"{func}: fetch proxy start".format(func=proxyGetter))
# for proxy in getattr(GetFreeProxy, proxyGetter.strip())(self.get()):
for proxy in getattr(GetFreeProxy, proxyGetter.strip())(None):
# 直接存储代理, 不用在代码中排重, hash 结构本身具有排重功能
proxy = proxy.strip()
if proxy and verify_proxy_format(proxy):
self.log.info('{func}: fetch proxy {proxy}'.format(
func=proxyGetter, proxy=proxy))
self._pm.db.change_table(self._pm.raw_proxy_queue)
self._pm.db.put(proxy)
else:
self.log.error(
'{func}: fetch proxy {proxy} error'.format(
func=proxyGetter, proxy=proxy))
pass
except Exception as e:
self.log.error("{func}: fetch proxy fail, {e}".format(
func=proxyGetter, e=e))
continue
def validate_raw_proxy(self):
"""
验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue
:return:
"""
self._pm.db.change_table(self._pm.raw_proxy_queue)
raw_proxy = self._pm.db.pop()
self.log.info('ProxyRefresher: %s start validProxy' % time.ctime())
# 计算剩余代理，用来减少重复计算
remaining_proxies = self._pm.get_all()
while raw_proxy:
if (raw_proxy not in remaining_proxies
) and valid_useful_proxy(raw_proxy):
self._pm.db.change_table(self._pm.useful_proxy_queue)
self._pm.db.put(raw_proxy)
self.log.info('ProxyRefresher: %s validation pass' % raw_proxy)
else:
self.log.info('ProxyRefresher: %s validation fail' % raw_proxy)
self._pm.db.change_table(self._pm.raw_proxy_queue)
raw_proxy = self._pm.db.pop()
remaining_proxies = self._pm.get_all()
self.log.info('ProxyRefresher: %s validProxy complete' % time.ctime())