def checkGetProxyFunc(func):
"""
检查指定的getFreeProxy某个function运行情况
Args:
func: getFreeProxy中某个可调用方法
Returns:
None
"""
func_name = getattr(func, '__name__', "None")
log.info("start running func: {}".format(func_name))
count = 0
for proxy in func():
if verifyProxyFormat(proxy):
log.info("{} fetch proxy: {}".format(func_name, proxy))
count += 1
log.info("{n} completed, fetch proxy number: {c}".format(n=func_name,
c=count))
def checkAllGetProxyFunc():
"""
检查getFreeProxy所有代理获取函数运行情况
Returns:
None
"""
import inspect
member_list = inspect.getmembers(GetFreeProxy,
predicate=inspect.isfunction)
proxy_count_dict = dict()
for func_name, func in member_list:
log.info(u"开始运行 {}".format(func_name))
try:
proxy_list = [_ for _ in func() if verifyProxyFormat(_)]
proxy_count_dict[func_name] = len(proxy_list)
except Exception as e:
log.info(u"代理获取函数 {} 运行出错!".format(func_name))
log.error(str(e))
log.info(u"所有函数运行完毕 " + "***" * 5)
for func_name, func in member_list:
log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name,
c=proxy_count_dict.get(
func_name, 0)))
def refresh(self):
self.db.changeTable(self.raw_proxy_queue)
for proxyGetter in options.ProxyGetter_Run:
# fetch
try:
self.log.info(
"{func}: fetch proxy start".format(func=proxyGetter))
for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
# 直接存储代理, 不用在代码中排重, hash 结构本身具有排重功能
proxy = proxy.strip()
if proxy and verifyProxyFormat(proxy):
self.log.info('{func}: fetch proxy {proxy}'.format(
func=proxyGetter, proxy=proxy))
self.db.put(proxy)
else:
self.log.error(
'{func}: fetch proxy {proxy} error'.format(
func=proxyGetter, proxy=proxy))
except Exception as e:
self.log.error(
"{func}: fetch proxy fail".format(func=proxyGetter))
continue