def getter_scheduler(self, cycle=GETTER_CYCLE):
""" IP获取器 """
getter = Getter()
while True:
print('ip数量不足，获取器开始运行')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
""" Get proxy each cycle."""
getter = Getter()
while True:
print('Getting proxies.')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=setting.GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
logger.info('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
Get proxy regularly
:param cycle: GETTER_CYCLE
"""
getter = Getter()
while True:
print('Start getting proxies')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=1):
"""
定时获取代理
"""
cycle = self.data['GETTER_CYCLE']
getter = Getter(self.data)
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def scheduler_getter(self, cycle=settings.GETTER_CYCLE):
"""
定时获取代理
:param cycle: 获取周期
:return: None
"""
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
:param cycle:
:return:
"""
getter = Getter()
while True:
print("开始抓取代理")
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
'''
定时获取代理
:param cycle:
:return:
'''
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
cycle：从网页获取代理的时间间隔
"""
getter = Getter()
while True:
print('*'*50)
print('开始抓取代理...')
print('*'*50)
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
print('开始抓取代理')
try:
getter.run()
time.sleep(cycle)
except Exception as e:
continue
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
# if RedisClient().db.exists(REDIS_KEY):
# print('清空之前剩余代理')
# RedisClient().db.delete(REDIS_KEY)
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedul_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理数据
:param cycle:
:return:
"""
getter = Getter()
while True:
print('定时获取代理 [开启]')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
try:
getter = Getter()
while True:
print('开始抓取代理')
self.log().info('--开始抓取代理--')
getter.run()
time.sleep(cycle)
except Exception as e:
self.log().exception(e)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""[summary]
定时获取代理
Keyword Arguments:
cycle {[type]} -- [description] (default: {GETTER_CYCLE})
"""
getter = Getter()
# print(getter.redis.count(getter.redis))
while True:
print('开始抓取代理:')
getter.run()
time.sleep(cycle)
def run_getter(self, cycle=CYCLE_GETTER):
"""
运行 Getter
"""
if not ENABLE_GETTER:
logger.info('Getter 未启动')
return
getter = Getter()
loop = 0
while True:
logger.debug(f'getter loop {loop} start...')
getter.run()
loop += 1
time.sleep(cycle)
class TestRedisClient(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestRedisClient, self).__init__(*args, **kwargs)
self.getter = Getter(proxy_key=TEST_REDIS_KEY)
self.tester = Tester(proxy_key=TEST_REDIS_KEY)
def setUp(self):
self.getter.redis.clear()
print('')
def tearDown(self):
self.getter.redis.clear()
def test_run(self):
self.getter.run()
logger.info('本次测试获取 {} 个代理'.format(self.getter.redis.get_count()))
self.tester.run()
def __init__(self, *args, **kwargs):
super(TestRedisClient, self).__init__(*args, **kwargs)
self.getter = Getter(proxy_key=TEST_REDIS_KEY)
self.tester = Tester(proxy_key=TEST_REDIS_KEY)
def schedule_getter(self,cycle=GETTER_CYCLE):
getter = Getter()
while True:
print('开启代理抓取')
getter.run()
time.sleep(cycle)
def scheduler_getter(self, cycle=GETTER_CYCLE):
getter = Getter()
while True:
getter.run()
time.sleep(cycle)
def schedule_getter(self):
getter = Getter()
while True:
print('获取器开始运行')
getter.run()
time.sleep(GETTER_CYCLE)
def schedule_getter(self, cycle=GETTER_CYCLE):
getter = Getter()
while True:
print("start to get proxy")
getter.run()
time.sleep(cycle)
def getter(self, cycle=GETTER_CYCLE):
getter = Getter()
while True:
getter.run()
print("获取器暂停")
time.sleep(cycle)
def test():
getter = Getter()
getter.run()
def schedule_getter(cycle=GETTER_CYCLE):
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
# -*- coding: utf-8 -*- from proxypool.getter import Getter getter = Getter() getter.run()
