def getter_scheduler(self, cycle=GETTER_CYCLE): """ IP获取器 """ getter = Getter() while True: print('ip数量不足,获取器开始运行') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ Get proxy each cycle.""" getter = Getter() while True: print('Getting proxies.') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=setting.GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while True: getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while True: print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while True: logger.info('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=1): """ 定时获取代理 """ cycle = self.data['GETTER_CYCLE'] getter = Getter(self.data) while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ Get proxy regularly :param cycle: GETTER_CYCLE """ getter = Getter() while True: print('Start getting proxies') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 :param cycle: :return: """ getter = Getter() while True: print("开始抓取代理") getter.run() time.sleep(cycle)
def scheduler_getter(self, cycle=settings.GETTER_CYCLE): """ 定时获取代理 :param cycle: 获取周期 :return: None """ getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): ''' 定时获取代理 :param cycle: :return: ''' getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ cycle:从网页获取代理的时间间隔 """ getter = Getter() while True: print('*'*50) print('开始抓取代理...') print('*'*50) getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while True: print('开始抓取代理') try: getter.run() time.sleep(cycle) except Exception as e: continue
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() # if RedisClient().db.exists(REDIS_KEY): # print('清空之前剩余代理') # RedisClient().db.delete(REDIS_KEY) while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedul_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理数据 :param cycle: :return: """ getter = Getter() while True: print('定时获取代理 [开启]') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ try: getter = Getter() while True: print('开始抓取代理') self.log().info('--开始抓取代理--') getter.run() time.sleep(cycle) except Exception as e: self.log().exception(e)
def schedule_getter(self, cycle=GETTER_CYCLE): """[summary] 定时获取代理 Keyword Arguments: cycle {[type]} -- [description] (default: {GETTER_CYCLE}) """ getter = Getter() # print(getter.redis.count(getter.redis)) while True: print('开始抓取代理:') getter.run() time.sleep(cycle)
def run_getter(self, cycle=CYCLE_GETTER): """ 运行 Getter """ if not ENABLE_GETTER: logger.info('Getter 未启动') return getter = Getter() loop = 0 while True: logger.debug(f'getter loop {loop} start...') getter.run() loop += 1 time.sleep(cycle)
class TestRedisClient(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestRedisClient, self).__init__(*args, **kwargs) self.getter = Getter(proxy_key=TEST_REDIS_KEY) self.tester = Tester(proxy_key=TEST_REDIS_KEY) def setUp(self): self.getter.redis.clear() print('') def tearDown(self): self.getter.redis.clear() def test_run(self): self.getter.run() logger.info('本次测试获取 {} 个代理'.format(self.getter.redis.get_count())) self.tester.run()
def schedule_getter(self,cycle=GETTER_CYCLE): getter = Getter() while True: print('开启代理抓取') getter.run() time.sleep(cycle)
def scheduler_getter(self, cycle=GETTER_CYCLE): getter = Getter() while True: getter.run() time.sleep(cycle)
def schedule_getter(self): getter = Getter() while True: print('获取器开始运行') getter.run() time.sleep(GETTER_CYCLE)
# -*- coding: utf-8 -*- from proxypool.getter import Getter getter = Getter() getter.run()
def test(): getter = Getter() getter.run()
def schedule_getter(self, cycle=GETTER_CYCLE): getter = Getter() while True: print("start to get proxy") getter.run() time.sleep(cycle)
def getter(self, cycle=GETTER_CYCLE): getter = Getter() while True: getter.run() print("获取器暂停") time.sleep(cycle)
def schedule_getter(cycle=GETTER_CYCLE): getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)