def run_getter(self, cycle=GETTER_CYCLE): """定时获取cookie""" getter = Getter() while True: print('开始抓取cookies') getter.run() time.sleep(cycle)
def schdule_getter(self, cycle=GETTER_CYCLE): """定时获取代理""" getter = Getter() while 1: print('start crawl proxy...') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """定时获取代理""" getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def scheduler_getter(self, cycle=GETTER_CYCLE): # 调度获取模块 '''定时获取代理''' getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=settings.GETTER_CYCLE): ''' 定时抓取 ''' getter = Getter() while True: getter.run() time.sleep(cycle)
def schedule_getter(self): getter = Getter() while GETTER_ENABLED: try: getter.run() except: getter.run() time.sleep(GETTER_CYLE)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while GETTER_ENABLED: print('抓取器开始运行') getter.run() time.sleep(cycle)
def scheduler_getter(self): """ 每隔5天获取一次cookies :return: """ while True: getter = Getter() getter.run() time.sleep(self.getter_cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 while循环运行获取器模块,运行一次之后间隔一段时间,间隔时间参数为GETTER_CYCLE配置。 """ getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ Get the agent regularly """ getter = Getter() while True: print('Start to grab the proxy') logger.log('INFOR', 'Start to Grab the proxy...') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): #定时获取代理 getter = Getter() count = 0 while True: count += 1 print('获取器开始运行', count) getter.run() time.sleep(cycle)
def getter_scheduler(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ print('获取器开始执行!') getter = Getter() while True: getter.run() print('休息', GETTER_CYCLE, '秒') time.sleep(GETTER_CYCLE)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 """ getter = Getter() while True: print('Start crawling proxy') getter.run() time.sleep(cycle)
def run(self, q): ## handles the publishing of data, should be run at an acceptable update interval # get data from queue data = q[self.name] # TODO: format data for publisher #print("nearest obs data ", data) # publish data Getter.run(self, data)
def process_getter(self): '''获取器执行子进程''' getter = Getter() redis = db() while True: if redis.count_proxie() < MAXPOOL: getter.run() time.sleep(GETTER_CYCLE) else: print("代理池已经达到最大容量") break
def schedule_getter(self, cycle=GETTER_CYCLE): ''' 定时获取代理 :param cycle: :return: ''' getter = Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): """ 定时获取代理 :param cycle: :return: """ getter = Getter() while True: print("开始抓取代理") getter.run() time.sleep(cycle)
def run_getter(self): """ run getter """ getter = Getter() loop = 0 while True: logger.debug(f'getter loop {loop} start...') getter.run() loop += 1 time.sleep(20)
def run(self, q, current_time): ## handles the publishing of data, should be run at an acceptable update interval # get data from queue data = [] labels = self.name if type(labels) != tuple: labels = (labels) for label in labels: data.append(q[label]) # publish data Getter.run(self, data[0])
def schedule_getter(self, cycle=GETTER_CYCLE): """Cycle get proxy""" coroutine = Getter() while True: loop = asyncio.get_event_loop() loop.run_until_complete(coroutine.run()) time.sleep(cycle)
def run(self, q, current_time): ## handles the publishing of data, should be run at an acceptable update interval # get data from queue data = q[self.name] # get the data from the message data = data.split('v=')[-1] if (data == ''): return # create x and y arrays from the base16 string of data strarr = data.split(':') x = [] y = [] (lambda a: (x.append(int(a[0], 16)), y.append(int(a[1], 16)), fx(a[2:])) if a.size > 1 else False)(strarr) # create pointcloud2 message dataout = PointCloud2() # add timestamp based on ros time msg.header.stamp = current_time # add frame msg.header.frame_id = self.hostname # format message N = len(x) xy = np.array(np.hstack([x, y]), dtype=np.float32) msg.height = 1 msg.width = N msg.fields = [ PointField('x', 0, PointField.FLOAT32, 1), PointField('y', 4, PointField.FLOAT32, 1), ] msg.is_bigendian = False msg.point_step = 8 msg.row_step = msg.point_step * N msg.is_dense = True msg.data = xy.tostring() # publish data Getter.run(self, dataout)
def run(self, q, current_time): ## handles the publishing of data, should be run at an acceptable update interval # get data from queue odom = Odometry() odom.header.stamp = current_time odom.header.frame_id = self.hostname data = [] labels = self.name if type(labels) != tuple: labels = (labels) for label in labels: data.append(q[label]) heading = tf.transformations.quaternion_from_euler(0., 0., data[2]) odom.pose.pose = Pose( Point(data[0], data[1], 0.), Quaternion(heading[0], heading[1], heading[2], heading[3])) odom.twist.twist = Twist(Vector3(data[3], 0, 0), Vector3(0, 0, data[4])) # publish data Getter.run(self, odom)
def scheduler_getter(self, cycle=GETTER_CYCLE): getter = Getter() while True: print("开始捉取代理") getter.run() time.sleep(cycle)
def set_getter(self, cycle=GETTER_CYCLE): getter = Getter() while True: getter.run() time.sleep(cycle)
def schedule_crawl(self, cycle=CRAWL_CYCLE_TIME): crawl = Getter() while True: print("抓取模块运行") crawl.run() time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE): getter = Getter() while True: print("start get proxy") getter.run() time.sleep(cycle)
def run_getter(): getter = Getter() getter.run()
def getter(self, cylcle=CYCLE_GETTER): getter = Getter() while True: print("Start to get proxy") getter.run() sleep(cylcle)
def scheduler_getter(cycle=GETTER_CYCLE): getter = Getter() while True: print('获取器开始运行:') getter.run() time.sleep(cycle)
def schedule_getter(cycle=GETTER_CYCLE): getter=Getter() while True: print('开始抓取代理') getter.run() time.sleep(cycle)