def run_getter(self, cycle=GETTER_CYCLE):
"""定时获取cookie"""
getter = Getter()
while True:
print('开始抓取cookies')
getter.run()
time.sleep(cycle)
def schdule_getter(self, cycle=GETTER_CYCLE):
"""定时获取代理"""
getter = Getter()
while 1:
print('start crawl proxy...')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""定时获取代理"""
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def scheduler_getter(self, cycle=GETTER_CYCLE):
# 调度获取模块
'''定时获取代理'''
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=settings.GETTER_CYCLE):
'''
定时抓取
'''
getter = Getter()
while True:
getter.run()
time.sleep(cycle)
def schedule_getter(self):
getter = Getter()
while GETTER_ENABLED:
try:
getter.run()
except:
getter.run()
time.sleep(GETTER_CYLE)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while GETTER_ENABLED:
print('抓取器开始运行')
getter.run()
time.sleep(cycle)
def scheduler_getter(self):
"""
每隔5天获取一次cookies
:return:
"""
while True:
getter = Getter()
getter.run()
time.sleep(self.getter_cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
while循环运行获取器模块，运行一次之后间隔一段时间，间隔时间参数为GETTER_CYCLE配置。
"""
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
Get the agent regularly
"""
getter = Getter()
while True:
print('Start to grab the proxy')
logger.log('INFOR', 'Start to Grab the proxy...')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
#定时获取代理
getter = Getter()
count = 0
while True:
count += 1
print('获取器开始运行', count)
getter.run()
time.sleep(cycle)
def getter_scheduler(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
print('获取器开始执行！')
getter = Getter()
while True:
getter.run()
print('休息', GETTER_CYCLE, '秒')
time.sleep(GETTER_CYCLE)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
"""
getter = Getter()
while True:
print('Start crawling proxy')
getter.run()
time.sleep(cycle)
def run(self, q):
## handles the publishing of data, should be run at an acceptable update interval
# get data from queue
data = q[self.name]
# TODO: format data for publisher
#print("nearest obs data ", data)
# publish data
Getter.run(self, data)
def process_getter(self):
'''获取器执行子进程'''
getter = Getter()
redis = db()
while True:
if redis.count_proxie() < MAXPOOL:
getter.run()
time.sleep(GETTER_CYCLE)
else:
print("代理池已经达到最大容量")
break
def schedule_getter(self, cycle=GETTER_CYCLE):
'''
定时获取代理
:param cycle:
:return:
'''
getter = Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
"""
定时获取代理
:param cycle:
:return:
"""
getter = Getter()
while True:
print("开始抓取代理")
getter.run()
time.sleep(cycle)
def run_getter(self):
"""
run getter
"""
getter = Getter()
loop = 0
while True:
logger.debug(f'getter loop {loop} start...')
getter.run()
loop += 1
time.sleep(20)
def run(self, q, current_time):
## handles the publishing of data, should be run at an acceptable update interval
# get data from queue
data = []
labels = self.name
if type(labels) != tuple:
labels = (labels)
for label in labels:
data.append(q[label])
# publish data
Getter.run(self, data[0])
def schedule_getter(self, cycle=GETTER_CYCLE):
"""Cycle get proxy"""
coroutine = Getter()
while True:
loop = asyncio.get_event_loop()
loop.run_until_complete(coroutine.run())
time.sleep(cycle)
def run(self, q, current_time):
## handles the publishing of data, should be run at an acceptable update interval
# get data from queue
data = q[self.name]
# get the data from the message
data = data.split('v=')[-1]
if (data == ''):
return
# create x and y arrays from the base16 string of data
strarr = data.split(':')
x = []
y = []
(lambda a:
(x.append(int(a[0], 16)), y.append(int(a[1], 16)), fx(a[2:]))
if a.size > 1 else False)(strarr)
# create pointcloud2 message
dataout = PointCloud2()
# add timestamp based on ros time
msg.header.stamp = current_time
# add frame
msg.header.frame_id = self.hostname
# format message
N = len(x)
xy = np.array(np.hstack([x, y]), dtype=np.float32)
msg.height = 1
msg.width = N
msg.fields = [
PointField('x', 0, PointField.FLOAT32, 1),
PointField('y', 4, PointField.FLOAT32, 1),
]
msg.is_bigendian = False
msg.point_step = 8
msg.row_step = msg.point_step * N
msg.is_dense = True
msg.data = xy.tostring()
# publish data
Getter.run(self, dataout)
def run(self, q, current_time):
## handles the publishing of data, should be run at an acceptable update interval
# get data from queue
odom = Odometry()
odom.header.stamp = current_time
odom.header.frame_id = self.hostname
data = []
labels = self.name
if type(labels) != tuple:
labels = (labels)
for label in labels:
data.append(q[label])
heading = tf.transformations.quaternion_from_euler(0., 0., data[2])
odom.pose.pose = Pose(
Point(data[0], data[1], 0.),
Quaternion(heading[0], heading[1], heading[2], heading[3]))
odom.twist.twist = Twist(Vector3(data[3], 0, 0),
Vector3(0, 0, data[4]))
# publish data
Getter.run(self, odom)
def scheduler_getter(self, cycle=GETTER_CYCLE):
getter = Getter()
while True:
print("开始捉取代理")
getter.run()
time.sleep(cycle)
def set_getter(self, cycle=GETTER_CYCLE):
getter = Getter()
while True:
getter.run()
time.sleep(cycle)
def schedule_crawl(self, cycle=CRAWL_CYCLE_TIME):
crawl = Getter()
while True:
print("抓取模块运行")
crawl.run()
time.sleep(cycle)
def schedule_getter(self, cycle=GETTER_CYCLE):
getter = Getter()
while True:
print("start get proxy")
getter.run()
time.sleep(cycle)
def run_getter():
getter = Getter()
getter.run()
def getter(self, cylcle=CYCLE_GETTER):
getter = Getter()
while True:
print("Start to get proxy")
getter.run()
sleep(cylcle)
def scheduler_getter(cycle=GETTER_CYCLE):
getter = Getter()
while True:
print('获取器开始运行：')
getter.run()
time.sleep(cycle)
def schedule_getter(cycle=GETTER_CYCLE):
getter=Getter()
while True:
print('开始抓取代理')
getter.run()
time.sleep(cycle)
