def _run_func(self, function, *arguments):
"""
执行回调函数
Running callback function with requested number of argumentsq
"""
# A named tuple ArgSpec(args, varargs, keywords, defaults) is returned.
# args is a list of the parameter names.
# varargs and keywords are the names of the * and ** parameters or None.
# defaults is a tuple of default argument values or None if there are no default arguments
# Deprecated since version 3.0: Use getfullargspec() for an updated API that is usually a drop-in replacement,
# but also correctly handles function annotations and keyword-only parameters.
args, varargs, keywords, defaults = inspect.getargspec(function)
task = arguments[-1]
process_time_limit = task['process'].get(
'process_time_limit', self.__env__.get('process_time_limit', 0))
if process_time_limit > 0:
# 有超时的process
with timeout(process_time_limit, 'process timeout'):
ret = function(*arguments[:len(args) - 1])
else:
# 执行函数
ret = function(*arguments[:len(args) - 1])
return ret
def setUpClass(self):
with utils.timeout(3):
self.q1 = rabbitmq.Queue('test_queue', maxsize=5)
self.q2 = rabbitmq.Queue('test_queue', maxsize=5)
self.q3 = rabbitmq.Queue('test_queue_for_threading_test')
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
with utils.timeout(3):
self.q1 = rabbitmq.Queue('test_queue', maxsize=5)
self.q2 = rabbitmq.Queue('test_queue', maxsize=5)
self.q3 = rabbitmq.Queue('test_queue_for_threading_test')
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.libs import rabbitmq
with utils.timeout(3):
self.q1 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
self.q2 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
self.q3 = rabbitmq.AmqpQueue('test_queue_for_threading_test')
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.message_queue import rabbitmq
with utils.timeout(3):
self.q1 = rabbitmq.PikaQueue('test_queue', maxsize=5)
self.q2 = rabbitmq.PikaQueue('test_queue', amqp_url='amqp://*****:*****@localhost:5672/')
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.libs import rabbitmq
with utils.timeout(3):
self.q1 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
self.q2 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
self.q3 = rabbitmq.AmqpQueue('test_queue_for_threading_test')
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.message_queue import rabbitmq
with utils.timeout(3):
self.q1 = rabbitmq.PikaQueue('test_queue', maxsize=5, lazy_limit=False)
self.q2 = rabbitmq.PikaQueue('test_queue', amqp_url='amqp://*****:*****@localhost:5672/', lazy_limit=False)
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue("test_queue", "amqp://*****:*****@localhost:5672/")
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.libs import beanstalk
with utils.timeout(3):
self.q1 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
self.q2 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
self.q3 = beanstalk.BeanstalkQueue('test_queue_for_threading_test')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.libs import redis_queue
with utils.timeout(3):
self.q1 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
self.q2 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
self.q3 = redis_queue.RedisQueue('test_queue_for_threading_test')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue('test_queue', self.kombu_url, maxsize=5)
self.q2 = connect_message_queue('test_queue', self.kombu_url, maxsize=5)
self.q3 = connect_message_queue('test_queue_for_threading_test', self.kombu_url)
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.libs import beanstalk
with utils.timeout(3):
self.q1 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
self.q2 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
self.q3 = beanstalk.BeanstalkQueue('test_queue_for_threading_test')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue('test_queue', self.kombu_url, maxsize=5, lazy_limit=False)
self.q2 = connect_message_queue('test_queue', self.kombu_url, maxsize=5, lazy_limit=False)
self.q3 = connect_message_queue('test_queue_for_threading_test', self.kombu_url, lazy_limit=False)
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue("test_queue", "beanstalk://localhost:11300", maxsize=5)
self.q2 = connect_message_queue("test_queue", "beanstalk://localhost:11300", maxsize=5)
self.q3 = connect_message_queue("test_queue_for_threading_test", "beanstalk://localhost:11300")
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue('test_queue', 'amqp://*****:*****@localhost:5672/')
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue('test_queue', 'amqp://*****:*****@localhost:5672/', lazy_limit=False)
self.q2.delete()
self.q2.reconnect()
self.q3.delete()
self.q3.reconnect()
def _run_func(self, function, *arguments):
"""
Running callback function with requested number of arguments
"""
args, varargs, keywords, defaults = inspect.getargspec(function)
task = arguments[-1]
process_time_limit = task['process'].get('process_time_limit',
self.__env__.get('process_time_limit', 0))
if process_time_limit > 0:
with timeout(process_time_limit, 'process timeout'):
ret = function(*arguments[:len(args) - 1])
else:
ret = function(*arguments[:len(args) - 1])
return ret
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
from pyspider.message_queue import redis_queue
with utils.timeout(3):
self.q1 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
self.q2 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
self.q3 = connect_message_queue('test_queue_for_threading_test',
'redis://localhost:6379/')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
from pyspider.message_queue import redis_queue
with utils.timeout(3):
self.q1 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
self.q2 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
self.q3 = connect_message_queue('test_queue_for_threading_test',
'redis://localhost:6379/')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = connect_message_queue('test_queue', 'beanstalk://localhost:11300',
maxsize=5)
self.q2 = connect_message_queue('test_queue', 'beanstalk://localhost:11300',
maxsize=5)
self.q3 = connect_message_queue('test_queue_for_threading_test',
'beanstalk://localhost:11300')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.libs import redis_queue
with utils.timeout(3):
self.q1 = redis_queue.RedisQueue('test_queue',
maxsize=5,
lazy_limit=False)
self.q2 = redis_queue.RedisQueue('test_queue',
maxsize=5,
lazy_limit=False)
self.q3 = redis_queue.RedisQueue('test_queue_for_threading_test')
while not self.q1.empty():
self.q1.get()
while not self.q2.empty():
self.q2.get()
while not self.q3.empty():
self.q3.get()
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
from pyspider.message_queue import redis_queue
with utils.timeout(3):
self.q1 = redis_queue.RedisQueue('test_queue',
host='119.27.185.96',
password='******',
maxsize=5,
lazy_limit=False)
self.q2 = redis_queue.RedisQueue('test_queue',
host='119.27.185.96',
password='******',
maxsize=5,
lazy_limit=False)
self.q3 = connect_message_queue(
'test_queue_for_threading_test',
'redis://:[email protected]:6379/1')
while not self.q1.empty():
print("get from q1: %s" % self.q1.get())
while not self.q2.empty():
print("get from q2: %s" % self.q2.get())
while not self.q3.empty():
print("get from q3: %s" % self.q3.get())
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = self.q2 = connect_message_queue('test_queue', maxsize=5)
self.q3 = connect_message_queue('test_queue_for_threading_test')
def setUpClass(self):
from pyspider.message_queue import connect_message_queue
with utils.timeout(3):
self.q1 = self.q2 = connect_message_queue('test_queue', maxsize=5)
self.q3 = connect_message_queue('test_queue_for_threading_test')