예제 #1
0
    def _run_func(self, function, *arguments):
        """
        执行回调函数
        Running callback function with requested number of argumentsq
        """

        # A named tuple ArgSpec(args, varargs, keywords, defaults) is returned.
        # args is a list of the parameter names.
        # varargs and keywords are the names of the * and ** parameters or None.
        # defaults is a tuple of default argument values or None if there are no default arguments

        # Deprecated since version 3.0: Use getfullargspec() for an updated API that is usually a drop-in replacement,
        # but also correctly handles function annotations and keyword-only parameters.
        args, varargs, keywords, defaults = inspect.getargspec(function)
        task = arguments[-1]
        process_time_limit = task['process'].get(
            'process_time_limit', self.__env__.get('process_time_limit', 0))
        if process_time_limit > 0:
            # 有超时的process
            with timeout(process_time_limit, 'process timeout'):
                ret = function(*arguments[:len(args) - 1])
        else:
            # 执行函数
            ret = function(*arguments[:len(args) - 1])
        return ret
예제 #2
0
 def setUpClass(self):
     with utils.timeout(3):
         self.q1 = rabbitmq.Queue('test_queue', maxsize=5)
         self.q2 = rabbitmq.Queue('test_queue', maxsize=5)
         self.q3 = rabbitmq.Queue('test_queue_for_threading_test')
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #3
0
 def setUpClass(self):
     with utils.timeout(3):
         self.q1 = rabbitmq.Queue('test_queue', maxsize=5)
         self.q2 = rabbitmq.Queue('test_queue', maxsize=5)
         self.q3 = rabbitmq.Queue('test_queue_for_threading_test')
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #4
0
 def setUpClass(self):
     from pyspider.libs import rabbitmq
     with utils.timeout(3):
         self.q1 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
         self.q2 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
         self.q3 = rabbitmq.AmqpQueue('test_queue_for_threading_test')
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #5
0
 def setUpClass(self):
     from pyspider.message_queue import rabbitmq
     with utils.timeout(3):
         self.q1 = rabbitmq.PikaQueue('test_queue', maxsize=5)
         self.q2 = rabbitmq.PikaQueue('test_queue', amqp_url='amqp://*****:*****@localhost:5672/')
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #6
0
 def setUpClass(self):
     from pyspider.libs import rabbitmq
     with utils.timeout(3):
         self.q1 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
         self.q2 = rabbitmq.AmqpQueue('test_queue', maxsize=5)
         self.q3 = rabbitmq.AmqpQueue('test_queue_for_threading_test')
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #7
0
 def setUpClass(self):
     from pyspider.message_queue import rabbitmq
     with utils.timeout(3):
         self.q1 = rabbitmq.PikaQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q2 = rabbitmq.PikaQueue('test_queue', amqp_url='amqp://*****:*****@localhost:5672/', lazy_limit=False)
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #8
0
    def setUpClass(self):
        from pyspider.message_queue import connect_message_queue

        with utils.timeout(3):
            self.q1 = connect_message_queue("test_queue", "amqp://*****:*****@localhost:5672/")
        self.q2.delete()
        self.q2.reconnect()
        self.q3.delete()
        self.q3.reconnect()
예제 #9
0
 def setUpClass(self):
     from pyspider.libs import beanstalk
     with utils.timeout(3):
         self.q1 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
         self.q2 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
         self.q3 = beanstalk.BeanstalkQueue('test_queue_for_threading_test')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
 def setUpClass(self):
     from pyspider.libs import redis_queue
     with utils.timeout(3):
         self.q1 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q2 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q3 = redis_queue.RedisQueue('test_queue_for_threading_test')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #11
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = connect_message_queue('test_queue', self.kombu_url, maxsize=5)
         self.q2 = connect_message_queue('test_queue', self.kombu_url, maxsize=5)
         self.q3 = connect_message_queue('test_queue_for_threading_test', self.kombu_url)
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #12
0
 def setUpClass(self):
     from pyspider.libs import beanstalk
     with utils.timeout(3):
         self.q1 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
         self.q2 = beanstalk.BeanstalkQueue('test_queue', maxsize=5)
         self.q3 = beanstalk.BeanstalkQueue('test_queue_for_threading_test')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #13
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = connect_message_queue('test_queue', self.kombu_url, maxsize=5, lazy_limit=False)
         self.q2 = connect_message_queue('test_queue', self.kombu_url, maxsize=5, lazy_limit=False)
         self.q3 = connect_message_queue('test_queue_for_threading_test', self.kombu_url, lazy_limit=False)
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #14
0
    def setUpClass(self):
        from pyspider.message_queue import connect_message_queue

        with utils.timeout(3):
            self.q1 = connect_message_queue("test_queue", "beanstalk://localhost:11300", maxsize=5)
            self.q2 = connect_message_queue("test_queue", "beanstalk://localhost:11300", maxsize=5)
            self.q3 = connect_message_queue("test_queue_for_threading_test", "beanstalk://localhost:11300")
            while not self.q1.empty():
                self.q1.get()
            while not self.q2.empty():
                self.q2.get()
            while not self.q3.empty():
                self.q3.get()
예제 #15
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = connect_message_queue('test_queue', 'amqp://*****:*****@localhost:5672/')
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #16
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = connect_message_queue('test_queue', 'amqp://*****:*****@localhost:5672/', lazy_limit=False)
     self.q2.delete()
     self.q2.reconnect()
     self.q3.delete()
     self.q3.reconnect()
예제 #17
0
 def _run_func(self, function, *arguments):
     """
     Running callback function with requested number of arguments
     """
     args, varargs, keywords, defaults = inspect.getargspec(function)
     task = arguments[-1]
     process_time_limit = task['process'].get('process_time_limit',
                                              self.__env__.get('process_time_limit', 0))
     if process_time_limit > 0:
         with timeout(process_time_limit, 'process timeout'):
             ret = function(*arguments[:len(args) - 1])
     else:
         ret = function(*arguments[:len(args) - 1])
     return ret
예제 #18
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     from pyspider.message_queue import redis_queue
     with utils.timeout(3):
         self.q1 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q2 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q3 = connect_message_queue('test_queue_for_threading_test',
                                         'redis://localhost:6379/')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #19
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     from pyspider.message_queue import redis_queue
     with utils.timeout(3):
         self.q1 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q2 = redis_queue.RedisQueue('test_queue', maxsize=5, lazy_limit=False)
         self.q3 = connect_message_queue('test_queue_for_threading_test',
                                         'redis://localhost:6379/')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #20
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = connect_message_queue('test_queue', 'beanstalk://localhost:11300',
                                         maxsize=5)
         self.q2 = connect_message_queue('test_queue', 'beanstalk://localhost:11300',
                                         maxsize=5)
         self.q3 = connect_message_queue('test_queue_for_threading_test',
                                         'beanstalk://localhost:11300')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #21
0
 def setUpClass(self):
     from pyspider.libs import redis_queue
     with utils.timeout(3):
         self.q1 = redis_queue.RedisQueue('test_queue',
                                          maxsize=5,
                                          lazy_limit=False)
         self.q2 = redis_queue.RedisQueue('test_queue',
                                          maxsize=5,
                                          lazy_limit=False)
         self.q3 = redis_queue.RedisQueue('test_queue_for_threading_test')
         while not self.q1.empty():
             self.q1.get()
         while not self.q2.empty():
             self.q2.get()
         while not self.q3.empty():
             self.q3.get()
예제 #22
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     from pyspider.message_queue import redis_queue
     with utils.timeout(3):
         self.q1 = redis_queue.RedisQueue('test_queue',
                                          host='119.27.185.96',
                                          password='******',
                                          maxsize=5,
                                          lazy_limit=False)
         self.q2 = redis_queue.RedisQueue('test_queue',
                                          host='119.27.185.96',
                                          password='******',
                                          maxsize=5,
                                          lazy_limit=False)
         self.q3 = connect_message_queue(
             'test_queue_for_threading_test',
             'redis://:[email protected]:6379/1')
         while not self.q1.empty():
             print("get from q1: %s" % self.q1.get())
         while not self.q2.empty():
             print("get from q2: %s" % self.q2.get())
         while not self.q3.empty():
             print("get from q3: %s" % self.q3.get())
예제 #23
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = self.q2 = connect_message_queue('test_queue', maxsize=5)
         self.q3 = connect_message_queue('test_queue_for_threading_test')
예제 #24
0
 def setUpClass(self):
     from pyspider.message_queue import connect_message_queue
     with utils.timeout(3):
         self.q1 = self.q2 = connect_message_queue('test_queue', maxsize=5)
         self.q3 = connect_message_queue('test_queue_for_threading_test')