示例#1
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        autodiscover_modules("tasks")

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
示例#2
0
文件: run_huey.py 项目: coleifer/huey
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
示例#3
0
    def handle(self, *args, **options):
        from huey.djhuey import HUEY
        try:
            consumer_options = settings.HUEY['consumer_options']
        except:
            consumer_options = {}

        if options['workers'] is not None:
            consumer_options['workers'] = options['workers']

        if options['periodic'] is not None:
            consumer_options['periodic'] = options['periodic']

        if options['initial_delay'] is not None:
            consumer_options['initial_delay'] = options['initial_delay']

        if options['max_delay'] is not None:
            consumer_options['max_delay'] = options['max_delay']

        self.autodiscover()

        loglevel = get_loglevel(consumer_options.pop('loglevel', None))
        logfile = consumer_options.pop('logfile', None)
        setup_logger(loglevel, logfile)

        consumer = Consumer(HUEY, **consumer_options)
        consumer.run()
示例#4
0
def consumer_main():
    parser = get_option_parser()
    options, args = parser.parse_args()

    setup_logger(get_loglevel(options.verbose), options.logfile,
                 options.worker_type)

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    if options.workers < 1:
        err('You must have at least one worker.')
        sys.exit(1)

    if options.list_registered:
        list_registered()

    huey_instance = load_huey(args[0])

    consumer = Consumer(huey_instance, options.workers, options.periodic,
                        options.initial_delay, options.backoff,
                        options.max_delay, options.utc,
                        options.scheduler_interval, options.worker_type)
    consumer.run()
示例#5
0
文件: run_huey.py 项目: yijxiang/huey
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        logger = logging.getLogger('huey')

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
示例#6
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        if isinstance(settings.HUEY, dict):
            consumer_options.update(settings.HUEY.get('consumer', {}))

        if options['workers'] is not None:
            consumer_options['workers'] = options['workers']

        if options['worker_type'] is not None:
            consumer_options['worker_type'] = options['worker_type']

        if options['periodic'] is not None:
            consumer_options['periodic'] = options['periodic']

        if options['initial_delay'] is not None:
            consumer_options['initial_delay'] = options['initial_delay']

        if options['max_delay'] is not None:
            consumer_options['max_delay'] = options['max_delay']

        self.autodiscover()

        loglevel = get_loglevel(consumer_options.pop('loglevel', None))
        logfile = consumer_options.pop('logfile', None)
        setup_logger(loglevel, logfile, consumer_options['worker_type'])

        consumer = Consumer(HUEY, **consumer_options)
        consumer.run()
示例#7
0
文件: run_huey.py 项目: joe-nas/huey
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        if isinstance(settings.HUEY, dict):
            consumer_options.update(settings.HUEY.get('consumer', {}))

        if options['workers'] is not None:
            consumer_options['workers'] = options['workers']

        if options['worker_type'] is not None:
            consumer_options['worker_type'] = options['worker_type']

        if options['periodic'] is not None:
            consumer_options['periodic'] = options['periodic']

        if options['initial_delay'] is not None:
            consumer_options['initial_delay'] = options['initial_delay']

        if options['max_delay'] is not None:
            consumer_options['max_delay'] = options['max_delay']

        self.autodiscover()

        loglevel = get_loglevel(consumer_options.pop('loglevel', None))
        logfile = consumer_options.pop('logfile', None)
        setup_logger(loglevel, logfile, consumer_options['worker_type'])

        consumer = Consumer(HUEY, **consumer_options)
        consumer.run()
示例#8
0
    def handle(self, *args, **options):

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        autodiscover_modules("tasks")

        huey_queue = settings.HUEY_QUEUES[options["queue"]]
        #del options["queue"]

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(huey_queue, **config.values)
        logger.info("The following tasks are available for this queue:")
        for command in consumer.huey.registry._registry:
            logger.info(command.replace('queue_task_', ''))
        consumer.run()
示例#9
0
def consumer_main():
    parser = get_option_parser()
    options, args = parser.parse_args()

    setup_logger(
        get_loglevel(options.verbose),
        options.logfile,
        options.worker_type)

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    if options.workers < 1:
        err('You must have at least one worker.')
        sys.exit(1)

    huey_instance = load_huey(args[0])

    consumer = Consumer(
        huey_instance,
        options.workers,
        options.periodic,
        options.initial_delay,
        options.backoff,
        options.max_delay,
        options.utc,
        options.scheduler_interval,
        options.worker_type)
    consumer.run()
示例#10
0
文件: run_huey.py 项目: CheckiO/huey
    def handle(self, *args, **options):
        from huey.djhuey import HUEY
        try:
            consumer_options = settings.HUEY['consumer_options']
        except:
            consumer_options = {}

        if options['workers'] is not None:
            consumer_options['workers'] = options['workers']

        if options['periodic'] is not None:
            consumer_options['periodic'] = options['periodic']

        if options['initial_delay'] is not None:
            consumer_options['initial_delay'] = options['initial_delay']

        if options['max_delay'] is not None:
            consumer_options['max_delay'] = options['max_delay']

        self.autodiscover()

        loglevel = get_loglevel(consumer_options.pop('loglevel', None))
        logfile = consumer_options.pop('logfile', None)
        setup_logger(loglevel, logfile)

        consumer = Consumer(HUEY, **consumer_options)
        consumer.run()
示例#11
0
    def handle(self, *args, **options):
        # добавлено
        if settings.MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION:
            raise CommandError('If you want to use queues, set MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION setting to False')

        # from huey.contrib.djhuey import HUEY

        # добавлено
        queue = options.pop('queue')

        consumer_options = {}
        # if isinstance(settings.HUEY, dict):
            # consumer_options.update(settings.HUEY.get('consumer', {}))

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))
        self.autodiscover()

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        # consumer = Consumer(HUEY, **config.values)
        consumer = Consumer(HUEYS[queue], **config.values)
        consumer.run()
示例#12
0
def consumer_main():
    parser_handler = OptionParserHandler()
    parser = parser_handler.get_option_parser()
    options, args = parser.parse_args()

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    config = ConsumerConfig(**options.__dict__)
    config.validate()

    huey_instance = load_huey(args[0])
    config.setup_logger()

    consumer = Consumer(huey_instance, **config.values)
    consumer.run()
示例#13
0
    def test_dequeue_errors(self):
        huey = BrokenHuey()
        consumer = Consumer(huey, max_delay=0.1, workers=2,
                            worker_type='thread')

        worker = consumer._create_worker()
        state = {}

        @huey.task()
        def modify_broken(k, v):
            state[k] = v

        with CaptureLogs() as capture:
            res = modify_broken('k', 'v')
            worker.loop()

        self.assertEqual(capture.messages, ['Error reading from queue'])
        self.assertEqual(state, {})
示例#14
0
    def test_dequeue_errors(self):
        huey = BrokenHuey()
        consumer = Consumer(huey, max_delay=0.1, workers=2,
                            worker_type='thread')

        worker = consumer._create_worker()
        state = {}

        @huey.task()
        def modify_broken(k, v):
            state[k] = v

        with CaptureLogs() as capture:
            res = modify_broken('k', 'v')
            worker.loop()

        self.assertEqual(capture.messages, ['Error reading from queue'])
        self.assertEqual(state, {})
示例#15
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        if isinstance(settings.HUEY, dict):
            consumer_options.update(settings.HUEY.get('consumer', {}))

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))
        self.autodiscover()

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
示例#16
0
文件: consumer.py 项目: timdaman/huey
    def setUp(self):
        global state
        state = {}

        self.orig_pc = registry._periodic_tasks
        registry._periodic_commands = [every_hour.task_class()]

        self.orig_sleep = time.sleep
        time.sleep = lambda x: None

        test_huey.queue.flush()
        test_huey.result_store.flush()
        test_huey.schedule.flush()
        test_events._events = deque()

        self.consumer = Consumer(test_huey, workers=2)
        self.consumer._create_threads()

        self.handler = TestLogHandler()
        logger.addHandler(self.handler)
        logger.setLevel(logging.INFO)
示例#17
0
    def test_consumer_integration(self):
        lock = threading.Lock()

        @self.huey.task()
        def add_values(a, b):
            return a + b

        consumer = Consumer(self.huey, max_delay=0.1, workers=2,
                            worker_type='thread', health_check_interval=0.01)

        with CaptureLogs() as capture:
            consumer.start()
            try:
                r1 = add_values(1, 2)
                r2 = add_values(2, 3)
                r3 = add_values(3, 5)

                self.assertEqual(r1.get(blocking=True, timeout=3), 3)
                self.assertEqual(r2.get(blocking=True, timeout=3), 5)
                self.assertEqual(r3.get(blocking=True, timeout=3), 8)
            finally:
                consumer.stop()
                for _, worker in consumer.worker_threads:
                    worker.join()

        messages = capture.messages[-4:-1]
        for message in messages:
            self.assertTrue(message.startswith('Executing huey.tests.test_'))
        self.assertTrue(capture.messages[-1].startswith('Shutting down'))
示例#18
0
文件: run_huey.py 项目: ydrallHT/huey
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        # Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
        # case we must explicitly configure mp to use fork().
        if sys.version_info >= (3, 8) and sys.platform == 'darwin':
            import multiprocessing
            multiprocessing.set_start_method('fork')

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        logger = logging.getLogger('huey')

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
示例#19
0
文件: consumer.py 项目: CheckiO/huey
    def setUp(self):
        global state
        state = {}

        self.orig_pc = registry._periodic_tasks
        registry._periodic_commands = [every_hour.task_class()]

        self.orig_sleep = time.sleep
        time.sleep = lambda x: None

        test_huey.queue.flush()
        test_huey.result_store.flush()
        test_huey.schedule.flush()
        test_events._events = deque()

        self.consumer = Consumer(test_huey, workers=2)
        self.consumer._create_threads()

        self.handler = TestLogHandler()
        logger.addHandler(self.handler)
        logger.setLevel(logging.INFO)
示例#20
0
文件: consumer.py 项目: timdaman/huey
class ConsumerTestCase(unittest.TestCase):
    def setUp(self):
        global state
        state = {}

        self.orig_pc = registry._periodic_tasks
        registry._periodic_commands = [every_hour.task_class()]

        self.orig_sleep = time.sleep
        time.sleep = lambda x: None

        test_huey.queue.flush()
        test_huey.result_store.flush()
        test_huey.schedule.flush()
        test_events._events = deque()

        self.consumer = Consumer(test_huey, workers=2)
        self.consumer._create_threads()

        self.handler = TestLogHandler()
        logger.addHandler(self.handler)
        logger.setLevel(logging.INFO)

    def tearDown(self):
        self.consumer.shutdown()
        logger.removeHandler(self.handler)
        registry._periodic_tasks = self.orig_pc
        time.sleep = self.orig_sleep

    def assertStatusTask(self, status_task):
        parsed = []
        i = 0
        while i < len(status_task):
            event = json.loads(test_events._events[i])
            status, task, extra = status_task[i]
            self.assertEqual(event['status'], status)
            self.assertEqual(event['id'], task.task_id)
            for k, v in extra.items():
                self.assertEqual(event[k], v)
            i += 1

    def spawn(self, func, *args, **kwargs):
        t = threading.Thread(target=func, args=args, kwargs=kwargs)
        t.start()
        return t

    def run_worker(self, task, ts=None):
        worker_t = WorkerThread(test_huey, self.consumer.default_delay,
                                self.consumer.max_delay, self.consumer.backoff,
                                self.consumer.utc, self.consumer._shutdown)
        ts = ts or datetime.datetime.utcnow()
        worker_t.handle_task(task, ts)

    def test_message_processing(self):
        self.consumer.worker_threads[0].start()

        self.assertFalse('k' in state)

        res = modify_state('k', 'v')
        res.get(blocking=True)

        self.assertTrue('k' in state)
        self.assertEqual(res.get(), 'v')

        self.assertEqual(len(test_events._events), 2)
        self.assertStatusTask([
            ('finished', res.task, {}),
            ('started', res.task, {}),
        ])

    def test_worker(self):
        modify_state('k', 'w')
        task = test_huey.dequeue()
        self.run_worker(task)
        self.assertEqual(state, {'k': 'w'})

    def test_worker_exception(self):
        blow_up()
        task = test_huey.dequeue()

        self.run_worker(task)
        self.assertTrue(
            'Unhandled exception in worker thread' in self.handler.messages)

        self.assertEqual(len(test_events._events), 2)
        self.assertStatusTask([
            ('error', task, {
                'error': True
            }),
            ('started', task, {}),
        ])

    def test_retries_and_logging(self):
        # this will continually fail
        retry_command('blampf')

        for i in reversed(range(4)):
            task = test_huey.dequeue()
            self.assertEqual(task.retries, i)
            self.run_worker(task)
            if i > 0:
                self.assertEqual(
                    self.handler.messages[-1],
                    'Re-enqueueing task %s, %s tries left' %
                    (task.task_id, i - 1))
                self.assertStatusTask([
                    ('enqueued', task, {}),
                    ('retrying', task, {}),
                    ('error', task, {}),
                    ('started', task, {}),
                ])
                last_idx = -2
            else:
                self.assertStatusTask([
                    ('error', task, {}),
                    ('started', task, {}),
                ])
                last_idx = -1
            self.assertEqual(self.handler.messages[last_idx],
                             'Unhandled exception in worker thread')

        self.assertEqual(test_huey.dequeue(), None)

    def test_retries_with_success(self):
        # this will fail once, then succeed
        retry_command('blampf', False)
        self.assertFalse('blampf' in state)

        task = test_huey.dequeue()
        self.run_worker(task)
        self.assertEqual(self.handler.messages, [
            'Executing %s' % task, 'Unhandled exception in worker thread',
            'Re-enqueueing task %s, 2 tries left' % task.task_id
        ])

        task = test_huey.dequeue()
        self.assertEqual(task.retries, 2)
        self.run_worker(task)

        self.assertEqual(state['blampf'], 'fixed')
        self.assertEqual(test_huey.dequeue(), None)

        self.assertStatusTask([
            ('finished', task, {}),
            ('started', task, {}),
            ('enqueued', task, {
                'retries': 2
            }),
            ('retrying', task, {
                'retries': 3
            }),
            ('error', task, {
                'error': True
            }),
            ('started', task, {}),
        ])

    def test_scheduling(self):
        dt = datetime.datetime(2011, 1, 1, 0, 0)
        dt2 = datetime.datetime(2037, 1, 1, 0, 0)
        ad1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
        ad2 = modify_state.schedule(args=('k2', 'v2'),
                                    eta=dt2,
                                    convert_utc=False)

        # dequeue the past-timestamped task and run it.
        worker = self.consumer.worker_threads[0]
        worker.check_message()

        self.assertTrue('k' in state)

        # dequeue the future-timestamped task.
        worker.check_message()

        # verify the task got stored in the schedule instead of executing
        self.assertFalse('k2' in state)

        self.assertStatusTask([
            ('scheduled', ad2.task, {}),
            ('finished', ad1.task, {}),
            ('started', ad1.task, {}),
        ])

        # run through an iteration of the scheduler
        self.consumer.scheduler_t.loop(dt)

        # our command was not enqueued and no events were emitted.
        self.assertEqual(len(test_queue._queue), 0)
        self.assertEqual(len(test_events._events), 3)

        # run through an iteration of the scheduler
        self.consumer.scheduler_t.loop(dt2)

        # our command was enqueued
        self.assertEqual(len(test_queue._queue), 1)
        self.assertEqual(len(test_events._events), 4)
        self.assertStatusTask([
            ('enqueued', ad2.task, {}),
        ])

    def test_retry_scheduling(self):
        # this will continually fail
        retry_command_slow('blampf')
        cur_time = datetime.datetime.utcnow()

        task = test_huey.dequeue()
        self.run_worker(task, ts=cur_time)
        self.assertEqual(self.handler.messages, [
            'Executing %s' % task,
            'Unhandled exception in worker thread',
            'Re-enqueueing task %s, 2 tries left' % task.task_id,
        ])

        in_11 = cur_time + datetime.timedelta(seconds=11)
        tasks_from_sched = test_huey.read_schedule(in_11)
        self.assertEqual(tasks_from_sched, [task])

        task = tasks_from_sched[0]
        self.assertEqual(task.retries, 2)
        exec_time = task.execute_time

        self.assertEqual((exec_time - cur_time).seconds, 10)
        self.assertStatusTask([
            ('scheduled', task, {
                'retries': 2,
                'retry_delay': 10,
                'execute_time': time.mktime(exec_time.timetuple())
            }),
            ('retrying', task, {
                'retries': 3,
                'retry_delay': 10,
                'execute_time': None
            }),
            ('error', task, {}),
            ('started', task, {}),
        ])

    def test_revoking_normal(self):
        # enqueue 2 normal commands
        r1 = modify_state('k', 'v')
        r2 = modify_state('k2', 'v2')

        # revoke the first *before it has been checked*
        r1.revoke()
        self.assertTrue(test_huey.is_revoked(r1.task))
        self.assertFalse(test_huey.is_revoked(r2.task))

        # dequeue a *single* message (r1)
        task = test_huey.dequeue()
        self.run_worker(task)

        self.assertEqual(len(test_events._events), 1)
        self.assertStatusTask([
            ('revoked', r1.task, {}),
        ])

        # no changes and the task was not added to the schedule
        self.assertFalse('k' in state)

        # dequeue a *single* message
        task = test_huey.dequeue()
        self.run_worker(task)

        self.assertTrue('k2' in state)

    def test_revoking_schedule(self):
        global state
        dt = datetime.datetime(2011, 1, 1)
        dt2 = datetime.datetime(2037, 1, 1)

        r1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
        r2 = modify_state.schedule(args=('k2', 'v2'),
                                   eta=dt,
                                   convert_utc=False)
        r3 = modify_state.schedule(args=('k3', 'v3'),
                                   eta=dt2,
                                   convert_utc=False)
        r4 = modify_state.schedule(args=('k4', 'v4'),
                                   eta=dt2,
                                   convert_utc=False)

        # revoke r1 and r3
        r1.revoke()
        r3.revoke()
        self.assertTrue(test_huey.is_revoked(r1.task))
        self.assertFalse(test_huey.is_revoked(r2.task))
        self.assertTrue(test_huey.is_revoked(r3.task))
        self.assertFalse(test_huey.is_revoked(r4.task))

        expected = [
            #state,        schedule
            ({}, 0),
            ({
                'k2': 'v2'
            }, 0),
            ({
                'k2': 'v2'
            }, 1),
            ({
                'k2': 'v2'
            }, 2),
        ]

        for i in range(4):
            estate, esc = expected[i]

            # dequeue a *single* message
            task = test_huey.dequeue()
            self.run_worker(task)

            self.assertEqual(state, estate)
            self.assertEqual(len(test_huey.schedule._schedule), esc)

        # lets pretend its 2037
        future = dt2 + datetime.timedelta(seconds=1)
        self.consumer.scheduler_t.loop(future)
        self.assertEqual(len(test_huey.schedule._schedule), 0)

        # There are two tasks in the queue now (r3 and r4) -- process both.
        for i in range(2):
            task = test_huey.dequeue()
            self.run_worker(task, future)

        self.assertEqual(state, {'k2': 'v2', 'k4': 'v4'})

    def test_revoking_periodic(self):
        global state

        def loop_periodic(ts):
            self.consumer.periodic_t.loop(ts)
            for i in range(len(test_queue._queue)):
                task = test_huey.dequeue()
                self.run_worker(task, ts)

        # revoke the command once
        every_hour.revoke(revoke_once=True)
        self.assertTrue(every_hour.is_revoked())

        # it will be skipped the first go-round
        dt = datetime.datetime(2011, 1, 1, 0, 0)
        loop_periodic(dt)

        # it has not been run
        self.assertEqual(state, {})

        # the next go-round it will be enqueued
        loop_periodic(dt)

        # our command was run
        self.assertEqual(state, {'p': 'y'})

        # reset state
        state = {}

        # revoke the command
        every_hour.revoke()
        self.assertTrue(every_hour.is_revoked())

        # it will no longer be enqueued
        loop_periodic(dt)
        loop_periodic(dt)
        self.assertEqual(state, {})

        # restore
        every_hour.restore()
        self.assertFalse(every_hour.is_revoked())

        # it will now be enqueued
        loop_periodic(dt)
        self.assertEqual(state, {'p': 'y'})

        # reset
        state = {}

        # revoke for an hour
        td = datetime.timedelta(seconds=3600)
        every_hour.revoke(revoke_until=dt + td)

        loop_periodic(dt)
        self.assertEqual(state, {})

        # after an hour it is back
        loop_periodic(dt + td)
        self.assertEqual(state, {'p': 'y'})

        # our data store should reflect the delay
        task_obj = every_hour.task_class()
        self.assertEqual(len(test_huey.result_store._results), 1)
        self.assertTrue(task_obj.revoke_id in test_huey.result_store._results)
示例#21
0
 def create_consumer(self, worker_type='thread'):
     return Consumer(self.huey,
                     max_delay=0.1,
                     workers=2,
                     worker_type=worker_type)
示例#22
0

if __name__ == '__main__':
    parser = get_option_parser()
    options, args = parser.parse_args()

    setup_logger(get_loglevel(options.verbose), options.logfile)

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    try:
        huey_instance = load_class(args[0])
    except:
        err('Error importing %s' % args[0])
        raise

    consumer = Consumer(
        huey_instance,
        options.workers,
        options.periodic,
        options.initial_delay,
        options.backoff,
        options.max_delay,
        options.utc,
        options.scheduler_interval,
        options.periodic_task_interval)
    consumer.run()
示例#23
0
文件: base.py 项目: zf-w11/huey
 def get_consumer(self, **kwargs):
     return Consumer(self.huey, **kwargs)
示例#24
0
 def create_consumer(self, **options):
     return Consumer(self, **options)
示例#25
0
def _run_consumer():
    consumer = Consumer(huey)
    consumer._logger.addHandler(logging.FileHandler(DEFAULT_LOG_PATH))
    consumer._logger.addHandler(logging.StreamHandler(sys.stdout))
    consumer.run()
示例#26
0
def _run_consumer():
    consumer = Consumer(huey)
    consumer._logger.addHandler(logging.FileHandler(DEFAULT_LOG_PATH))
    consumer._logger.addHandler(logging.StreamHandler(sys.stdout))
    consumer.run()
示例#27
0
文件: consumer.py 项目: CheckiO/huey
class ConsumerTestCase(unittest.TestCase):
    def setUp(self):
        global state
        state = {}

        self.orig_pc = registry._periodic_tasks
        registry._periodic_commands = [every_hour.task_class()]

        self.orig_sleep = time.sleep
        time.sleep = lambda x: None

        test_huey.queue.flush()
        test_huey.result_store.flush()
        test_huey.schedule.flush()
        test_events._events = deque()

        self.consumer = Consumer(test_huey, workers=2)
        self.consumer._create_threads()

        self.handler = TestLogHandler()
        logger.addHandler(self.handler)
        logger.setLevel(logging.INFO)

    def tearDown(self):
        self.consumer.shutdown()
        logger.removeHandler(self.handler)
        registry._periodic_tasks = self.orig_pc
        time.sleep = self.orig_sleep

    def assertStatusTask(self, status_task):
        parsed = []
        i = 0
        while i < len(status_task):
            event = json.loads(test_events._events[i])
            status, task, extra = status_task[i]
            self.assertEqual(event['status'], status)
            self.assertEqual(event['id'], task.task_id)
            for k, v in extra.items():
                self.assertEqual(event[k], v)
            i += 1

    def spawn(self, func, *args, **kwargs):
        t = threading.Thread(target=func, args=args, kwargs=kwargs)
        t.start()
        return t

    def run_worker(self, task, ts=None):
        worker_t = WorkerThread(
            test_huey,
            self.consumer.default_delay,
            self.consumer.max_delay,
            self.consumer.backoff,
            self.consumer.utc,
            self.consumer._shutdown)
        ts = ts or datetime.datetime.utcnow()
        worker_t.handle_task(task, ts)

    def test_message_processing(self):
        self.consumer.worker_threads[0].start()

        self.assertFalse('k' in state)

        res = modify_state('k', 'v')
        res.get(blocking=True)

        self.assertTrue('k' in state)
        self.assertEqual(res.get(), 'v')

        self.assertEqual(len(test_events._events), 2)
        self.assertStatusTask([
            ('finished', res.task, {}),
            ('started', res.task, {}),
        ])

    def test_worker(self):
        modify_state('k', 'w')
        task = test_huey.dequeue()
        self.run_worker(task)
        self.assertEqual(state, {'k': 'w'})

    def test_worker_exception(self):
        blow_up()
        task = test_huey.dequeue()

        self.run_worker(task)
        self.assertTrue(
            'Unhandled exception in worker thread' in self.handler.messages)

        self.assertEqual(len(test_events._events), 2)
        self.assertStatusTask([
            ('error', task, {'error': True}),
            ('started', task, {}),
        ])

    def test_retries_and_logging(self):
        # this will continually fail
        retry_command('blampf')

        for i in reversed(range(4)):
            task = test_huey.dequeue()
            self.assertEqual(task.retries, i)
            self.run_worker(task)
            if i > 0:
                self.assertEqual(
                    self.handler.messages[-1],
                    'Re-enqueueing task %s, %s tries left' % (
                        task.task_id, i - 1))
                self.assertStatusTask([
                    ('enqueued', task, {}),
                    ('retrying', task, {}),
                    ('error', task,{}),
                    ('started', task, {}),
                ])
                last_idx = -2
            else:
                self.assertStatusTask([
                    ('error', task,{}),
                    ('started', task, {}),
                ])
                last_idx = -1
            self.assertEqual(self.handler.messages[last_idx],
                             'Unhandled exception in worker thread')

        self.assertEqual(test_huey.dequeue(), None)

    def test_retries_with_success(self):
        # this will fail once, then succeed
        retry_command('blampf', False)
        self.assertFalse('blampf' in state)

        task = test_huey.dequeue()
        self.run_worker(task)
        self.assertEqual(self.handler.messages, [
            'Executing %s' % task,
            'Unhandled exception in worker thread',
            'Re-enqueueing task %s, 2 tries left' % task.task_id])

        task = test_huey.dequeue()
        self.assertEqual(task.retries, 2)
        self.run_worker(task)

        self.assertEqual(state['blampf'], 'fixed')
        self.assertEqual(test_huey.dequeue(), None)

        self.assertStatusTask([
            ('finished', task, {}),
            ('started', task, {}),
            ('enqueued', task, {'retries': 2}),
            ('retrying', task, {'retries': 3}),
            ('error', task, {'error': True}),
            ('started', task, {}),
        ])

    def test_scheduling(self):
        dt = datetime.datetime(2011, 1, 1, 0, 0)
        dt2 = datetime.datetime(2037, 1, 1, 0, 0)
        ad1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
        ad2 = modify_state.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False)

        # dequeue the past-timestamped task and run it.
        worker = self.consumer.worker_threads[0]
        worker.check_message()

        self.assertTrue('k' in state)

        # dequeue the future-timestamped task.
        worker.check_message()

        # verify the task got stored in the schedule instead of executing
        self.assertFalse('k2' in state)

        self.assertStatusTask([
            ('scheduled', ad2.task, {}),
            ('finished', ad1.task, {}),
            ('started', ad1.task, {}),
        ])

        # run through an iteration of the scheduler
        self.consumer.scheduler_t.loop(dt)

        # our command was not enqueued and no events were emitted.
        self.assertEqual(len(test_queue._queue), 0)
        self.assertEqual(len(test_events._events), 3)

        # run through an iteration of the scheduler
        self.consumer.scheduler_t.loop(dt2)

        # our command was enqueued
        self.assertEqual(len(test_queue._queue), 1)
        self.assertEqual(len(test_events._events), 4)
        self.assertStatusTask([
            ('enqueued', ad2.task, {}),
        ])

    def test_retry_scheduling(self):
        # this will continually fail
        retry_command_slow('blampf')
        cur_time = datetime.datetime.utcnow()

        task = test_huey.dequeue()
        self.run_worker(task, ts=cur_time)
        self.assertEqual(self.handler.messages, [
            'Executing %s' % task,
            'Unhandled exception in worker thread',
            'Re-enqueueing task %s, 2 tries left' % task.task_id,
        ])

        in_11 = cur_time + datetime.timedelta(seconds=11)
        tasks_from_sched = test_huey.read_schedule(in_11)
        self.assertEqual(tasks_from_sched, [task])

        task = tasks_from_sched[0]
        self.assertEqual(task.retries, 2)
        exec_time = task.execute_time

        self.assertEqual((exec_time - cur_time).seconds, 10)
        self.assertStatusTask([
            ('scheduled', task, {
                'retries': 2,
                'retry_delay': 10,
                'execute_time': time.mktime(exec_time.timetuple())}),
            ('retrying', task, {
                'retries': 3,
                'retry_delay': 10,
                'execute_time': None}),
            ('error', task, {}),
            ('started', task, {}),
        ])

    def test_revoking_normal(self):
        # enqueue 2 normal commands
        r1 = modify_state('k', 'v')
        r2 = modify_state('k2', 'v2')

        # revoke the first *before it has been checked*
        r1.revoke()
        self.assertTrue(test_huey.is_revoked(r1.task))
        self.assertFalse(test_huey.is_revoked(r2.task))

        # dequeue a *single* message (r1)
        task = test_huey.dequeue()
        self.run_worker(task)

        self.assertEqual(len(test_events._events), 1)
        self.assertStatusTask([
            ('revoked', r1.task, {}),
        ])

        # no changes and the task was not added to the schedule
        self.assertFalse('k' in state)

        # dequeue a *single* message
        task = test_huey.dequeue()
        self.run_worker(task)

        self.assertTrue('k2' in state)

    def test_revoking_schedule(self):
        global state
        dt = datetime.datetime(2011, 1, 1)
        dt2 = datetime.datetime(2037, 1, 1)

        r1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
        r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt, convert_utc=False)
        r3 = modify_state.schedule(args=('k3', 'v3'), eta=dt2, convert_utc=False)
        r4 = modify_state.schedule(args=('k4', 'v4'), eta=dt2, convert_utc=False)

        # revoke r1 and r3
        r1.revoke()
        r3.revoke()
        self.assertTrue(test_huey.is_revoked(r1.task))
        self.assertFalse(test_huey.is_revoked(r2.task))
        self.assertTrue(test_huey.is_revoked(r3.task))
        self.assertFalse(test_huey.is_revoked(r4.task))

        expected = [
            #state,        schedule
            ({},           0),
            ({'k2': 'v2'}, 0),
            ({'k2': 'v2'}, 1),
            ({'k2': 'v2'}, 2),
        ]

        for i in range(4):
            estate, esc = expected[i]

            # dequeue a *single* message
            task = test_huey.dequeue()
            self.run_worker(task)

            self.assertEqual(state, estate)
            self.assertEqual(len(test_huey.schedule._schedule), esc)

        # lets pretend its 2037
        future = dt2 + datetime.timedelta(seconds=1)
        self.consumer.scheduler_t.loop(future)
        self.assertEqual(len(test_huey.schedule._schedule), 0)

        # There are two tasks in the queue now (r3 and r4) -- process both.
        for i in range(2):
            task = test_huey.dequeue()
            self.run_worker(task, future)

        self.assertEqual(state, {'k2': 'v2', 'k4': 'v4'})

    def test_revoking_periodic(self):
        global state
        def loop_periodic(ts):
            self.consumer.periodic_t.loop(ts)
            for i in range(len(test_queue._queue)):
                task = test_huey.dequeue()
                self.run_worker(task, ts)

        # revoke the command once
        every_hour.revoke(revoke_once=True)
        self.assertTrue(every_hour.is_revoked())

        # it will be skipped the first go-round
        dt = datetime.datetime(2011, 1, 1, 0, 0)
        loop_periodic(dt)

        # it has not been run
        self.assertEqual(state, {})

        # the next go-round it will be enqueued
        loop_periodic(dt)

        # our command was run
        self.assertEqual(state, {'p': 'y'})

        # reset state
        state = {}

        # revoke the command
        every_hour.revoke()
        self.assertTrue(every_hour.is_revoked())

        # it will no longer be enqueued
        loop_periodic(dt)
        loop_periodic(dt)
        self.assertEqual(state, {})

        # restore
        every_hour.restore()
        self.assertFalse(every_hour.is_revoked())

        # it will now be enqueued
        loop_periodic(dt)
        self.assertEqual(state, {'p': 'y'})

        # reset
        state = {}

        # revoke for an hour
        td = datetime.timedelta(seconds=3600)
        every_hour.revoke(revoke_until=dt + td)

        loop_periodic(dt)
        self.assertEqual(state, {})

        # after an hour it is back
        loop_periodic(dt + td)
        self.assertEqual(state, {'p': 'y'})

        # our data store should reflect the delay
        task_obj = every_hour.task_class()
        self.assertEqual(len(test_huey.result_store._results), 1)
        self.assertTrue(task_obj.revoke_id in test_huey.result_store._results)
示例#28
0
 def create_consumer(self, **config):
     return Consumer(self, **config)
示例#29
0
    parser.add_option('--localtime',
                      dest='utc',
                      action='store_false',
                      help='use local time for all tasks')
    return parser


if __name__ == '__main__':
    parser = get_option_parser()
    options, args = parser.parse_args()

    setup_logger(get_loglevel(options.verbose), options.logfile)

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    try:
        huey_instance = load_class(args[0])
    except:
        err('Error importing %s' % args[0])
        raise

    consumer = Consumer(huey_instance, options.workers, options.periodic,
                        options.initial_delay, options.backoff,
                        options.max_delay, options.utc,
                        options.scheduler_interval,
                        options.periodic_task_interval)
    consumer.run()