Example #1
0
    def handle(self, *args, **options):
        from huey.djhuey import HUEY
        try:
            consumer_options = settings.HUEY['consumer_options']
        except:
            consumer_options = {}

        if options['workers'] is not None:
            consumer_options['workers'] = options['workers']

        if options['periodic'] is not None:
            consumer_options['periodic'] = options['periodic']

        if options['initial_delay'] is not None:
            consumer_options['initial_delay'] = options['initial_delay']

        if options['max_delay'] is not None:
            consumer_options['max_delay'] = options['max_delay']

        self.autodiscover()

        loglevel = get_loglevel(consumer_options.pop('loglevel', None))
        logfile = consumer_options.pop('logfile', None)
        setup_logger(loglevel, logfile)

        consumer = Consumer(HUEY, **consumer_options)
        consumer.run()
Example #2
0
    def handle(self, *args, **options):

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        autodiscover_modules("tasks")

        huey_queue = settings.HUEY_QUEUES[options["queue"]]
        #del options["queue"]

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(huey_queue, **config.values)
        logger.info("The following tasks are available for this queue:")
        for command in consumer.huey.registry._registry:
            logger.info(command.replace('queue_task_', ''))
        consumer.run()
Example #3
0
    def handle(self, *args, **options):
        # добавлено
        if settings.MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION:
            raise CommandError('If you want to use queues, set MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION setting to False')

        # from huey.contrib.djhuey import HUEY

        # добавлено
        queue = options.pop('queue')

        consumer_options = {}
        # if isinstance(settings.HUEY, dict):
            # consumer_options.update(settings.HUEY.get('consumer', {}))

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))
        self.autodiscover()

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        # consumer = Consumer(HUEY, **config.values)
        consumer = Consumer(HUEYS[queue], **config.values)
        consumer.run()
Example #4
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        logger = logging.getLogger('huey')

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Example #5
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        if isinstance(settings.HUEY, dict):
            consumer_options.update(settings.HUEY.get('consumer', {}))

        if options['workers'] is not None:
            consumer_options['workers'] = options['workers']

        if options['worker_type'] is not None:
            consumer_options['worker_type'] = options['worker_type']

        if options['periodic'] is not None:
            consumer_options['periodic'] = options['periodic']

        if options['initial_delay'] is not None:
            consumer_options['initial_delay'] = options['initial_delay']

        if options['max_delay'] is not None:
            consumer_options['max_delay'] = options['max_delay']

        self.autodiscover()

        loglevel = get_loglevel(consumer_options.pop('loglevel', None))
        logfile = consumer_options.pop('logfile', None)
        setup_logger(loglevel, logfile, consumer_options['worker_type'])

        consumer = Consumer(HUEY, **consumer_options)
        consumer.run()
Example #6
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        autodiscover_modules("tasks")

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Example #7
0
    def test_consumer_integration(self):
        lock = threading.Lock()

        @self.huey.task()
        def add_values(a, b):
            return a + b

        consumer = Consumer(self.huey, max_delay=0.1, workers=2,
                            worker_type='thread', health_check_interval=0.01)

        with CaptureLogs() as capture:
            consumer.start()
            try:
                r1 = add_values(1, 2)
                r2 = add_values(2, 3)
                r3 = add_values(3, 5)

                self.assertEqual(r1.get(blocking=True, timeout=3), 3)
                self.assertEqual(r2.get(blocking=True, timeout=3), 5)
                self.assertEqual(r3.get(blocking=True, timeout=3), 8)
            finally:
                consumer.stop()
                for _, worker in consumer.worker_threads:
                    worker.join()

        messages = capture.messages[-4:-1]
        for message in messages:
            self.assertTrue(message.startswith('Executing huey.tests.test_'))
        self.assertTrue(capture.messages[-1].startswith('Shutting down'))
Example #8
0
def consumer_main():
    parser = get_option_parser()
    options, args = parser.parse_args()

    setup_logger(get_loglevel(options.verbose), options.logfile,
                 options.worker_type)

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    if options.workers < 1:
        err('You must have at least one worker.')
        sys.exit(1)

    if options.list_registered:
        list_registered()

    huey_instance = load_huey(args[0])

    consumer = Consumer(huey_instance, options.workers, options.periodic,
                        options.initial_delay, options.backoff,
                        options.max_delay, options.utc,
                        options.scheduler_interval, options.worker_type)
    consumer.run()
Example #9
0
def consumer_main():
    parser_handler = OptionParserHandler()
    parser = parser_handler.get_option_parser()
    options, args = parser.parse_args()

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    config = ConsumerConfig(**options.__dict__)
    config.validate()

    huey_instance = load_huey(args[0])
    config.setup_logger()

    consumer = Consumer(huey_instance, **config.values)
    consumer.run()
Example #10
0
    def test_dequeue_errors(self):
        huey = BrokenHuey()
        consumer = Consumer(huey, max_delay=0.1, workers=2,
                            worker_type='thread')

        worker = consumer._create_worker()
        state = {}

        @huey.task()
        def modify_broken(k, v):
            state[k] = v

        with CaptureLogs() as capture:
            res = modify_broken('k', 'v')
            worker.loop()

        self.assertEqual(capture.messages, ['Error reading from queue'])
        self.assertEqual(state, {})
Example #11
0
    def setUp(self):
        global state
        state = {}

        self.orig_pc = registry._periodic_tasks
        registry._periodic_commands = [every_hour.task_class()]

        self.orig_sleep = time.sleep
        time.sleep = lambda x: None

        test_huey.queue.flush()
        test_huey.result_store.flush()
        test_huey.schedule.flush()
        test_events._events = deque()

        self.consumer = Consumer(test_huey, workers=2)
        self.consumer._create_threads()

        self.handler = TestLogHandler()
        logger.addHandler(self.handler)
        logger.setLevel(logging.INFO)
Example #12
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        # Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
        # case we must explicitly configure mp to use fork().
        if sys.version_info >= (3, 8) and sys.platform == 'darwin':
            import multiprocessing
            multiprocessing.set_start_method('fork')

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        logger = logging.getLogger('huey')

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Example #13
0
 def create_consumer(self, worker_type='thread'):
     return Consumer(self.huey,
                     max_delay=0.1,
                     workers=2,
                     worker_type=worker_type)
Example #14
0
File: base.py Project: zf-w11/huey
 def get_consumer(self, **kwargs):
     return Consumer(self.huey, **kwargs)
Example #15
0
 def create_consumer(self, **options):
     return Consumer(self, **options)
Example #16
0
def _run_consumer():
    consumer = Consumer(huey)
    consumer._logger.addHandler(logging.FileHandler(DEFAULT_LOG_PATH))
    consumer._logger.addHandler(logging.StreamHandler(sys.stdout))
    consumer.run()
Example #17
0
 def create_consumer(self, **config):
     return Consumer(self, **config)
Example #18
0
    parser.add_option('--localtime',
                      dest='utc',
                      action='store_false',
                      help='use local time for all tasks')
    return parser


if __name__ == '__main__':
    parser = get_option_parser()
    options, args = parser.parse_args()

    setup_logger(get_loglevel(options.verbose), options.logfile)

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    try:
        huey_instance = load_class(args[0])
    except:
        err('Error importing %s' % args[0])
        raise

    consumer = Consumer(huey_instance, options.workers, options.periodic,
                        options.initial_delay, options.backoff,
                        options.max_delay, options.utc,
                        options.scheduler_interval,
                        options.periodic_task_interval)
    consumer.run()