Ejemplo n.º 1
0
def work():
    from brightsky.worker import huey
    huey.flush()
    config = ConsumerConfig(worker_type='thread', workers=2*cpu_count()+1)
    config.validate()
    consumer = huey.create_consumer(**config.values)
    consumer.run()
Ejemplo n.º 2
0
def work():
    """Start brightsky worker."""
    huey.flush()
    config = ConsumerConfig(worker_type='thread', workers=2 * cpu_count() + 1)
    config.validate()
    consumer = huey.create_consumer(**config.values)
    consumer.run()
Ejemplo n.º 3
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Ejemplo n.º 4
0
 def test_default_config(self):
     cfg = ConsumerConfig()
     cfg.validate()
     consumer = self.huey.create_consumer(**cfg.values)
     self.assertEqual(consumer.workers, 1)
     self.assertEqual(consumer.worker_type, 'thread')
     self.assertTrue(consumer.periodic)
     self.assertEqual(consumer.default_delay, 0.1)
     self.assertEqual(consumer.scheduler_interval, 1)
     self.assertTrue(consumer._health_check)
Ejemplo n.º 5
0
    def test_consumer_config(self):
        cfg = ConsumerConfig(workers=3, worker_type='process', initial_delay=1,
                             backoff=2, max_delay=4, check_worker_health=False,
                             scheduler_interval=30, periodic=False)
        cfg.validate()
        consumer = self.huey.create_consumer(**cfg.values)

        self.assertEqual(consumer.workers, 3)
        self.assertEqual(consumer.worker_type, 'process')
        self.assertFalse(consumer.periodic)
        self.assertEqual(consumer.default_delay, 1)
        self.assertEqual(consumer.backoff, 2)
        self.assertEqual(consumer.max_delay, 4)
        self.assertEqual(consumer.scheduler_interval, 30)
        self.assertFalse(consumer._health_check)
Ejemplo n.º 6
0
def start_task_queue(num_workers: int) -> None:
    """Start the Huey task queue."""
    schedule_tasks()

    config = ConsumerConfig(workers=num_workers)
    consumer = huey.create_consumer(**config.values)
    consumer.run()
Ejemplo n.º 7
0
def main(
    port: int = os.getenv("CONSUMER_PROMETHEUS_PORT", 9102),
    load_env_secrets: bool = False,
):
    start_http_server(port)

    if load_env_secrets:
        load_secrets()

    config = ConsumerConfig(
        workers=int(os.getenv("HUEY_CONSUMER_WORKERS", 32)),
        periodic=False,
        worker_type=WORKER_THREAD,
    )
    logger = logging.getLogger("huey")
    config.setup_logger(logger)
    huey.create_consumer(**config.values).run()
Ejemplo n.º 8
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        if isinstance(settings.HUEY, dict):
            consumer_options.update(settings.HUEY.get('consumer', {}))

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))
        self.autodiscover()

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Ejemplo n.º 9
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        logger = logging.getLogger('huey')

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Ejemplo n.º 10
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        autodiscover_modules("tasks")

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(HUEY, **config.values)
        consumer.run()
Ejemplo n.º 11
0
    def handle(self, *args, **options):
        # добавлено
        if settings.MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION:
            raise CommandError('If you want to use queues, set MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION setting to False')

        # from huey.contrib.djhuey import HUEY

        # добавлено
        queue = options.pop('queue')

        consumer_options = {}
        # if isinstance(settings.HUEY, dict):
            # consumer_options.update(settings.HUEY.get('consumer', {}))

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))
        self.autodiscover()

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        # consumer = Consumer(HUEY, **config.values)
        consumer = Consumer(HUEYS[queue], **config.values)
        consumer.run()
Ejemplo n.º 12
0
    def handle(self, *args, **options):

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        autodiscover_modules("tasks")

        huey_queue = settings.HUEY_QUEUES[options["queue"]]
        #del options["queue"]

        config = ConsumerConfig(**consumer_options)
        config.validate()
        config.setup_logger()

        consumer = Consumer(huey_queue, **config.values)
        logger.info("The following tasks are available for this queue:")
        for command in consumer.huey.registry._registry:
            logger.info(command.replace('queue_task_', ''))
        consumer.run()
Ejemplo n.º 13
0
    def handle(self, *args, **options):
        from django_huey import get_queue, get_queue_name

        # Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
        # case we must explicitly configure mp to use fork().
        if sys.version_info >= (3, 8) and sys.platform == "darwin":
            # Apparently this was causing a "context has already been set"
            # error for some user. We'll just pass and hope for the best.
            # They're apple users so presumably nothing important will be lost.
            import multiprocessing

            try:
                multiprocessing.set_start_method("fork")
            except RuntimeError:
                pass

        queue_name = options.get("queue")
        queue = get_queue(queue_name)
        queue_name = get_queue_name(queue_name)

        consumer_options = {}
        consumer_options.update(self.default_queue_settings(queue_name))

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault("verbose",
                                    consumer_options.pop("huey_verbose", None))

        if not options.get("disable_autoload"):
            autodiscover_modules("tasks")

        logger = logging.getLogger("huey")

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = queue.create_consumer(**config.values)
        consumer.run()
Ejemplo n.º 14
0
    def run_consumer(self, queue_name):

        multiple_scheduler_locking = self.consumer_options.pop(
            'multiple_scheduler_locking', False)

        HUEY = settings_reader.configurations[queue_name].huey_instance

        config = ConsumerConfig(**self.consumer_options)
        config.validate()
        config.setup_logger()

        logger.info(f'Run huey on {queue_name}')
        consumer = HueyxConsumer(
            HUEY,
            multiple_scheduler_locking=multiple_scheduler_locking,
            **config.values)
        consumer.run()
Ejemplo n.º 15
0
def main():
    os.nice(10)
    parser_handler = OptionParserHandler()
    parser = parser_handler.get_option_parser()
    options, args = parser.parse_args()
    options = {k: v for k, v in options.__dict__.items()
           if v is not None}
    defaultConf = {'workers': workers, 'worker_type': 'process'}
    config = ConsumerConfig(**collections.ChainMap(options,defaultConf))
    config.validate()

    huey_instance = load_class("lcars.settings.HUEY")

    logger = logging.getLogger('huey')
    config.setup_logger(logger)
    consumer =  huey_instance.create_consumer(**config.values)
    consumer.run()
Ejemplo n.º 16
0
    def handle(self, *args, **options):
        from huey.contrib.djhuey import HUEY

        # Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
        # case we must explicitly configure mp to use fork().
        if sys.version_info >= (3, 8) and sys.platform == 'darwin':
            # Apparently this was causing a "context has already been set"
            # error for some user. We'll just pass and hope for the best.
            # They're apple users so presumably nothing important will be lost.
            import multiprocessing
            try:
                multiprocessing.set_start_method('fork')
            except RuntimeError:
                pass

        consumer_options = {}
        try:
            if isinstance(settings.HUEY, dict):
                consumer_options.update(settings.HUEY.get('consumer', {}))
        except AttributeError:
            pass

        for key, value in options.items():
            if value is not None:
                consumer_options[key] = value

        consumer_options.setdefault('verbose',
                                    consumer_options.pop('huey_verbose', None))

        if not options.get('disable_autoload'):
            autodiscover_modules("tasks")

        logger = logging.getLogger('huey')

        config = ConsumerConfig(**consumer_options)
        config.validate()

        # Only configure the "huey" logger if it has no handlers. For example,
        # some users may configure the huey logger via the Django global
        # logging config. This prevents duplicating log messages:
        if not logger.handlers:
            config.setup_logger(logger)

        consumer = HUEY.create_consumer(**config.values)
        consumer.run()
Ejemplo n.º 17
0
def consumer_main():
    parser_handler = OptionParserHandler()
    parser = parser_handler.get_option_parser()
    options, args = parser.parse_args()

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    config = ConsumerConfig(**options.__dict__)
    config.validate()

    huey_instance = load_huey(args[0])
    config.setup_logger()

    consumer = Consumer(huey_instance, **config.values)
    consumer.run()
def start_consumer(workers, worker_type, scheduler_interval, enable_verbose):
    parser_handler = OptionParserHandler()
    parser = parser_handler.get_option_parser()
    options, args = parser.parse_args()

    options = {k: v for k, v in options.__dict__.items() if v is not None}
    options["workers"] = workers
    options["worker-type"] = worker_type
    options["scheduler-interval"] = scheduler_interval

    if enable_verbose:
        options["verbose"] = ""

    config = ConsumerConfig(**options)
    config.validate()

    config.setup_logger()

    consumer = huey_queue.create_consumer(**config.values)
    consumer.run()
Ejemplo n.º 19
0
def consumer_main():
    parser_handler = OptionParserHandler()
    parser = parser_handler.get_option_parser()
    options, args = parser.parse_args()

    if len(args) == 0:
        err('Error:   missing import path to `Huey` instance')
        err('Example: huey_consumer.py app.queue.huey_instance')
        sys.exit(1)

    options = {k: v for k, v in options.__dict__.items() if v is not None}
    config = ConsumerConfig(**options)
    config.validate()

    huey_instance = load_huey(args[0])

    # Set up logging for the "huey" namespace.
    logger = logging.getLogger('huey')
    config.setup_logger(logger)

    consumer = huey_instance.create_consumer(**config.values)
    consumer.run()
Ejemplo n.º 20
0
 def assertInvalid(**kwargs):
     cfg = ConsumerConfig(**kwargs)
     self.assertRaises(ValueError, cfg.validate)