def work():
from brightsky.worker import huey
huey.flush()
config = ConsumerConfig(worker_type='thread', workers=2*cpu_count()+1)
config.validate()
consumer = huey.create_consumer(**config.values)
consumer.run()
def work():
"""Start brightsky worker."""
huey.flush()
config = ConsumerConfig(worker_type='thread', workers=2 * cpu_count() + 1)
config.validate()
consumer = huey.create_consumer(**config.values)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
if not options.get('disable_autoload'):
autodiscover_modules("tasks")
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
consumer = Consumer(HUEY, **config.values)
consumer.run()
def test_default_config(self):
cfg = ConsumerConfig()
cfg.validate()
consumer = self.huey.create_consumer(**cfg.values)
self.assertEqual(consumer.workers, 1)
self.assertEqual(consumer.worker_type, 'thread')
self.assertTrue(consumer.periodic)
self.assertEqual(consumer.default_delay, 0.1)
self.assertEqual(consumer.scheduler_interval, 1)
self.assertTrue(consumer._health_check)
def test_consumer_config(self):
cfg = ConsumerConfig(workers=3, worker_type='process', initial_delay=1,
backoff=2, max_delay=4, check_worker_health=False,
scheduler_interval=30, periodic=False)
cfg.validate()
consumer = self.huey.create_consumer(**cfg.values)
self.assertEqual(consumer.workers, 3)
self.assertEqual(consumer.worker_type, 'process')
self.assertFalse(consumer.periodic)
self.assertEqual(consumer.default_delay, 1)
self.assertEqual(consumer.backoff, 2)
self.assertEqual(consumer.max_delay, 4)
self.assertEqual(consumer.scheduler_interval, 30)
self.assertFalse(consumer._health_check)
def start_task_queue(num_workers: int) -> None:
"""Start the Huey task queue."""
schedule_tasks()
config = ConsumerConfig(workers=num_workers)
consumer = huey.create_consumer(**config.values)
consumer.run()
def main(
port: int = os.getenv("CONSUMER_PROMETHEUS_PORT", 9102),
load_env_secrets: bool = False,
):
start_http_server(port)
if load_env_secrets:
load_secrets()
config = ConsumerConfig(
workers=int(os.getenv("HUEY_CONSUMER_WORKERS", 32)),
periodic=False,
worker_type=WORKER_THREAD,
)
logger = logging.getLogger("huey")
config.setup_logger(logger)
huey.create_consumer(**config.values).run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
self.autodiscover()
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
consumer = Consumer(HUEY, **config.values)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
if not options.get('disable_autoload'):
autodiscover_modules("tasks")
logger = logging.getLogger('huey')
config = ConsumerConfig(**consumer_options)
config.validate()
# Only configure the "huey" logger if it has no handlers. For example,
# some users may configure the huey logger via the Django global
# logging config. This prevents duplicating log messages:
if not logger.handlers:
config.setup_logger(logger)
consumer = Consumer(HUEY, **config.values)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
autodiscover_modules("tasks")
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
consumer = Consumer(HUEY, **config.values)
consumer.run()
def handle(self, *args, **options):
# добавлено
if settings.MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION:
raise CommandError('If you want to use queues, set MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION setting to False')
# from huey.contrib.djhuey import HUEY
# добавлено
queue = options.pop('queue')
consumer_options = {}
# if isinstance(settings.HUEY, dict):
# consumer_options.update(settings.HUEY.get('consumer', {}))
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
self.autodiscover()
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
# consumer = Consumer(HUEY, **config.values)
consumer = Consumer(HUEYS[queue], **config.values)
consumer.run()
def handle(self, *args, **options):
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
autodiscover_modules("tasks")
huey_queue = settings.HUEY_QUEUES[options["queue"]]
#del options["queue"]
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
consumer = Consumer(huey_queue, **config.values)
logger.info("The following tasks are available for this queue:")
for command in consumer.huey.registry._registry:
logger.info(command.replace('queue_task_', ''))
consumer.run()
def handle(self, *args, **options):
from django_huey import get_queue, get_queue_name
# Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
# case we must explicitly configure mp to use fork().
if sys.version_info >= (3, 8) and sys.platform == "darwin":
# Apparently this was causing a "context has already been set"
# error for some user. We'll just pass and hope for the best.
# They're apple users so presumably nothing important will be lost.
import multiprocessing
try:
multiprocessing.set_start_method("fork")
except RuntimeError:
pass
queue_name = options.get("queue")
queue = get_queue(queue_name)
queue_name = get_queue_name(queue_name)
consumer_options = {}
consumer_options.update(self.default_queue_settings(queue_name))
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault("verbose",
consumer_options.pop("huey_verbose", None))
if not options.get("disable_autoload"):
autodiscover_modules("tasks")
logger = logging.getLogger("huey")
config = ConsumerConfig(**consumer_options)
config.validate()
# Only configure the "huey" logger if it has no handlers. For example,
# some users may configure the huey logger via the Django global
# logging config. This prevents duplicating log messages:
if not logger.handlers:
config.setup_logger(logger)
consumer = queue.create_consumer(**config.values)
consumer.run()
def run_consumer(self, queue_name):
multiple_scheduler_locking = self.consumer_options.pop(
'multiple_scheduler_locking', False)
HUEY = settings_reader.configurations[queue_name].huey_instance
config = ConsumerConfig(**self.consumer_options)
config.validate()
config.setup_logger()
logger.info(f'Run huey on {queue_name}')
consumer = HueyxConsumer(
HUEY,
multiple_scheduler_locking=multiple_scheduler_locking,
**config.values)
consumer.run()
def main():
os.nice(10)
parser_handler = OptionParserHandler()
parser = parser_handler.get_option_parser()
options, args = parser.parse_args()
options = {k: v for k, v in options.__dict__.items()
if v is not None}
defaultConf = {'workers': workers, 'worker_type': 'process'}
config = ConsumerConfig(**collections.ChainMap(options,defaultConf))
config.validate()
huey_instance = load_class("lcars.settings.HUEY")
logger = logging.getLogger('huey')
config.setup_logger(logger)
consumer = huey_instance.create_consumer(**config.values)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
# Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
# case we must explicitly configure mp to use fork().
if sys.version_info >= (3, 8) and sys.platform == 'darwin':
# Apparently this was causing a "context has already been set"
# error for some user. We'll just pass and hope for the best.
# They're apple users so presumably nothing important will be lost.
import multiprocessing
try:
multiprocessing.set_start_method('fork')
except RuntimeError:
pass
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
if not options.get('disable_autoload'):
autodiscover_modules("tasks")
logger = logging.getLogger('huey')
config = ConsumerConfig(**consumer_options)
config.validate()
# Only configure the "huey" logger if it has no handlers. For example,
# some users may configure the huey logger via the Django global
# logging config. This prevents duplicating log messages:
if not logger.handlers:
config.setup_logger(logger)
consumer = HUEY.create_consumer(**config.values)
consumer.run()
def consumer_main():
parser_handler = OptionParserHandler()
parser = parser_handler.get_option_parser()
options, args = parser.parse_args()
if len(args) == 0:
err('Error: missing import path to `Huey` instance')
err('Example: huey_consumer.py app.queue.huey_instance')
sys.exit(1)
config = ConsumerConfig(**options.__dict__)
config.validate()
huey_instance = load_huey(args[0])
config.setup_logger()
consumer = Consumer(huey_instance, **config.values)
consumer.run()
def start_consumer(workers, worker_type, scheduler_interval, enable_verbose):
parser_handler = OptionParserHandler()
parser = parser_handler.get_option_parser()
options, args = parser.parse_args()
options = {k: v for k, v in options.__dict__.items() if v is not None}
options["workers"] = workers
options["worker-type"] = worker_type
options["scheduler-interval"] = scheduler_interval
if enable_verbose:
options["verbose"] = ""
config = ConsumerConfig(**options)
config.validate()
config.setup_logger()
consumer = huey_queue.create_consumer(**config.values)
consumer.run()
def consumer_main():
parser_handler = OptionParserHandler()
parser = parser_handler.get_option_parser()
options, args = parser.parse_args()
if len(args) == 0:
err('Error: missing import path to `Huey` instance')
err('Example: huey_consumer.py app.queue.huey_instance')
sys.exit(1)
options = {k: v for k, v in options.__dict__.items() if v is not None}
config = ConsumerConfig(**options)
config.validate()
huey_instance = load_huey(args[0])
# Set up logging for the "huey" namespace.
logger = logging.getLogger('huey')
config.setup_logger(logger)
consumer = huey_instance.create_consumer(**config.values)
consumer.run()
def assertInvalid(**kwargs):
cfg = ConsumerConfig(**kwargs)
self.assertRaises(ValueError, cfg.validate)