def handle(self, *args, **options):
from huey.djhuey import HUEY
try:
consumer_options = settings.HUEY['consumer_options']
except:
consumer_options = {}
if options['workers'] is not None:
consumer_options['workers'] = options['workers']
if options['periodic'] is not None:
consumer_options['periodic'] = options['periodic']
if options['initial_delay'] is not None:
consumer_options['initial_delay'] = options['initial_delay']
if options['max_delay'] is not None:
consumer_options['max_delay'] = options['max_delay']
self.autodiscover()
loglevel = get_loglevel(consumer_options.pop('loglevel', None))
logfile = consumer_options.pop('logfile', None)
setup_logger(loglevel, logfile)
consumer = Consumer(HUEY, **consumer_options)
consumer.run()
def handle(self, *args, **options):
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
autodiscover_modules("tasks")
huey_queue = settings.HUEY_QUEUES[options["queue"]]
#del options["queue"]
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
consumer = Consumer(huey_queue, **config.values)
logger.info("The following tasks are available for this queue:")
for command in consumer.huey.registry._registry:
logger.info(command.replace('queue_task_', ''))
consumer.run()
def handle(self, *args, **options):
# добавлено
if settings.MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION:
raise CommandError('If you want to use queues, set MESTO_TASK_QUEUES_SYNCRONOUS_EXECUTION setting to False')
# from huey.contrib.djhuey import HUEY
# добавлено
queue = options.pop('queue')
consumer_options = {}
# if isinstance(settings.HUEY, dict):
# consumer_options.update(settings.HUEY.get('consumer', {}))
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
self.autodiscover()
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
# consumer = Consumer(HUEY, **config.values)
consumer = Consumer(HUEYS[queue], **config.values)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
if not options.get('disable_autoload'):
autodiscover_modules("tasks")
logger = logging.getLogger('huey')
config = ConsumerConfig(**consumer_options)
config.validate()
# Only configure the "huey" logger if it has no handlers. For example,
# some users may configure the huey logger via the Django global
# logging config. This prevents duplicating log messages:
if not logger.handlers:
config.setup_logger(logger)
consumer = Consumer(HUEY, **config.values)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
if options['workers'] is not None:
consumer_options['workers'] = options['workers']
if options['worker_type'] is not None:
consumer_options['worker_type'] = options['worker_type']
if options['periodic'] is not None:
consumer_options['periodic'] = options['periodic']
if options['initial_delay'] is not None:
consumer_options['initial_delay'] = options['initial_delay']
if options['max_delay'] is not None:
consumer_options['max_delay'] = options['max_delay']
self.autodiscover()
loglevel = get_loglevel(consumer_options.pop('loglevel', None))
logfile = consumer_options.pop('logfile', None)
setup_logger(loglevel, logfile, consumer_options['worker_type'])
consumer = Consumer(HUEY, **consumer_options)
consumer.run()
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
autodiscover_modules("tasks")
config = ConsumerConfig(**consumer_options)
config.validate()
config.setup_logger()
consumer = Consumer(HUEY, **config.values)
consumer.run()
def test_consumer_integration(self):
lock = threading.Lock()
@self.huey.task()
def add_values(a, b):
return a + b
consumer = Consumer(self.huey, max_delay=0.1, workers=2,
worker_type='thread', health_check_interval=0.01)
with CaptureLogs() as capture:
consumer.start()
try:
r1 = add_values(1, 2)
r2 = add_values(2, 3)
r3 = add_values(3, 5)
self.assertEqual(r1.get(blocking=True, timeout=3), 3)
self.assertEqual(r2.get(blocking=True, timeout=3), 5)
self.assertEqual(r3.get(blocking=True, timeout=3), 8)
finally:
consumer.stop()
for _, worker in consumer.worker_threads:
worker.join()
messages = capture.messages[-4:-1]
for message in messages:
self.assertTrue(message.startswith('Executing huey.tests.test_'))
self.assertTrue(capture.messages[-1].startswith('Shutting down'))
def consumer_main():
parser = get_option_parser()
options, args = parser.parse_args()
setup_logger(get_loglevel(options.verbose), options.logfile,
options.worker_type)
if len(args) == 0:
err('Error: missing import path to `Huey` instance')
err('Example: huey_consumer.py app.queue.huey_instance')
sys.exit(1)
if options.workers < 1:
err('You must have at least one worker.')
sys.exit(1)
if options.list_registered:
list_registered()
huey_instance = load_huey(args[0])
consumer = Consumer(huey_instance, options.workers, options.periodic,
options.initial_delay, options.backoff,
options.max_delay, options.utc,
options.scheduler_interval, options.worker_type)
consumer.run()
def consumer_main():
parser_handler = OptionParserHandler()
parser = parser_handler.get_option_parser()
options, args = parser.parse_args()
if len(args) == 0:
err('Error: missing import path to `Huey` instance')
err('Example: huey_consumer.py app.queue.huey_instance')
sys.exit(1)
config = ConsumerConfig(**options.__dict__)
config.validate()
huey_instance = load_huey(args[0])
config.setup_logger()
consumer = Consumer(huey_instance, **config.values)
consumer.run()
def test_dequeue_errors(self):
huey = BrokenHuey()
consumer = Consumer(huey, max_delay=0.1, workers=2,
worker_type='thread')
worker = consumer._create_worker()
state = {}
@huey.task()
def modify_broken(k, v):
state[k] = v
with CaptureLogs() as capture:
res = modify_broken('k', 'v')
worker.loop()
self.assertEqual(capture.messages, ['Error reading from queue'])
self.assertEqual(state, {})
def setUp(self):
global state
state = {}
self.orig_pc = registry._periodic_tasks
registry._periodic_commands = [every_hour.task_class()]
self.orig_sleep = time.sleep
time.sleep = lambda x: None
test_huey.queue.flush()
test_huey.result_store.flush()
test_huey.schedule.flush()
test_events._events = deque()
self.consumer = Consumer(test_huey, workers=2)
self.consumer._create_threads()
self.handler = TestLogHandler()
logger.addHandler(self.handler)
logger.setLevel(logging.INFO)
def handle(self, *args, **options):
from huey.contrib.djhuey import HUEY
# Python 3.8+ on MacOS uses an incompatible multiprocess model. In this
# case we must explicitly configure mp to use fork().
if sys.version_info >= (3, 8) and sys.platform == 'darwin':
import multiprocessing
multiprocessing.set_start_method('fork')
consumer_options = {}
try:
if isinstance(settings.HUEY, dict):
consumer_options.update(settings.HUEY.get('consumer', {}))
except AttributeError:
pass
for key, value in options.items():
if value is not None:
consumer_options[key] = value
consumer_options.setdefault('verbose',
consumer_options.pop('huey_verbose', None))
if not options.get('disable_autoload'):
autodiscover_modules("tasks")
logger = logging.getLogger('huey')
config = ConsumerConfig(**consumer_options)
config.validate()
# Only configure the "huey" logger if it has no handlers. For example,
# some users may configure the huey logger via the Django global
# logging config. This prevents duplicating log messages:
if not logger.handlers:
config.setup_logger(logger)
consumer = Consumer(HUEY, **config.values)
consumer.run()
def create_consumer(self, worker_type='thread'):
return Consumer(self.huey,
max_delay=0.1,
workers=2,
worker_type=worker_type)
def get_consumer(self, **kwargs):
return Consumer(self.huey, **kwargs)
def create_consumer(self, **options):
return Consumer(self, **options)
def _run_consumer():
consumer = Consumer(huey)
consumer._logger.addHandler(logging.FileHandler(DEFAULT_LOG_PATH))
consumer._logger.addHandler(logging.StreamHandler(sys.stdout))
consumer.run()
def create_consumer(self, **config):
return Consumer(self, **config)
parser.add_option('--localtime',
dest='utc',
action='store_false',
help='use local time for all tasks')
return parser
if __name__ == '__main__':
parser = get_option_parser()
options, args = parser.parse_args()
setup_logger(get_loglevel(options.verbose), options.logfile)
if len(args) == 0:
err('Error: missing import path to `Huey` instance')
err('Example: huey_consumer.py app.queue.huey_instance')
sys.exit(1)
try:
huey_instance = load_class(args[0])
except:
err('Error importing %s' % args[0])
raise
consumer = Consumer(huey_instance, options.workers, options.periodic,
options.initial_delay, options.backoff,
options.max_delay, options.utc,
options.scheduler_interval,
options.periodic_task_interval)
consumer.run()
