def test_from_pyfile(self):
"""Config is loaded from Python file"""
dirname = os.path.dirname(__file__)
path = os.path.join(dirname, 'config.py')
config = Config()
config.from_pyfile(path)
self.assertEqual(config.WORKER_MAX_LOAD, 20)
def test_from_pyfile(self):
"""Config is loaded from Python file"""
dirname = os.path.dirname(__file__)
path = os.path.join(dirname, 'config.py')
config = Config()
config.from_pyfile(path)
self.assertEqual(config.WORKER_MAX_LOAD, 20)
def test_from_env_vars(self):
"""Config is laoded from environment variables"""
os.environ['KUYRUK_WORKER_MAX_LOAD'] = '21'
try:
config = Config()
config.from_env_vars()
self.assertEqual(config.WORKER_MAX_LOAD, 21)
finally:
del os.environ['KUYRUK_WORKER_MAX_LOAD']
def test_from_env_vars(self):
"""Config is laoded from environment variables"""
os.environ['KUYRUK_WORKER_MAX_LOAD'] = '21'
try:
config = Config()
config.from_env_vars()
self.assertEqual(config.WORKER_MAX_LOAD, 21)
finally:
del os.environ['KUYRUK_WORKER_MAX_LOAD']
def __init__(self, config=None, task_class=Task):
self.task_class = task_class
self.config = Config()
self._connection = None
self._channel = None
if config:
self.config.from_object(config)
def __init__(self, config=None):
if config is None:
config = Config()
if not isinstance(config, Config):
raise TypeError
self.config = config
self.extensions = {}
def __init__(self, config=None, task_class=None):
self.config = Config()
self._connection = None
self._channel = None
if config:
self.config.from_object(config)
self.task_class = (importer.import_class_str(self.config.TASK_CLASS)
if task_class is None else task_class)
def create_config(args):
"""Creates Config object and overrides it's values from args."""
config = Config()
if args.config:
# Load config file from command line option
config.from_pyfile(args.config)
else:
# Load config file from environment variable
env_config = os.environ.get('KUYRUK_CONFIG')
if env_config:
assert os.path.isabs(env_config)
config.from_pyfile(env_config)
config.from_env_vars()
config.from_cmd_args(args)
return config
def test_from_object(self):
user_config.MAX_LOAD = 21
config = Config()
config.from_object(user_config)
self.assertEqual(config.MAX_LOAD, 21)
return 1 / 0
@kuyruk.task(retry=1)
def retry_task():
return 1 / 0
@kuyruk.task
def loop_forever():
while 1:
print('looping forever')
sleep(1)
config_eager = Config()
config_eager.EAGER = True
kuyruk_eager = Kuyruk(config_eager)
@kuyruk_eager.task()
def eager_task():
must_be_called()
@kuyruk.task
def rejecting_task():
raise kuyruk.Reject
@kuyruk.task(max_run_time=1)
def test_from_pymodule(self):
"""Config is loaded from Python module"""
config = Config()
config.from_pymodule('config2')
self.assertEqual(config.WORKER_MAX_LOAD, 20)
def test_from_config(self):
"""Config is loaded from dict"""
config = Config()
config.from_dict({'WORKER_MAX_LOAD': 21})
self.assertEqual(config.WORKER_MAX_LOAD, 21)
def run_scheduler(config):
from kuyruk.__main__ import run_scheduler
from kuyruk.config import Config
c = Config()
c.from_dict(config)
run_scheduler(Kuyruk(c), None)
def test_from_pymodule(self):
config = Config()
config.from_pymodule('config2')
self.assertEqual(config.WORKER_MAX_LOAD, 20)
def test_from_object(self):
"""Config is loaded from Python object"""
user_config.WORKER_MAX_LOAD = 21
config = Config()
config.from_object(user_config)
self.assertEqual(config.WORKER_MAX_LOAD, 21)
def __init__(self, config: Config = None) -> None:
if config is None:
config = Config()
self.config = config
self.extensions = {} # type: Dict[str, Any]
def test_from_pymodule(self):
"""Config is loaded from Python module"""
config = Config()
config.from_pymodule('config2')
self.assertEqual(config.WORKER_MAX_LOAD, 20)
return 1 / 0
@kuyruk.task(retry=1)
def retry_task():
return 1 / 0
@kuyruk.task
def loop_forever():
while 1:
print('looping forever')
sleep(1)
config_eager = Config()
config_eager.EAGER = True
kuyruk_eager = Kuyruk(config_eager)
@kuyruk_eager.task()
def eager_task():
must_be_called()
@kuyruk.task
def rejecting_task():
raise kuyruk.Reject
@kuyruk.task(max_run_time=1)
def sleeping_task(seconds):
sleep(seconds)
class Kuyruk(EventMixin):
"""
Main class for Kuyruk distributed task queue. It holds the configuration
values and provides a task decorator for user application
:param config: A module that contains configuration options.
See :ref:`configuration-options` for default values.
"""
Reject = exceptions.Reject # Shortcut for raising from tasks
def __init__(self, config=None, task_class=None):
self.config = Config()
self._connection = None
self._channel = None
if config:
self.config.from_object(config)
self.task_class = (importer.import_class_str(self.config.TASK_CLASS)
if task_class is None else task_class)
def task(self, queue='kuyruk', eager=False, retry=0, task_class=None,
max_run_time=None, local=False, arg_class=None):
"""
Wrap functions with this decorator to convert them to background
tasks. After wrapping, calling the function will send a message to
queue instead of running the function.
:param queue: Queue name for the tasks.
:param eager: Run task in process, do not use RabbitMQ.
:param retry: Retry this times before give up. Task will be re-routed
and executed on another worker.
:param task_class: Custom task class.
Must be a subclass of :class:`~Task`.
If this is :const:`None` then :attr:`Task.task_class` will be used.
:param max_run_time: Maximum allowed time in seconds for task to
complete.
:param arg_class: Class of the first argument. If it is present,
the first argument will be converted to it's ``id`` when sending
the task to the queue and it will be reloaded on worker when
running the task.
:return: Callable :class:`~Task` object wrapping the original function.
"""
def decorator():
def inner(f):
# Function may be wrapped with no-arg decorator
queue_ = 'kuyruk' if callable(queue) else queue
task_class_ = task_class or self.task_class
return task_class_(
f, self,
queue=queue_, eager=eager, local=local, retry=retry,
max_run_time=max_run_time, arg_class=arg_class)
return inner
if callable(queue):
logger.debug('task without args')
return decorator()(queue)
else:
logger.debug('task with args')
return decorator()
def connection(self):
"""
Returns the shared RabbitMQ connection.
Creates a new connection if it is not connected.
"""
if self._connection is None or not self._connection.is_open:
self._connection = self._connect()
return self._connection
def channel(self):
"""
Returns the shared channel.
Creates a new channel if there is no available.
"""
if self._channel is None or not self._channel.is_open:
self._channel = self._open_channel()
return self._channel
def close(self):
if self._connection is not None:
if self._connection.is_open:
self._connection.close()
def _connect(self):
"""Returns new connection object."""
parameters = pika.ConnectionParameters(
host=self.config.RABBIT_HOST,
port=self.config.RABBIT_PORT,
virtual_host=self.config.RABBIT_VIRTUAL_HOST,
credentials=pika.PlainCredentials(
self.config.RABBIT_USER,
self.config.RABBIT_PASSWORD),
heartbeat_interval=0, # We don't want heartbeats
socket_timeout=2,
connection_attempts=2)
connection = Connection(parameters)
logger.info('Connected to RabbitMQ')
return connection
def _open_channel(self):
"""Returns a new channel."""
CLOSED = (pika.exceptions.ConnectionClosed,
pika.exceptions.ChannelClosed)
try:
channel = self.connection().channel()
except CLOSED:
logger.warning("Connection is closed. Reconnecting...")
# If there is a connection, try to close it
if self._connection:
try:
self._connection.close()
except CLOSED:
pass
self._connection = self._connect()
channel = self._connection.channel()
return channel
def test_from_config(self):
"""Config is loaded from dict"""
config = Config()
config.from_dict({'WORKER_MAX_LOAD': 21})
self.assertEqual(config.WORKER_MAX_LOAD, 21)
def test_from_object(self):
"""Config is loaded from Python object"""
user_config.WORKER_MAX_LOAD = 21
config = Config()
config.from_object(user_config)
self.assertEqual(config.WORKER_MAX_LOAD, 21)
def test_from_object(self):
user_config.WORKER_MAX_LOAD = 21
config = Config()
config.from_object(user_config)
self.assertEqual(config.WORKER_MAX_LOAD, 21)
def test_from_pyfile(self):
dirname = os.path.dirname(__file__)
path = os.path.join(dirname, 'config.py')
config = Config()
config.from_pyfile(path)
self.assertEqual(config.MAX_LOAD, 20)
def test_from_pymodule(self):
config = Config()
config.from_pymodule('config2')
self.assertEqual(config.WORKER_MAX_LOAD, 20)
