Exemple #1
0
 def test_from_pyfile(self):
     """Config is loaded from Python file"""
     dirname = os.path.dirname(__file__)
     path = os.path.join(dirname, 'config.py')
     config = Config()
     config.from_pyfile(path)
     self.assertEqual(config.WORKER_MAX_LOAD, 20)
Exemple #2
0
 def test_from_pyfile(self):
     """Config is loaded from Python file"""
     dirname = os.path.dirname(__file__)
     path = os.path.join(dirname, 'config.py')
     config = Config()
     config.from_pyfile(path)
     self.assertEqual(config.WORKER_MAX_LOAD, 20)
Exemple #3
0
 def test_from_env_vars(self):
     """Config is laoded from environment variables"""
     os.environ['KUYRUK_WORKER_MAX_LOAD'] = '21'
     try:
         config = Config()
         config.from_env_vars()
         self.assertEqual(config.WORKER_MAX_LOAD, 21)
     finally:
         del os.environ['KUYRUK_WORKER_MAX_LOAD']
Exemple #4
0
 def test_from_env_vars(self):
     """Config is laoded from environment variables"""
     os.environ['KUYRUK_WORKER_MAX_LOAD'] = '21'
     try:
         config = Config()
         config.from_env_vars()
         self.assertEqual(config.WORKER_MAX_LOAD, 21)
     finally:
         del os.environ['KUYRUK_WORKER_MAX_LOAD']
Exemple #5
0
 def __init__(self, config=None, task_class=Task):
     self.task_class = task_class
     self.config = Config()
     self._connection = None
     self._channel = None
     if config:
         self.config.from_object(config)
Exemple #6
0
 def __init__(self, config=None):
     if config is None:
         config = Config()
     if not isinstance(config, Config):
         raise TypeError
     self.config = config
     self.extensions = {}
Exemple #7
0
 def __init__(self, config=None, task_class=None):
     self.config = Config()
     self._connection = None
     self._channel = None
     if config:
         self.config.from_object(config)
     self.task_class = (importer.import_class_str(self.config.TASK_CLASS)
                        if task_class is None else task_class)
Exemple #8
0
def create_config(args):
    """Creates Config object and overrides it's values from args."""
    config = Config()

    if args.config:
        # Load config file from command line option
        config.from_pyfile(args.config)
    else:
        # Load config file from environment variable
        env_config = os.environ.get('KUYRUK_CONFIG')
        if env_config:
            assert os.path.isabs(env_config)
            config.from_pyfile(env_config)

    config.from_env_vars()
    config.from_cmd_args(args)
    return config
Exemple #9
0
 def test_from_object(self):
     user_config.MAX_LOAD = 21
     config = Config()
     config.from_object(user_config)
     self.assertEqual(config.MAX_LOAD, 21)
Exemple #10
0
    return 1 / 0


@kuyruk.task(retry=1)
def retry_task():
    return 1 / 0


@kuyruk.task
def loop_forever():
    while 1:
        print('looping forever')
        sleep(1)


config_eager = Config()
config_eager.EAGER = True
kuyruk_eager = Kuyruk(config_eager)


@kuyruk_eager.task()
def eager_task():
    must_be_called()


@kuyruk.task
def rejecting_task():
    raise kuyruk.Reject


@kuyruk.task(max_run_time=1)
Exemple #11
0
 def test_from_pymodule(self):
     """Config is loaded from Python module"""
     config = Config()
     config.from_pymodule('config2')
     self.assertEqual(config.WORKER_MAX_LOAD, 20)
Exemple #12
0
 def test_from_config(self):
     """Config is loaded from dict"""
     config = Config()
     config.from_dict({'WORKER_MAX_LOAD': 21})
     self.assertEqual(config.WORKER_MAX_LOAD, 21)
Exemple #13
0
def run_scheduler(config):
    from kuyruk.__main__ import run_scheduler
    from kuyruk.config import Config
    c = Config()
    c.from_dict(config)
    run_scheduler(Kuyruk(c), None)
Exemple #14
0
 def test_from_pymodule(self):
     config = Config()
     config.from_pymodule('config2')
     self.assertEqual(config.WORKER_MAX_LOAD, 20)
Exemple #15
0
 def test_from_object(self):
     """Config is loaded from Python object"""
     user_config.WORKER_MAX_LOAD = 21
     config = Config()
     config.from_object(user_config)
     self.assertEqual(config.WORKER_MAX_LOAD, 21)
Exemple #16
0
    def __init__(self, config: Config = None) -> None:
        if config is None:
            config = Config()

        self.config = config
        self.extensions = {}  # type: Dict[str, Any]
Exemple #17
0
 def test_from_pymodule(self):
     """Config is loaded from Python module"""
     config = Config()
     config.from_pymodule('config2')
     self.assertEqual(config.WORKER_MAX_LOAD, 20)
Exemple #18
0
    return 1 / 0


@kuyruk.task(retry=1)
def retry_task():
    return 1 / 0


@kuyruk.task
def loop_forever():
    while 1:
        print('looping forever')
        sleep(1)


config_eager = Config()
config_eager.EAGER = True
kuyruk_eager = Kuyruk(config_eager)
@kuyruk_eager.task()
def eager_task():
    must_be_called()


@kuyruk.task
def rejecting_task():
    raise kuyruk.Reject


@kuyruk.task(max_run_time=1)
def sleeping_task(seconds):
    sleep(seconds)
Exemple #19
0
class Kuyruk(EventMixin):
    """
    Main class for Kuyruk distributed task queue. It holds the configuration
    values and provides a task decorator for user application

    :param config: A module that contains configuration options.
                   See :ref:`configuration-options` for default values.

    """
    Reject = exceptions.Reject  # Shortcut for raising from tasks

    def __init__(self, config=None, task_class=None):
        self.config = Config()
        self._connection = None
        self._channel = None
        if config:
            self.config.from_object(config)
        self.task_class = (importer.import_class_str(self.config.TASK_CLASS)
                           if task_class is None else task_class)

    def task(self, queue='kuyruk', eager=False, retry=0, task_class=None,
             max_run_time=None, local=False, arg_class=None):
        """
        Wrap functions with this decorator to convert them to background
        tasks. After wrapping, calling the function will send a message to
        queue instead of running the function.

        :param queue: Queue name for the tasks.
        :param eager: Run task in process, do not use RabbitMQ.
        :param retry: Retry this times before give up. Task will be re-routed
            and executed on another worker.
        :param task_class: Custom task class.
            Must be a subclass of :class:`~Task`.
            If this is :const:`None` then :attr:`Task.task_class` will be used.
        :param max_run_time: Maximum allowed time in seconds for task to
            complete.
        :param arg_class: Class of the first argument. If it is present,
            the first argument will be converted to it's ``id`` when sending
            the task to the queue and it will be reloaded on worker when
            running the task.
        :return: Callable :class:`~Task` object wrapping the original function.

        """
        def decorator():
            def inner(f):
                # Function may be wrapped with no-arg decorator
                queue_ = 'kuyruk' if callable(queue) else queue

                task_class_ = task_class or self.task_class
                return task_class_(
                    f, self,
                    queue=queue_, eager=eager, local=local, retry=retry,
                    max_run_time=max_run_time, arg_class=arg_class)
            return inner

        if callable(queue):
            logger.debug('task without args')
            return decorator()(queue)
        else:
            logger.debug('task with args')
            return decorator()

    def connection(self):
        """
        Returns the shared RabbitMQ connection.
        Creates a new connection if it is not connected.

        """
        if self._connection is None or not self._connection.is_open:
            self._connection = self._connect()

        return self._connection

    def channel(self):
        """
        Returns the shared channel.
        Creates a new channel if there is no available.

        """
        if self._channel is None or not self._channel.is_open:
            self._channel = self._open_channel()

        return self._channel

    def close(self):
        if self._connection is not None:
            if self._connection.is_open:
                self._connection.close()

    def _connect(self):
        """Returns new connection object."""
        parameters = pika.ConnectionParameters(
            host=self.config.RABBIT_HOST,
            port=self.config.RABBIT_PORT,
            virtual_host=self.config.RABBIT_VIRTUAL_HOST,
            credentials=pika.PlainCredentials(
                self.config.RABBIT_USER,
                self.config.RABBIT_PASSWORD),
            heartbeat_interval=0,  # We don't want heartbeats
            socket_timeout=2,
            connection_attempts=2)
        connection = Connection(parameters)
        logger.info('Connected to RabbitMQ')
        return connection

    def _open_channel(self):
        """Returns a new channel."""
        CLOSED = (pika.exceptions.ConnectionClosed,
                  pika.exceptions.ChannelClosed)

        try:
            channel = self.connection().channel()
        except CLOSED:
            logger.warning("Connection is closed. Reconnecting...")

            # If there is a connection, try to close it
            if self._connection:
                try:
                    self._connection.close()
                except CLOSED:
                    pass

            self._connection = self._connect()
            channel = self._connection.channel()

        return channel
Exemple #20
0
 def test_from_config(self):
     """Config is loaded from dict"""
     config = Config()
     config.from_dict({'WORKER_MAX_LOAD': 21})
     self.assertEqual(config.WORKER_MAX_LOAD, 21)
Exemple #21
0
 def test_from_object(self):
     """Config is loaded from Python object"""
     user_config.WORKER_MAX_LOAD = 21
     config = Config()
     config.from_object(user_config)
     self.assertEqual(config.WORKER_MAX_LOAD, 21)
Exemple #22
0
 def test_from_object(self):
     user_config.WORKER_MAX_LOAD = 21
     config = Config()
     config.from_object(user_config)
     self.assertEqual(config.WORKER_MAX_LOAD, 21)
Exemple #23
0
 def test_from_pyfile(self):
     dirname = os.path.dirname(__file__)
     path = os.path.join(dirname, 'config.py')
     config = Config()
     config.from_pyfile(path)
     self.assertEqual(config.MAX_LOAD, 20)
Exemple #24
0
 def test_from_pymodule(self):
     config = Config()
     config.from_pymodule('config2')
     self.assertEqual(config.WORKER_MAX_LOAD, 20)