Exemplo n.º 1
0
 def test_no_warning(self, mock_patcher):
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.return_value = True
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(0, len(capture))
 def test_no_warning(self, mock_patcher):
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.return_value = True
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(0, len(capture))
Exemplo n.º 3
0
    def __init__(self, transport, dispatcher, executor=None):
        """Construct a message handling server.

        The dispatcher parameter is a DispatcherBase instance which is used
        for routing request to endpoint for processing.

        The executor parameter controls how incoming messages will be received
        and dispatched. Executor is automatically detected from
        execution environment.
        It handles many message in parallel. If your application need
        asynchronism then you need to consider to use the eventlet executor.

        :param transport: the messaging transport
        :type transport: Transport
        :param dispatcher: has a dispatch() method which is invoked for each
                           incoming request
        :type dispatcher: DispatcherBase
        :param executor: name of message executor - available values are
                         'eventlet' and 'threading'
        :type executor: str
        """
        if executor and executor not in ("threading", "eventlet"):
            raise ExecutorLoadFailure(
                executor,
                "Executor should be None or 'eventlet' and 'threading'")
        if not executor:
            executor = utils.get_executor_with_context()

        self.conf = transport.conf
        self.conf.register_opts(_pool_opts)
        self.conf.register_opts(_metrics_opts)

        self.metrics = metrics.Metrics(self.conf.statsd_host,
                                       self.conf.statsd_port)
        self.transport = transport
        self.dispatcher = dispatcher
        self.executor_type = executor
        if self.executor_type == "eventlet":
            eventletutils.warn_eventlet_not_patched(
                expected_patched_modules=['thread'],
                what="the 'oslo.messaging eventlet executor'")

        self.listener = None

        try:
            mgr = driver.DriverManager('oslo.messaging.executors',
                                       self.executor_type)
        except RuntimeError as ex:
            raise ExecutorLoadFailure(self.executor_type, ex)

        self._executor_cls = mgr.driver

        self._work_executor = None

        self._started = False

        super(MessageHandlingServer, self).__init__()
 def test_warning_not_patched(self, mock_patcher):
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.return_value = False
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(1, len(capture))
     w = capture[0]
     self.assertEqual(RuntimeWarning, w.category)
     self.assertIn('os', six.text_type(w.message))
Exemplo n.º 5
0
 def test_warning_not_patched(self, mock_patcher):
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.return_value = False
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(1, len(capture))
     w = capture[0]
     self.assertEqual(RuntimeWarning, w.category)
     self.assertIn('os', six.text_type(w.message))
Exemplo n.º 6
0
    def start(self, override_pool_size=None):
        """Start handling incoming messages.

        This method causes the server to begin polling the transport for
        incoming messages and passing them to the dispatcher. Message
        processing will continue until the stop() method is called.

        The executor controls how the server integrates with the applications
        I/O handling strategy - it may choose to poll for messages in a new
        process, thread or co-operatively scheduled coroutine or simply by
        registering a callback with an event loop. Similarly, the executor may
        choose to dispatch messages in a new thread, coroutine or simply the
        current thread.
        """
        # Warn that restarting will be deprecated
        if self._started:
            LOG.warning(
                _LW('Restarting a MessageHandlingServer is inherently '
                    'racy. It is deprecated, and will become a noop '
                    'in a future release of oslo.messaging. If you '
                    'need to restart MessageHandlingServer you should '
                    'instantiate a new object.'))
        self._started = True

        executor_opts = {}

        if self.executor_type == "threading":
            executor_opts["max_workers"] = (
                override_pool_size or self.conf.executor_thread_pool_size)
        elif self.executor_type == "eventlet":
            eventletutils.warn_eventlet_not_patched(
                expected_patched_modules=['thread'],
                what="the 'oslo.messaging eventlet executor'")
            executor_opts["max_workers"] = (
                override_pool_size or self.conf.executor_thread_pool_size)

        self._work_executor = self._executor_cls(**executor_opts)

        try:
            self.listener = self._create_listener()
        except driver_base.TransportDriverError as ex:
            raise ServerListenError(self.target, ex)

        # HACK(sileht): We temporary pass the executor to the rabbit
        # listener to fix a race with the deprecated blocking executor.
        # We do this hack because this is need only for 'synchronous'
        # executor like blocking. And this one is deprecated. Making
        # driver working in an sync and an async way is complicated
        # and blocking have 0% tests coverage.
        if hasattr(self.listener, '_poll_style_listener'):
            l = self.listener._poll_style_listener
            if hasattr(l, "_message_operations_handler"):
                l._message_operations_handler._executor = (self.executor_type)

        self.listener.start(self._on_incoming)
Exemplo n.º 7
0
    def __init__(self, transport, dispatcher, executor='blocking'):
        """Construct a message handling server.

        The dispatcher parameter is a DispatcherBase instance which is used
        for routing request to endpoint for processing.

        The executor parameter controls how incoming messages will be received
        and dispatched. By default, the most simple executor is used - the
        blocking executor. It handles only one message at once. It's
        recommended to use threading or eventlet.

        :param transport: the messaging transport
        :type transport: Transport
        :param dispatcher: has a dispatch() method which is invoked for each
                           incoming request
        :type dispatcher: DispatcherBase
        :param executor: name of message executor - available values are
                         'eventlet', 'blocking' and 'threading'
        :type executor: str
        """
        self.conf = transport.conf
        self.conf.register_opts(_pool_opts)

        self.transport = transport
        self.dispatcher = dispatcher
        self.executor_type = executor
        if self.executor_type == 'blocking':
            # NOTE(sileht): We keep blocking as default to not enforce the
            # application to use threading or eventlet. Because application
            # have to be preprepared accordingly for each one (monkeypatching,
            # threadsafe, ...)
            LOG.info(_LI("blocking executor handles only one message at "
                         "once. threading or eventlet executor is "
                         "recommended."))
        elif self.executor_type == "eventlet":
            eventletutils.warn_eventlet_not_patched(
                expected_patched_modules=['thread'],
                what="the 'oslo.messaging eventlet executor'")

        self.listener = None

        try:
            mgr = driver.DriverManager('oslo.messaging.executors',
                                       self.executor_type)
        except RuntimeError as ex:
            raise ExecutorLoadFailure(self.executor_type, ex)

        self._executor_cls = mgr.driver

        self._work_executor = None

        self._started = False

        super(MessageHandlingServer, self).__init__()
Exemplo n.º 8
0
 def test_warning_not_patched_all(self, mock_patcher):
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.return_value = False
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['all'])
     self.assertEqual(1, len(capture))
     w = capture[0]
     self.assertEqual(RuntimeWarning, w.category)
     for m in eventletutils._ALL_PATCH:
         self.assertIn(m, str(w.message))
Exemplo n.º 9
0
    def __init__(self, transport, dispatcher, executor='blocking'):
        """Construct a message handling server.

        The dispatcher parameter is a DispatcherBase instance which is used
        for routing request to endpoint for processing.

        The executor parameter controls how incoming messages will be received
        and dispatched. By default, the most simple executor is used - the
        blocking executor. It handles only one message at once. It's
        recommended to use threading or eventlet.

        :param transport: the messaging transport
        :type transport: Transport
        :param dispatcher: has a dispatch() method which is invoked for each
                           incoming request
        :type dispatcher: DispatcherBase
        :param executor: name of message executor - available values are
                         'eventlet' and 'threading'
        :type executor: str
        """
        self.conf = transport.conf
        self.conf.register_opts(_pool_opts)

        self.transport = transport
        self.dispatcher = dispatcher
        self.executor_type = executor
        if self.executor_type == 'blocking':
            debtcollector.deprecate(
                'blocking executor is deprecated. Executor default will be '
                'removed. Use explicitly threading or eventlet instead',
                version="pike",
                removal_version="rocky",
                category=FutureWarning)
        elif self.executor_type == "eventlet":
            eventletutils.warn_eventlet_not_patched(
                expected_patched_modules=['thread'],
                what="the 'oslo.messaging eventlet executor'")

        self.listener = None

        try:
            mgr = driver.DriverManager(
                'oslo.messaging.executors',  #futurist:GreenThreadPoolExecutor  线程池
                self.executor_type)
        except RuntimeError as ex:
            raise ExecutorLoadFailure(self.executor_type, ex)

        self._executor_cls = mgr.driver

        self._work_executor = None

        self._started = False

        super(MessageHandlingServer, self).__init__()
Exemplo n.º 10
0
    def __init__(self, transport, dispatcher, executor='blocking'):
        """Construct a message handling server.

        The dispatcher parameter is a DispatcherBase instance which is used
        for routing request to endpoint for processing.

        The executor parameter controls how incoming messages will be received
        and dispatched. By default, the most simple executor is used - the
        blocking executor. It handles only one message at once. It's
        recommended to use threading or eventlet.

        :param transport: the messaging transport
        :type transport: Transport
        :param dispatcher: has a dispatch() method which is invoked for each
                           incoming request
        :type dispatcher: DispatcherBase
        :param executor: name of message executor - available values are
                         'eventlet' and 'threading'
        :type executor: str
        """
        self.conf = transport.conf
        self.conf.register_opts(_pool_opts)

        self.transport = transport
        self.dispatcher = dispatcher
        self.executor_type = executor
        if self.executor_type == 'blocking':
            debtcollector.deprecate(
                'blocking executor is deprecated. Executor default will be '
                'removed. Use explicitly threading or eventlet instead',
                version="pike", removal_version="rocky",
                category=FutureWarning)
        elif self.executor_type == "eventlet":
            eventletutils.warn_eventlet_not_patched(
                expected_patched_modules=['thread'],
                what="the 'oslo.messaging eventlet executor'")

        self.listener = None

        try:
            mgr = driver.DriverManager('oslo.messaging.executors',
                                       self.executor_type)
        except RuntimeError as ex:
            raise ExecutorLoadFailure(self.executor_type, ex)

        self._executor_cls = mgr.driver

        self._work_executor = None

        self._started = False

        super(MessageHandlingServer, self).__init__()
Exemplo n.º 11
0
 def test_partially_patched_warning(self, mock_patcher):
     is_patched = set()
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.side_effect = lambda m: m in is_patched
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(1, len(capture))
     is_patched.add('os')
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(0, len(capture))
     is_patched.add('thread')
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os', 'thread'])
     self.assertEqual(0, len(capture))
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['all'])
     self.assertEqual(1, len(capture))
     w = capture[0]
     self.assertEqual(RuntimeWarning, w.category)
     for m in ['os', 'thread']:
         self.assertNotIn(m, six.text_type(w.message))
Exemplo n.º 12
0
 def test_partially_patched_warning(self, mock_patcher):
     is_patched = set()
     mock_patcher.already_patched = True
     mock_patcher.is_monkey_patched.side_effect = lambda m: m in is_patched
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(1, len(capture))
     is_patched.add('os')
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os'])
     self.assertEqual(0, len(capture))
     is_patched.add('thread')
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['os', 'thread'])
     self.assertEqual(0, len(capture))
     with warnings.catch_warnings(record=True) as capture:
         warnings.simplefilter("always")
         eventletutils.warn_eventlet_not_patched(['all'])
     self.assertEqual(1, len(capture))
     w = capture[0]
     self.assertEqual(RuntimeWarning, w.category)
     for m in ['os', 'thread']:
         self.assertNotIn(m, six.text_type(w.message))
Exemplo n.º 13
0
    def start(self, override_pool_size=None):
        """Start handling incoming messages.

        This method causes the server to begin polling the transport for
        incoming messages and passing them to the dispatcher. Message
        processing will continue until the stop() method is called.

        The executor controls how the server integrates with the applications
        I/O handling strategy - it may choose to poll for messages in a new
        process, thread or co-operatively scheduled coroutine or simply by
        registering a callback with an event loop. Similarly, the executor may
        choose to dispatch messages in a new thread, coroutine or simply the
        current thread.
        """
        # Warn that restarting will be deprecated
        if self._started:
            LOG.warning(_LW('Restarting a MessageHandlingServer is inherently '
                            'racy. It is deprecated, and will become a noop '
                            'in a future release of oslo.messaging. If you '
                            'need to restart MessageHandlingServer you should '
                            'instantiate a new object.'))
        self._started = True

        try:
            self.listener = self.dispatcher._listen(self.transport)
        except driver_base.TransportDriverError as ex:
            raise ServerListenError(self.target, ex)

        executor_opts = {}

        if self.executor_type == "threading":
            executor_opts["max_workers"] = (
                override_pool_size or self.conf.executor_thread_pool_size
            )
        elif self.executor_type == "eventlet":
            eventletutils.warn_eventlet_not_patched(
                expected_patched_modules=['thread'],
                what="the 'oslo.messaging eventlet executor'")
            executor_opts["max_workers"] = (
                override_pool_size or self.conf.executor_thread_pool_size
            )

        self._work_executor = self._executor_cls(**executor_opts)
        self._poll_executor = self._executor_cls(**executor_opts)

        return lambda: self._poll_executor.submit(self._runner)
Exemplo n.º 14
0
#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_utils import eventletutils as _eventletutils

# Give a nice warning that if eventlet is being used these modules
# are highly recommended to be patched (or otherwise bad things could
# happen).
_eventletutils.warn_eventlet_not_patched(
    expected_patched_modules=['time', 'thread'])


# Promote helpers to this module namespace (for easy access).
from taskflow.engines.helpers import flow_from_detail   # noqa
from taskflow.engines.helpers import load               # noqa
from taskflow.engines.helpers import load_from_detail   # noqa
from taskflow.engines.helpers import load_from_factory  # noqa
from taskflow.engines.helpers import run                # noqa
from taskflow.engines.helpers import save_factory_details  # noqa
Exemplo n.º 15
0
#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_utils import eventletutils as _eventletutils

# Give a nice warning that if eventlet is being used these modules
# are highly recommended to be patched (or otherwise bad things could
# happen).
_eventletutils.warn_eventlet_not_patched(
    expected_patched_modules=['time', 'thread'])

# Promote helpers to this module namespace (for easy access).
from zag.engines.helpers import flow_from_detail  # noqa
from zag.engines.helpers import load  # noqa
from zag.engines.helpers import load_from_detail  # noqa
from zag.engines.helpers import load_from_factory  # noqa
from zag.engines.helpers import run  # noqa
from zag.engines.helpers import save_factory_details  # noqa
Exemplo n.º 16
0
'''
monkey-patch
dynamically replace modules

green thread
http://eventlet.net/doc/modules/greenthread.html

Or
如果使用线程做过重要的编程,你就知道写出程序有多么困难,因为调度程序任何时候都能中断线程。
必须记住保留锁,去保护程序中的重要部分,防止多步操作在执行的过程中中断,防止数据处于无效状态。
而协程默认会做好全方位保护,以防止中断。我们必须显式产出才能让程序的余下部分运行。
对协程来说,无需保留锁,在多个线程之间同步操作,协程自身就会同步,因为在任意时刻只有一个协程运行。
想交出控制权时,可以使用 yield 或 yield from 把控制权交还调度程序。
这就是能够安全地取消协程的原因:按照定义,协程只能在暂停的 yield处取消,
因此可以处理 CancelledError 异常,执行清理操作。

总而言之,协程比线程更节省资源,效率更高,并且更安全。

'''

from oslo_utils import eventletutils

print(locals())
print(globals())

print(eventletutils.fetch_current_thread_functor())

print(eventletutils.is_monkey_patched('sys'))

eventletutils.warn_eventlet_not_patched(expected_patched_modules=None, what='this library')
Exemplo n.º 17
0
 def __init__(self, conf, listener, dispatcher):
     super(EventletExecutor, self).__init__(conf, listener, dispatcher)
     eventletutils.warn_eventlet_not_patched(
         expected_patched_modules=['thread'],
         what="the 'oslo.messaging eventlet executor'")