def test_unknown_executor(self):
transport = messaging.get_transport(self.conf, url='fake:')
try:
messaging.get_rpc_server(transport, None, [], executor='foo')
except Exception as ex:
self.assertIsInstance(ex, messaging.ExecutorLoadFailure)
self.assertEqual('foo', ex.executor)
else:
self.assertTrue(False)
def test_unknown_executor(self):
transport = messaging.get_transport(self.conf, url='fake:')
try:
messaging.get_rpc_server(transport, None, [], executor='foo')
except Exception as ex:
self.assertIsInstance(ex, messaging.ExecutorLoadFailure)
self.assertEqual('foo', ex.executor)
else:
self.assertTrue(False)
def start(self):
"""Bind the UDP socket and handle incoming data."""
# ensure dispatcher is configured before starting other services
self.dispatcher_manager = dispatcher.load_dispatcher_manager()
self.rpc_server = None
self.notification_server = None
super(CollectorService, self).start()
if cfg.CONF.collector.udp_address:
self.tg.add_thread(self.start_udp)
allow_requeue = cfg.CONF.collector.requeue_sample_on_dispatcher_error
transport = messaging.get_transport(optional=True)
if transport:
self.rpc_server = messaging.get_rpc_server(
transport, cfg.CONF.publisher_rpc.metering_topic, self)
target = oslo.messaging.Target(
topic=cfg.CONF.publisher_notifier.metering_topic)
self.notification_server = messaging.get_notification_listener(
transport, [target], [self],
allow_requeue=allow_requeue)
self.rpc_server.start()
self.notification_server.start()
if not cfg.CONF.collector.udp_address:
# Add a dummy thread to have wait() working
self.tg.add_timer(604800, lambda: None)
def __init__(self, target):
self.__server = messaging.get_rpc_server(
target=target,
transport=messaging.get_transport(cfg.CONF),
endpoints=[ContextEndpointHandler(self)],
executor='eventlet'
)
def setUp(self):
super(RpcServerFixture, self).setUp()
endpoints = [self.endpoint, self]
self.server = messaging.get_rpc_server(self.transport, self.target,
endpoints)
self._ctrl = messaging.RPCClient(self.transport, self.ctrl_target)
self._start()
def main():
try:
config.parse_args()
logging.setup('Mistral')
# TODO(rakhmerov): This is a temporary hack.
# We have to initialize engine in executor process because
# executor now calls engine.convey_task_result() directly.
engine.load_engine()
# Please refer to the oslo.messaging documentation for transport
# configuration. The default transport for oslo.messaging is rabbitMQ.
# The available transport drivers are listed under oslo.messaging at
# ./oslo/messaging/rpc/_drivers. The drivers are prefixed with "impl".
# The transport driver is specified using the rpc_backend option in the
# default section of the oslo configuration file. The expected value
# for rpc_backend is the last part of the driver name. For example,
# the driver for rabbit is impl_rabbit and for the fake driver is
# impl_fake. The rpc_backend value for these are "rabbit" and "fake"
# respectively. There are additional options such as ssl and credential
# that can be specified depending on the driver. Please refer to the
# driver implementation for those additional options.
transport = messaging.get_transport(cfg.CONF)
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
endpoints = [server.Executor()]
ex_server = messaging.get_rpc_server(transport, target, endpoints)
ex_server.start()
ex_server.wait()
except RuntimeError, e:
sys.stderr.write("ERROR: %s\n" % e)
sys.exit(1)
def __init__(self, target):
super(RPCServer, self).__init__()
self._server = messaging.get_rpc_server(
target=target,
transport=messaging.get_transport(cfg.CONF),
endpoints=[ContextEndpointHandler(self, target)],
)
def launch_agent(transport):
print "launching agent"
target = messaging.Target(
topic=cfg.CONF.agent.topic,
server=cfg.CONF.agent.host
)
notifier = messaging.Notifier(transport, cfg.CONF.monitor.host, driver='messaging',
topic=cfg.CONF.monitor.notifications)
agent_engine = def_agent.DefaultAgent(rpc.get_monitorclient(), notifier)
agent_endpoints = [rpc.AgentServer(agent_engine)]
agent_server = messaging.get_rpc_server(
transport,
target,
agent_endpoints,
executor='eventlet'
)
try:
agent_server.start()
while True:
time.sleep(0)
except KeyboardInterrupt:
print ("Stopping server")
agent_server.stop()
agent_server.wait()
def launch_monitor(transport):
print "launching monitor"
target = messaging.Target(
topic=cfg.CONF.monitor.topic,
server=cfg.CONF.monitor.host
)
notifier = messaging.Notifier(transport, cfg.CONF.agent.host, driver='messaging',
topic=cfg.CONF.agent.notifications)
monitor_engine = def_monitor.DefaultMonitor(rpc.get_agentclient(), notifier)
monitor_endpoints = [rpc.MonitorServer(monitor_engine)]
monitor_server = messaging.get_rpc_server(
transport,
target,
monitor_endpoints,
executor='eventlet'
)
try:
monitor_server.start()
while True:
time.sleep(1)
except KeyboardInterrupt:
print ("Stopping server")
monitor_server.stop()
monitor_server.wait()
def __init__(self,
host,
pipeline_driver=None,
topic=None,
allow_stop=False):
topic = topic or 'process'
self.host = host
self.topic = 'gate.%s' % topic
self._pipeline_driver = pipeline_driver
if not self._pipeline_driver:
self._pipeline_driver = get_pipeline_driver()
self._transport = messaging.get_transport(cfg.CONF)
self._target = messaging.Target(topic=self.topic, server=self.host)
self._engine_client = EngineClient(self._transport)
self._endpoints = [ProcessAPI(self, engine_client=self._engine_client)]
if allow_stop:
self._endpoints.append(self)
self._server = messaging.get_rpc_server(self._transport, self._target,
self._endpoints)
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor='eventlet',
serializer=serializer)
def __init__(self, topic, endpoints):
self.hostname = socket.gethostname()
self.topic = topic
self.transport = messaging.get_transport(cfg.CONF)
self.target = messaging.Target(topic=self.topic, server=self.hostname, version=self.version)
self.endpoints = endpoints
self.server = messaging.get_rpc_server(self.transport, self.target, self.endpoints)
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor='eventlet',
serializer=serializer)
def test_published_no_mock(self):
publisher = msg_publisher.RPCPublisher(
netutils.urlsplit('rpc://'))
endpoint = mock.MagicMock(['record_metering_data'])
collector = messaging.get_rpc_server(
self.transport, self.CONF.publisher_rpc.metering_topic, endpoint)
endpoint.record_metering_data.side_effect = (lambda *args, **kwds:
collector.stop())
collector.start()
eventlet.sleep()
publisher.publish_samples(context.RequestContext(),
self.test_sample_data)
collector.wait()
class Matcher(object):
@staticmethod
def __eq__(data):
for i, sample_item in enumerate(data):
if (sample_item['counter_name'] !=
self.test_sample_data[i].name):
return False
return True
endpoint.record_metering_data.assert_called_once_with(
mock.ANY, data=Matcher())
def test_published_no_mock(self):
publisher = msg_publisher.RPCPublisher(netutils.urlsplit('rpc://'))
endpoint = mock.MagicMock(['record_metering_data'])
collector = messaging.get_rpc_server(
self.transport, self.CONF.publisher_rpc.metering_topic, endpoint)
endpoint.record_metering_data.side_effect = (
lambda *args, **kwds: collector.stop())
collector.start()
eventlet.sleep()
publisher.publish_samples(context.RequestContext(),
self.test_sample_data)
collector.wait()
class Matcher(object):
@staticmethod
def __eq__(data):
for i, sample_item in enumerate(data):
if (sample_item['counter_name'] !=
self.test_sample_data[i].name):
return False
return True
endpoint.record_metering_data.assert_called_once_with(mock.ANY,
data=Matcher())
def __init__(self, target):
super(RPCServer, self).__init__()
self._server = messaging.get_rpc_server(
target=target,
transport=messaging.get_transport(cfg.CONF),
endpoints=[ContextEndpointHandler(self, target)],
)
def setUp(self):
super(RpcServerFixture, self).setUp()
endpoints = [self.endpoint, self]
self.server = messaging.get_rpc_server(self.transport,
self.target,
endpoints)
self._ctrl = messaging.RPCClient(self.transport, self.ctrl_target)
self._start()
def launch_executor(transport):
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
endpoints = [server.Executor(transport)]
ex_server = messaging.get_rpc_server(
transport, target, endpoints, executor='eventlet')
ex_server.start()
ex_server.wait()
def __init__(self, transport):
self.target = messaging.Target(topic='testtopic',
server=transport.conf.host,
version='2.5')
self._server = messaging.get_rpc_server(transport,
self.target,
[self])
super(Server, self).__init__()
def get_server(target, endpoints, serializer = None):
assert TRANSPORT is not None
#serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor = 'blocking',
serializer = serializer)
def launch_engine(transport):
target = messaging.Target(topic=cfg.CONF.engine.topic,
server=cfg.CONF.engine.host)
endpoints = [engine.get_engine(cfg.CONF.engine.engine, transport)]
server = messaging.get_rpc_server(
transport, target, endpoints, executor='eventlet')
server.start()
server.wait()
def server():
#eventlet.monkey_patch()
target = messaging.Target(topic='testtopic', server='server1', version='1.0')
server = messaging.get_rpc_server(TRANSPORT, target, [TestEndpoint()],
executor='blocking')
server.start()
server.wait()
def start(self):
transport = messaging.get_transport(myconf)
topic_ex = '<<%s>>' % (self.dist)
target = messaging.Target(topic=self.SERVER_TOPIC, server=topic_ex, version=self.BASE_RPC_API_VERSION)
endpoints = [self]
server = messaging.get_rpc_server(transport, target, endpoints, executor='blocking')
server.start()
server.wait()
def __init__(self, topic, server, handlers):
serializer = RequestContextSerializer(JsonPayloadSerializer())
transport = messaging.get_transport(cfg.CONF,
aliases=TRANSPORT_ALIASES)
# TODO(asalkeld) add support for version='x.y'
target = messaging.Target(topic=topic, server=server)
self._server = messaging.get_rpc_server(transport, target, handlers,
serializer=serializer)
def __init__(self, transport):
self.target = messaging.Target(topic='topic',
server=transport.conf.host)
self._server = messaging.get_rpc_server(transport,
self.target,
[self],
executor='eventlet')
super(Server, self).__init__()
def launch_engine(transport):
target = messaging.Target(topic=cfg.CONF.engine.topic,
server=cfg.CONF.engine.host)
endpoints = [engine.Engine(transport)]
en_server = messaging.get_rpc_server(transport,
target,
endpoints,
executor='eventlet')
en_server.start()
en_server.wait()
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
if serializer is None:
serializer = DesignateObjectSerializer()
serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor='eventlet',
serializer=serializer)
def __init__(self, topic, server, handlers):
serializer = RequestContextSerializer(JsonPayloadSerializer())
transport = messaging.get_transport(cfg.CONF,
aliases=TRANSPORT_ALIASES)
# TODO(asalkeld) add support for version='x.y'
target = messaging.Target(topic=topic, server=server)
self._server = messaging.get_rpc_server(transport,
target,
handlers,
serializer=serializer)
def launch_executor(transport):
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
# Since engine and executor are tightly coupled, use the engine
# configuration to decide which executor to get.
endpoints = [executor.get_executor(cfg.CONF.engine.engine, transport)]
server = messaging.get_rpc_server(
transport, target, endpoints, executor='eventlet')
server.start()
server.wait()
def launch_engine(transport):
serializer = context.RpcContextSerializer(context.JsonPayloadSerializer())
target = messaging.Target(topic=cfg.CONF.engine.topic,
server=cfg.CONF.engine.host)
endpoints = [engine.get_engine(cfg.CONF.engine.engine, transport)]
server = messaging.get_rpc_server(
transport, target, endpoints, executor='eventlet',
serializer=serializer)
server.start()
server.wait()
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
if serializer is None:
serializer = DesignateObjectSerializer()
serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor='eventlet',
serializer=serializer)
def test_no_server_topic(self):
transport = messaging.get_transport(self.conf, url='fake:')
target = messaging.Target(server='testserver')
server = messaging.get_rpc_server(transport, target, [])
try:
server.start()
except Exception as ex:
self.assertIsInstance(ex, messaging.InvalidTarget, ex)
self.assertEqual('testserver', ex.target.server)
else:
self.assertTrue(False)
def test_no_server_topic(self):
transport = messaging.get_transport(self.conf, url='fake:')
target = messaging.Target(server='testserver')
server = messaging.get_rpc_server(transport, target, [])
try:
server.start()
except Exception as ex:
self.assertTrue(isinstance(ex, messaging.ServerListenError), ex)
self.assertEqual(ex.target.server, 'testserver')
else:
self.assertTrue(False)
def test_no_target_server(self):
transport = messaging.get_transport(self.conf, url="fake:")
server = messaging.get_rpc_server(transport, messaging.Target(topic="testtopic"), [])
try:
server.start()
except Exception as ex:
self.assertIsInstance(ex, messaging.InvalidTarget, ex)
self.assertEqual("testtopic", ex.target.topic)
else:
self.assertTrue(False)
def test_no_server_topic(self):
transport = messaging.get_transport(self.conf, url='fake:')
target = messaging.Target(server='testserver')
server = messaging.get_rpc_server(transport, target, [])
try:
server.start()
except Exception as ex:
self.assertIsInstance(ex, messaging.InvalidTarget, ex)
self.assertEqual('testserver', ex.target.server)
else:
self.assertTrue(False)
def main():
setup_agent()
logger.info('Running os-collect-config from main()')
transport = messaging.get_transport(cfg.CONF)
target = messaging.Target(topic=CONF.topic_name, version='1.0',
server=CONF.server_id)
server = messaging.get_rpc_server(transport, target,
endpoints=[AgentEndpoint()])
server.start()
server.wait()
def test_no_target_server(self):
transport = messaging.get_transport(self.conf, url='fake:')
server = messaging.get_rpc_server(transport,
messaging.Target(topic='testtopic'),
[])
try:
server.start()
except Exception as ex:
self.assertIsInstance(ex, messaging.InvalidTarget, ex)
self.assertEqual(ex.target.topic, 'testtopic')
else:
self.assertTrue(False)
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
# Thread module is not monkeypatched if remote debugging is enabled.
# Using eventlet executor without monkepatching thread module will
# lead to unpredictable results.
from trove.common import debug_utils
debug_utils.setup()
executor = "blocking" if debug_utils.enabled() else "eventlet"
serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT, target, endpoints, executor=executor, serializer=serializer)
def test_constructor(self):
transport = messaging.get_transport(self.conf, url="fake:")
target = messaging.Target(topic="foo", server="bar")
endpoints = [object()]
serializer = object()
server = messaging.get_rpc_server(transport, target, endpoints, serializer=serializer)
self.assertIs(server.conf, self.conf)
self.assertIs(server.transport, transport)
self.assertIsInstance(server.dispatcher, messaging.RPCDispatcher)
self.assertIs(server.dispatcher.endpoints, endpoints)
self.assertIs(server.dispatcher.serializer, serializer)
self.assertEqual("blocking", server.executor)
def test_constructor(self):
transport = messaging.get_transport(self.conf, url='fake:')
target = messaging.Target(topic='foo', server='bar')
endpoints = [object()]
serializer = object()
server = messaging.get_rpc_server(transport, target, endpoints,
serializer=serializer)
self.assertIs(server.conf, self.conf)
self.assertIs(server.transport, transport)
self.assertIsInstance(server.dispatcher, messaging.RPCDispatcher)
self.assertIs(server.dispatcher.endpoints, endpoints)
self.assertIs(server.dispatcher.serializer, serializer)
self.assertIs(server.executor, 'blocking')
def __init__(self, transport, name, controller, workers):
target = messaging.Target(exchange="test-exchange",
topic="test-topic",
server=name)
endpoints = [Test(), Control(transport, target, name)]
if controller:
self.collector = Collector()
if workers:
self.collector.workers = workers
endpoints.append(self.collector)
else:
self.collector = None
self._server = messaging.get_rpc_server(transport, target, endpoints)
self._ctrl = messaging.RPCClient(transport, target, timeout=2)
def __init__(self, transport, name, controller, workers):
target = messaging.Target(exchange="test-exchange",
topic="test-topic",
server=name)
endpoints = [Test(), Control(transport, target, name)]
if controller:
self.collector = Collector()
if workers:
self.collector.workers = workers
endpoints.append(self.collector)
else:
self.collector = None
self._server = messaging.get_rpc_server(transport, target, endpoints)
self._ctrl = messaging.RPCClient(transport, target, timeout=2)
def setUp(self):
# Initialize configuration for the ExecutorClient.
super(TestExecutor, self).setUp()
if not 'executor' in cfg.CONF:
cfg_grp = cfg.OptGroup(name='executor', title='Executor options')
opts = [cfg.StrOpt('host', default='0.0.0.0'),
cfg.StrOpt('topic', default='executor')]
cfg.CONF.register_group(cfg_grp)
cfg.CONF.register_opts(opts, group=cfg_grp)
# Start the Executor.
transport = self.get_transport()
target = messaging.Target(topic='executor', server='0.0.0.0')
endpoints = [server.Executor()]
self.server = messaging.get_rpc_server(transport, target,
endpoints, executor='eventlet')
self.server.start()
def start(self):
super(RpcService, self).start()
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
transport = messaging.get_transport(self.conf,
aliases=TRANSPORT_ALIASES)
self.rpcserver = messaging.get_rpc_server(
transport, target, endpoints, executor='eventlet',
serializer=self.serializer)
# Hook to allow the manager to do other initializations after
# the rpc connection is created.
if callable(getattr(self.manager, 'initialize_service_hook', None)):
self.manager.initialize_service_hook(self)
self.rpcserver.start()
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
# Thread module is not monkeypatched if remote debugging is enabled.
# Using eventlet executor without monkepatching thread module will
# lead to unpredictable results.
from trove.common import debug_utils
debug_utils.setup()
executor = "blocking" if debug_utils.enabled() else "eventlet"
serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT,
target,
endpoints,
executor=executor,
serializer=serializer)
def test_server_wait_method(self):
transport = messaging.get_transport(self.conf, url='fake:')
target = messaging.Target(topic='foo', server='bar')
endpoints = [object()]
serializer = object()
server = messaging.get_rpc_server(transport, target, endpoints,
serializer=serializer)
# Mocking executor
server._executor = mock.Mock()
# Here assigning executor's listener object to listener variable
# before calling wait method, beacuse in wait method we are
# setting executor to None.
listener = server._executor.listener
# call server wait method
server.wait()
self.assertIsNone(server._executor)
self.assertEqual(1, listener.cleanup.call_count)
def launch_executor(transport):
target = messaging.Target(
topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host
)
executor_v2 = def_executor.DefaultExecutor(rpc.get_engine_client())
endpoints = [rpc.ExecutorServer(executor_v2)]
server = messaging.get_rpc_server(
transport,
target,
endpoints,
executor='eventlet',
serializer=ctx.RpcContextSerializer(ctx.JsonPayloadSerializer())
)
server.start()
server.wait()
def start(self):
self.dispatcher_manager = dispatcher.load_dispatcher_manager()
self.notification_server = None
super(ConsumerService, self).start()
allow_requeue = cfg.CONF.consumer.requeue_on_error
transport = messaging.get_transport(optional=True)
if transport:
self.rpc_server = messaging.get_rpc_server(transport,
cfg.CONF.consumer.topic,
self)
target = oslo.messaging.Target(topic=cfg.CONF.consumer.topic)
self.notification_server = messaging.get_notification_listener(
transport, [target], [
ConsumerEndpoint(self.dispatcher_manager,
'record_metering_data',
cfg.CONF.consumer.priority)
],
allow_requeue=allow_requeue)
self.notification_server.start()
def launch_engine(transport):
target = messaging.Target(
topic=cfg.CONF.engine.topic,
server=cfg.CONF.engine.host
)
engine_v2 = def_eng.DefaultEngine(rpc.get_engine_client())
endpoints = [rpc.EngineServer(engine_v2)]
# Setup scheduler in engine.
db_api.setup_db()
scheduler.setup()
server = messaging.get_rpc_server(
transport,
target,
endpoints,
executor='eventlet',
serializer=ctx.RpcContextSerializer(ctx.JsonPayloadSerializer())
)
server.start()
server.wait()
def __init__(self, host, storage_driver=None, topic=None, allow_stop=False):
topic = topic or 'engine'
self.host = host
self.topic = 'gate.%s' % topic
self._storage_driver = storage_driver
if not self._storage_driver:
self._storage_driver = get_storage_driver()
self._check_indexes()
self._transport = messaging.get_transport(cfg.CONF)
self._target = messaging.Target(topic=self.topic, server=self.host)
self._process_client = ProcessClient(self._transport)
self._endpoints = [
EngineAPI(self, process_client=self._process_client)
]
if allow_stop:
self._endpoints.append(self)
self._server = messaging.get_rpc_server(
self._transport, self._target, self._endpoints)
def __init__(self, transport, topic, server, endpoint, serializer):
target = messaging.Target(topic=topic, server=server)
self._server = messaging.get_rpc_server(transport,
target, [endpoint, self],
serializer=serializer)
def _prepare_rpc_service(server_id):
endpoints = [TaskProcessingEndpoint()]
transport = messaging.get_transport(config.CONF)
s_target = target.Target('murano', 'tasks', server=server_id)
return messaging.get_rpc_server(transport, s_target, endpoints, 'eventlet')
def get_server(target, endpoints, serializer=None):
assert TRANSPORT is not None
serializer = RequestContextSerializer(serializer)
return messaging.get_rpc_server(TRANSPORT, target, endpoints, 'eventlet',
serializer)
config_file_args = ['--config-file', './msg.conf']
class ServerManager(object):
rid = 0
def echo(self, ctx, msg, sentime):
print '%s request msg: %s' % (sentime, msg)
self.rid = self.rid + 1
s = '%s server(%d) accepted no.%d' % (datetime.datetime.now(),
os.getpid(), self.rid)
print s
return s
endpoints = [ServerManager()]
if __name__ == '__main__':
if len(sys.argv) >= 2:
config_file_args = sys.argv[1:3]
CONF(config_file_args)
transport = messaging.get_transport(CONF)
target = messaging.Target(topic='demoservice',
server='server' + str(os.getpid()))
server = messaging.get_rpc_server(transport, target, endpoints)
print 'server(%d) starting' % os.getpid()
server.start()
server.wait()
def __init__(self, target):
self.__server = messaging.get_rpc_server(
target=target,
transport=messaging.get_transport(cfg.CONF),
endpoints=[self],
executor='eventlet')
