def __init__(self, router=None):
BaseNode.__init__(self)
self._own_router = True
if router is not None:
self._local_router = router
self._own_router = False
else:
self._local_router = LocalRouter(get_sys_name())
self._channel_id_pool = IDPool()
def setUp(self):
self.lr = LocalRouter(get_sys_name())
self.lr.start()
self.addCleanup(self.lr.stop)
# make a hook so we can tell when a message gets routed
self.ev = Event()
def new_route(*args, **kwargs):
self.ev.set()
self.lr._oldroute(*args, **kwargs)
self.lr._oldroute = self.lr._route
self.lr._route = new_route
def test__run_consumer(self):
propsmock = Mock()
propsmock.copy.return_value = sentinel.props
gqueue = Mock()
m = (sentinel.exchange, sentinel.routing_key, sentinel.body, propsmock)
gqueue.get.side_effect = [m, LocalRouter.ConsumerClosedMessage()]
cb = Mock()
self.lr._generate_dtag = Mock(return_value=sentinel.dtag)
self.lr._run_consumer(sentinel.ctag, sentinel.queue, gqueue, cb)
self.assertEquals(cb.call_count, 1)
self.assertEquals(cb.call_args[0][0], self.lr)
self.assertEquals(
dict(cb.call_args[0][1]), {
'consumer_tag': sentinel.ctag,
'delivery_tag': sentinel.dtag,
'redelivered': False,
'exchange': sentinel.exchange,
'routing_key': sentinel.routing_key
})
self.assertEquals(dict(cb.call_args[0][2]),
{'headers': sentinel.props})
self.assertEquals(cb.call_args[0][3], sentinel.body)
self.assertIn((sentinel.ctag, sentinel.queue, m),
self.lr._unacked.itervalues())
def __init__(self, router=None):
BaseNode.__init__(self)
self._own_router = True
if router is not None:
self._local_router = router
self._own_router = False
else:
self._local_router = LocalRouter(get_sys_name())
self._channel_id_pool = IDPool()
def setUp(self):
self.lr = LocalRouter(get_sys_name())
self.lr.start()
self.addCleanup(self.lr.stop)
# make a hook so we can tell when a message gets routed
self.ev = Event()
def new_route(*args, **kwargs):
self.ev.set()
self.lr._oldroute(*args, **kwargs)
self.lr._oldroute = self.lr._route
self.lr._route = new_route
class LocalNode(BaseNode):
def __init__(self, router=None):
BaseNode.__init__(self)
self._own_router = True
if router is not None:
self._local_router = router
self._own_router = False
else:
self._local_router = LocalRouter(get_sys_name())
self._channel_id_pool = IDPool()
def start_node(self):
BaseNode.start_node(self)
if self._own_router:
self._local_router.start()
def stop_node(self):
if self.running:
if self._own_router:
self._local_router.stop()
self.running = False
def _new_transport(self, ch_number=None):
trans = LocalTransport(self._local_router, ch_number)
return trans
def channel(self, ch_type, transport=None):
ch = self._new_channel(ch_type,
ch_number=self._channel_id_pool.get_id(),
transport=transport)
# @TODO keep track of all channels to close them later from the top
ch._transport.add_on_close_callback(self._on_channel_close)
return ch
def _on_channel_close(self, ch, code, text):
log.debug("LocalNode._on_channel_close (%s)", ch.channel_number)
self._channel_id_pool.release_id(ch.channel_number)
class LocalNode(BaseNode):
def __init__(self, router=None):
BaseNode.__init__(self)
self._own_router = True
if router is not None:
self._local_router = router
self._own_router = False
else:
self._local_router = LocalRouter(get_sys_name())
self._channel_id_pool = IDPool()
def start_node(self):
BaseNode.start_node(self)
if self._own_router:
self._local_router.start()
def stop_node(self):
if self.running:
if self._own_router:
self._local_router.stop()
self.running = False
def _new_transport(self, ch_number=None):
trans = LocalTransport(self._local_router, ch_number)
return trans
def channel(self, ch_type, transport=None):
ch = self._new_channel(ch_type, ch_number=self._channel_id_pool.get_id(), transport=transport)
# @TODO keep track of all channels to close them later from the top
ch._transport.add_on_close_callback(self._on_channel_close)
return ch
def _on_channel_close(self, ch, code, text):
log.debug("ZeroMQNode._on_channel_close (%s)", ch.channel_number)
self._channel_id_pool.release_id(ch.channel_number)
class TestLocalRouter(PyonTestCase):
def setUp(self):
self.lr = LocalRouter(get_sys_name())
self.lr.start()
self.addCleanup(self.lr.stop)
# make a hook so we can tell when a message gets routed
self.ev = Event()
def new_route(*args, **kwargs):
self.ev.set()
self.lr._oldroute(*args, **kwargs)
self.lr._oldroute = self.lr._route
self.lr._route = new_route
def test_publish_to_unknown_exchange(self):
self.assertEquals(len(self.lr.errors), 0)
self.lr.publish('ex', 'rkey', 'body', 'props')
self.ev.wait(timeout=10)
self.assertEquals(len(self.lr.errors), 1)
def test_publish_to_known_exchange(self):
# declare exchange
self.lr.declare_exchange('known')
# send message
self.lr.publish('known', 'rkey', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# message is binned but no errors
self.assertEquals(len(self.lr.errors), 0)
def test_publish_to_queue(self):
# declare exchange/queue/binding
self.lr.declare_exchange('known')
self.lr.declare_queue('iamqueue')
self.lr.bind('known', 'iamqueue', 'binzim')
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 0)
# send message
self.lr.publish('known', 'binzim', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# should be sitting in a queue waiting
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 1)
#self.assertIn(('known', 'binzim', 'body', 'props'), self.lr._queues['iamqueue'])
def test_publish_to_many_queues(self):
# declare exchange/queue/binding
self.lr.declare_exchange('known')
self.lr.declare_queue('q1')
self.lr.bind('known', 'q1', 'a.*')
self.lr.declare_queue('q2')
self.lr.bind('known', 'q2', 'a.b')
self.lr.declare_queue('q3')
self.lr.bind('known', 'q3', '*.b')
# send message
self.lr.publish('known', 'a.b', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# should be in 3 queues
for q in ['q1', 'q2', 'q3']:
self.assertEquals(self.lr._queues[q].qsize(), 1)
#self.assertIn(('known', 'a.b', 'body', 'props'), self.lr._queues[q])
def test_publish_to_queue_with_multiple_matching_binds_only_makes_one_message(
self):
# exchange/queue/bindings
self.lr.declare_exchange('known')
self.lr.declare_queue('iamqueue')
self.lr.bind('known', 'iamqueue', 'a.*')
self.lr.bind('known', 'iamqueue', 'a.b')
self.lr.bind('known', 'iamqueue', '*.b')
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 0)
# send message
self.lr.publish('known', 'a.b', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# should be in the queue
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 1)
#self.assertIn(('known', 'a.b', 'body', 'props'), self.lr._queues['iamqueue'])
def test_publish_with_binds_and_unbinds(self):
# declare exchange/queue
self.lr.declare_exchange('known')
self.lr.declare_queue('ein')
self.assertEquals(self.lr._queues['ein'].qsize(), 0)
# send message
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# not bound, so doesn't go into the queue
self.assertEquals(self.lr._queues['ein'].qsize(), 0)
# bind now
self.lr.bind('known', 'ein', 'ein')
# send message
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# should be in queue
self.assertEquals(self.lr._queues['ein'].qsize(), 1)
# send again
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# now 2 in queue
self.assertEquals(self.lr._queues['ein'].qsize(), 2)
# unbind
self.lr.unbind('known', 'ein', 'ein')
# send again
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# still 2 in queue
self.assertEquals(self.lr._queues['ein'].qsize(), 2)
def test__connect_addr(self):
self.assertEquals(self.lr._connect_addr,
"inproc://%s" % get_sys_name())
def test__child_failed(self):
self.lr.gl_ioloop = Mock()
fail = Mock()
self.lr._child_failed(fail)
self.lr.gl_ioloop.kill.assert_called_once_with(
exception=fail.exception, block=False)
def test__run_ioloop(self):
self.lr._gl_pool = Mock()
self.lr._run_ioloop()
self.lr._gl_pool.join.assert_called_once_with()
def test_declare_exchange(self):
self.lr.declare_exchange(sentinel.exchange)
self.assertIn(sentinel.exchange, self.lr._exchanges)
self.assertIsInstance(self.lr._exchanges[sentinel.exchange], TopicTrie)
def test_declare_exchange_existing(self):
tt = TopicTrie()
self.lr._exchanges[sentinel.exchange] = tt
self.lr.declare_exchange(sentinel.exchange)
self.assertEquals(tt, self.lr._exchanges[sentinel.exchange])
def test_delete_exchange(self):
self.lr.declare_exchange(sentinel.exchange)
self.lr.delete_exchange(sentinel.exchange)
self.assertEquals(self.lr._exchanges, {})
def test_declare_queue(self):
self.lr.declare_queue(sentinel.queue)
self.assertIn(sentinel.queue, self.lr._queues)
def test_declare_queue_made_up(self):
q = self.lr.declare_queue('')
self.assertIn("q-", q)
self.assertIn(q, self.lr._queues)
def test_delete_queue(self):
self.lr.declare_queue(sentinel.queue)
self.lr.delete_queue(sentinel.queue)
self.assertNotIn(sentinel.queue, self.lr._queues)
def test_delete_queue_deletes_bindings(self):
self.lr.declare_exchange(sentinel.exchange)
self.lr.declare_queue(sentinel.queue)
self.lr.bind(sentinel.exchange, sentinel.queue, 'binder')
self.lr.delete_queue(sentinel.queue)
self.assertNotIn(sentinel.queue, self.lr._queues)
self.assertNotIn(sentinel.queue, self.lr._bindings_by_queue)
def test_start_consume(self):
self.lr._gl_pool = Mock()
self.lr.declare_queue(sentinel.queue)
ctag = self.lr.start_consume(sentinel.callback, sentinel.queue)
self.assertIn(sentinel.queue, self.lr._consumers)
self.assertIn(ctag, self.lr._consumers_by_ctag)
self.assertEquals(self.lr._consumers[sentinel.queue],
[(ctag, sentinel.callback, False, False,
self.lr._gl_pool.spawn.return_value)])
def test_stop_consume(self):
self.lr._gl_pool = Mock()
self.lr.declare_queue(sentinel.queue)
ctag = self.lr.start_consume(sentinel.callback, sentinel.queue)
self.lr.stop_consume(ctag)
self.assertNotIn(ctag, self.lr._consumers_by_ctag)
self.lr._gl_pool.spawn().join.assert_called_once_with(timeout=5)
self.lr._gl_pool.spawn().kill.assert_called_once_with()
self.assertEquals(len(self.lr._consumers[sentinel.queue]), 0)
def test__run_consumer(self):
propsmock = Mock()
propsmock.copy.return_value = sentinel.props
gqueue = Mock()
m = (sentinel.exchange, sentinel.routing_key, sentinel.body, propsmock)
gqueue.get.side_effect = [m, LocalRouter.ConsumerClosedMessage()]
cb = Mock()
self.lr._generate_dtag = Mock(return_value=sentinel.dtag)
self.lr._run_consumer(sentinel.ctag, sentinel.queue, gqueue, cb)
self.assertEquals(cb.call_count, 1)
self.assertEquals(cb.call_args[0][0], self.lr)
self.assertEquals(
dict(cb.call_args[0][1]), {
'consumer_tag': sentinel.ctag,
'delivery_tag': sentinel.dtag,
'redelivered': False,
'exchange': sentinel.exchange,
'routing_key': sentinel.routing_key
})
self.assertEquals(dict(cb.call_args[0][2]),
{'headers': sentinel.props})
self.assertEquals(cb.call_args[0][3], sentinel.body)
self.assertIn((sentinel.ctag, sentinel.queue, m),
self.lr._unacked.itervalues())
def test__generate_ctag(self):
self.lr._ctag_pool = Mock()
self.lr._ctag_pool.get_id.return_value = sentinel.ctagid
self.assertEquals(self.lr._generate_ctag(),
"zctag-%s" % str(sentinel.ctagid))
def test__return_ctag(self):
self.lr._ctag_pool = Mock()
self.lr._return_ctag("m-5")
self.lr._ctag_pool.release_id.assert_called_once_with(5)
def test__generate_and_return_ctag(self):
ctag = self.lr._generate_ctag()
ctagnum = int(ctag.split("-")[-1])
self.assertIn(ctagnum, self.lr._ctag_pool._ids_in_use)
self.lr._return_ctag(ctag)
self.assertIn(ctagnum, self.lr._ctag_pool._ids_free)
self.assertNotIn(ctagnum, self.lr._ctag_pool._ids_in_use)
def test__generate_dtag(self):
dtag = self.lr._generate_dtag(sentinel.ctag, sentinel.cnt)
self.assertIn(str(sentinel.ctag), dtag)
self.assertIn(str(sentinel.cnt), dtag)
def test_ack(self):
self.lr._unacked[sentinel.dtag] = True
self.lr.ack(sentinel.dtag)
self.assertEquals(len(self.lr._unacked), 0)
def test_reject(self):
self.lr._unacked[sentinel.dtag] = (None, None, None)
self.lr.reject(sentinel.dtag)
self.assertEquals(len(self.lr._unacked), 0)
def test_reject_requeue(self):
q = Mock()
self.lr._queues[sentinel.queue] = q
self.lr._unacked[sentinel.dtag] = (None, sentinel.queue, sentinel.m)
self.lr.reject(sentinel.dtag, requeue=True)
self.assertEquals(len(self.lr._unacked), 0)
q.put.assert_called_once_with(sentinel.m)
def test_transport_close(self):
# no body in localrouter method
pass
def test_get_stats(self):
self.lr.declare_queue(sentinel.queue)
self.lr._queues[sentinel.queue].put(sentinel.m)
mc, cc = self.lr.get_stats(sentinel.queue)
self.assertEquals((mc, cc), (1, 0))
def test_get_stats_with_consumers(self):
self.lr.declare_queue(sentinel.queue)
self.lr._queues[sentinel.queue].put(sentinel.m)
self.lr._consumers[sentinel.queue] = [sentinel.ctag]
mc, cc = self.lr.get_stats(sentinel.queue)
self.assertEquals((mc, cc), (1, 1))
def test_purge(self):
self.lr.declare_queue(sentinel.queue)
self.lr._queues[sentinel.queue].put(sentinel.m)
self.assertEquals(self.lr._queues[sentinel.queue].qsize(), 1)
self.lr.purge(sentinel.queue)
self.assertEquals(self.lr._queues[sentinel.queue].qsize(), 0)
class Container(BaseContainerAgent):
"""
The Capability Container. Its purpose is to spawn/monitor processes and services
that do the bulk of the work in the ION system. It also manages connections to the Exchange
and the various forms of datastores in the systems.
"""
# Singleton static variables
#node = None
id = None
name = None
pidfile = None
instance = None
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set container id and cc_agent name (as they are set in base class call)
self.id = get_default_container_id()
self.name = "cc_agent_%s" % self.id
self._capabilities = []
bootstrap.container_instance = self
Container.instance = self
log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# TODO: Do not start a capability here. Symmetric start/stop
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Instantiate Directory
self.directory = Directory()
# internal router
self.local_router = None
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
# publisher, initialized in start()
self.event_pub = None
# context-local storage
self.context = LocalContextMixin()
log.debug("Container initialized, OK.")
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError("Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s" % self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name()}
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit() # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
# set up greenlet debugging signal handler
gevent.signal(signal.SIGUSR2, self._handle_sigusr2)
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects", DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# internal router for local transports
self.local_router = LocalRouter(bootstrap.get_sys_name())
self.local_router.start()
self.local_router.ready.wait(timeout=2)
self._capabilities.append("LOCAL_ROUTER")
# Start ExchangeManager, which starts the node (broker connection)
self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.get_safe('container.sflow.enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node, from_name=self.name, service=self, process=self)
cleanup = lambda _: self.proc_manager._cleanup_method(self.name, rsvc)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(name=self.name, listeners=[rsvc], service=self, cleanup_method=cleanup)
self.proc_manager.proc_sup.ensure_ready(proc)
proc.start_listeners()
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container (%s) started, OK." , self.id)
def _handle_sigusr2(self):#, signum, frame):
"""
Handles SIGUSR2, prints debugging greenlet information.
"""
gls = GreenletLeak.get_greenlets()
allgls = []
for gl in gls:
status = GreenletLeak.format_greenlet(gl)
# build formatted output:
# Greenlet at 0xdeadbeef
# self: <EndpointUnit at 0x1ffcceef>
# func: bound, EndpointUnit.some_func
status[0].insert(0, "%s at %s:" % (gl.__class__.__name__, hex(id(gl))))
# indent anything in status a second time
prefmt = [s.replace("\t", "\t\t") for s in status[0]]
prefmt.append("traceback:")
for line in status[1]:
for subline in line.split("\n")[0:2]:
prefmt.append(subline)
glstr = "\n\t".join(prefmt)
allgls.append(glstr)
# print it out!
print >>sys.stderr, "\n\n".join(allgls)
with open("gls-%s" % os.getpid(), "w") as f:
f.write("\n\n".join(allgls))
@property
def node(self):
"""
Returns the active/default Node that should be used for most communication in the system.
Defers to exchange manager, but only if it has been started, otherwise returns None.
"""
if "EXCHANGE_MANAGER" in self._capabilities:
return self.ex_manager.default_node
return None
@contextmanager
def _push_status(self, new_status):
"""
Temporarily sets the internal status flag.
Use this as a decorator or in a with-statement before calling a temporary status changing
method, like start_rel_from_url.
"""
curstatus = self._status
self._status = new_status
try:
yield
finally:
self._status = curstatus
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# serve forever short-circuits if immediate is on and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if not (immediate and num_procs == 1): # only spawned greenlet is the CC-Agent
# print a warning just in case
if immediate and num_procs != 1:
log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info('Received a kill signal, shutting down the container.')
if hasattr(self, 'gl_parent_watch') and self.gl_parent_watch is not None:
self.gl_parent_watch.kill()
except:
log.exception('Unhandled error! Forcing container shutdown')
else:
log.debug("Container.serve_forever short-circuiting due to CFG.system.immediate")
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def status(self):
"""
Returns the internal status.
"""
return self._status
def _cleanup_pid(self):
if self.pidfile:
log.debug("Cleanup pidfile: %s", self.pidfile)
try:
os.remove(self.pidfile)
except Exception, e:
log.warn("Pidfile could not be deleted: %s" % str(e))
self.pidfile = None
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError("Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s" % self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name()}
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit() # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
# set up greenlet debugging signal handler
gevent.signal(signal.SIGUSR2, self._handle_sigusr2)
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects", DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# internal router for local transports
self.local_router = LocalRouter(bootstrap.get_sys_name())
self.local_router.start()
self.local_router.ready.wait(timeout=2)
self._capabilities.append("LOCAL_ROUTER")
# Start ExchangeManager, which starts the node (broker connection)
self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.get_safe('container.sflow.enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node, from_name=self.name, service=self, process=self)
cleanup = lambda _: self.proc_manager._cleanup_method(self.name, rsvc)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(name=self.name, listeners=[rsvc], service=self, cleanup_method=cleanup)
self.proc_manager.proc_sup.ensure_ready(proc)
proc.start_listeners()
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container (%s) started, OK." , self.id)
def start(self):
# internal router for local transports
self.container.local_router = LocalRouter(bootstrap.get_sys_name())
self.container.local_router.start()
self.container.local_router.ready.wait(timeout=2)
class TestLocalRouter(PyonTestCase):
def setUp(self):
self.lr = LocalRouter(get_sys_name())
self.lr.start()
self.addCleanup(self.lr.stop)
# make a hook so we can tell when a message gets routed
self.ev = Event()
def new_route(*args, **kwargs):
self.ev.set()
self.lr._oldroute(*args, **kwargs)
self.lr._oldroute = self.lr._route
self.lr._route = new_route
def test_publish_to_unknown_exchange(self):
self.assertEquals(len(self.lr.errors), 0)
self.lr.publish('ex', 'rkey', 'body', 'props')
self.ev.wait(timeout=10)
self.assertEquals(len(self.lr.errors), 1)
def test_publish_to_known_exchange(self):
# declare exchange
self.lr.declare_exchange('known')
# send message
self.lr.publish('known', 'rkey', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# message is binned but no errors
self.assertEquals(len(self.lr.errors), 0)
def test_publish_to_queue(self):
# declare exchange/queue/binding
self.lr.declare_exchange('known')
self.lr.declare_queue('iamqueue')
self.lr.bind('known', 'iamqueue', 'binzim')
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 0)
# send message
self.lr.publish('known', 'binzim', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# should be sitting in a queue waiting
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 1)
#self.assertIn(('known', 'binzim', 'body', 'props'), self.lr._queues['iamqueue'])
def test_publish_to_many_queues(self):
# declare exchange/queue/binding
self.lr.declare_exchange('known')
self.lr.declare_queue('q1')
self.lr.bind('known', 'q1', 'a.*')
self.lr.declare_queue('q2')
self.lr.bind('known', 'q2', 'a.b')
self.lr.declare_queue('q3')
self.lr.bind('known', 'q3', '*.b')
# send message
self.lr.publish('known', 'a.b', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# should be in 3 queues
for q in ['q1','q2','q3']:
self.assertEquals(self.lr._queues[q].qsize(), 1)
#self.assertIn(('known', 'a.b', 'body', 'props'), self.lr._queues[q])
def test_publish_to_queue_with_multiple_matching_binds_only_makes_one_message(self):
# exchange/queue/bindings
self.lr.declare_exchange('known')
self.lr.declare_queue('iamqueue')
self.lr.bind('known', 'iamqueue', 'a.*')
self.lr.bind('known', 'iamqueue', 'a.b')
self.lr.bind('known', 'iamqueue', '*.b')
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 0)
# send message
self.lr.publish('known', 'a.b', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
# should be in the queue
self.assertEquals(self.lr._queues['iamqueue'].qsize(), 1)
#self.assertIn(('known', 'a.b', 'body', 'props'), self.lr._queues['iamqueue'])
def test_publish_with_binds_and_unbinds(self):
# declare exchange/queue
self.lr.declare_exchange('known')
self.lr.declare_queue('ein')
self.assertEquals(self.lr._queues['ein'].qsize(), 0)
# send message
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# not bound, so doesn't go into the queue
self.assertEquals(self.lr._queues['ein'].qsize(), 0)
# bind now
self.lr.bind('known', 'ein', 'ein')
# send message
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# should be in queue
self.assertEquals(self.lr._queues['ein'].qsize(), 1)
# send again
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# now 2 in queue
self.assertEquals(self.lr._queues['ein'].qsize(), 2)
# unbind
self.lr.unbind('known', 'ein', 'ein')
# send again
self.lr.publish('known', 'ein', 'body', 'props')
# wait for route
self.ev.wait(timeout=10)
self.ev.clear() # we need to use again
# still 2 in queue
self.assertEquals(self.lr._queues['ein'].qsize(), 2)
def test__connect_addr(self):
self.assertEquals(self.lr._connect_addr, "inproc://%s" % get_sys_name())
def test__child_failed(self):
self.lr.gl_ioloop = Mock()
fail = Mock()
self.lr._child_failed(fail)
self.lr.gl_ioloop.kill.assert_called_once_with(exception=fail.exception,
block=False)
def test__run_ioloop(self):
self.lr._gl_pool = Mock()
self.lr._run_ioloop()
self.lr._gl_pool.join.assert_called_once_with()
def test_declare_exchange(self):
self.lr.declare_exchange(sentinel.exchange)
self.assertIn(sentinel.exchange, self.lr._exchanges)
self.assertIsInstance(self.lr._exchanges[sentinel.exchange], TopicTrie)
def test_declare_exchange_existing(self):
tt = TopicTrie()
self.lr._exchanges[sentinel.exchange] = tt
self.lr.declare_exchange(sentinel.exchange)
self.assertEquals(tt, self.lr._exchanges[sentinel.exchange])
def test_delete_exchange(self):
self.lr.declare_exchange(sentinel.exchange)
self.lr.delete_exchange(sentinel.exchange)
self.assertEquals(self.lr._exchanges, {})
def test_declare_queue(self):
self.lr.declare_queue(sentinel.queue)
self.assertIn(sentinel.queue, self.lr._queues)
def test_declare_queue_made_up(self):
q = self.lr.declare_queue('')
self.assertIn("q-", q)
self.assertIn(q, self.lr._queues)
def test_delete_queue(self):
self.lr.declare_queue(sentinel.queue)
self.lr.delete_queue(sentinel.queue)
self.assertNotIn(sentinel.queue, self.lr._queues)
def test_delete_queue_deletes_bindings(self):
self.lr.declare_exchange(sentinel.exchange)
self.lr.declare_queue(sentinel.queue)
self.lr.bind(sentinel.exchange, sentinel.queue, 'binder')
self.lr.delete_queue(sentinel.queue)
self.assertNotIn(sentinel.queue, self.lr._queues)
self.assertNotIn(sentinel.queue, self.lr._bindings_by_queue)
def test_start_consume(self):
self.lr._gl_pool = Mock()
self.lr.declare_queue(sentinel.queue)
ctag = self.lr.start_consume(sentinel.callback, sentinel.queue)
self.assertIn(sentinel.queue, self.lr._consumers)
self.assertIn(ctag, self.lr._consumers_by_ctag)
self.assertEquals(self.lr._consumers[sentinel.queue], [(ctag,
sentinel.callback,
False,
False,
self.lr._gl_pool.spawn.return_value)])
def test_stop_consume(self):
self.lr._gl_pool = Mock()
self.lr.declare_queue(sentinel.queue)
ctag = self.lr.start_consume(sentinel.callback, sentinel.queue)
self.lr.stop_consume(ctag)
self.assertNotIn(ctag, self.lr._consumers_by_ctag)
self.lr._gl_pool.spawn().join.assert_called_once_with(timeout=5)
self.lr._gl_pool.spawn().kill.assert_called_once_with()
self.assertEquals(len(self.lr._consumers[sentinel.queue]), 0)
def test__run_consumer(self):
propsmock = Mock()
propsmock.copy.return_value = sentinel.props
gqueue = Mock()
m = (sentinel.exchange, sentinel.routing_key, sentinel.body, propsmock)
gqueue.get.side_effect = [m,
LocalRouter.ConsumerClosedMessage()]
cb = Mock()
self.lr._generate_dtag=Mock(return_value=sentinel.dtag)
self.lr._run_consumer(sentinel.ctag, sentinel.queue, gqueue, cb)
self.assertEquals(cb.call_count, 1)
self.assertEquals(cb.call_args[0][0], self.lr)
self.assertEquals(dict(cb.call_args[0][1]), {'consumer_tag': sentinel.ctag,
'delivery_tag': sentinel.dtag,
'redelivered': False,
'exchange': sentinel.exchange,
'routing_key': sentinel.routing_key})
self.assertEquals(dict(cb.call_args[0][2]), {'headers':sentinel.props})
self.assertEquals(cb.call_args[0][3], sentinel.body)
self.assertIn((sentinel.ctag, sentinel.queue, m), self.lr._unacked.itervalues())
def test__generate_ctag(self):
self.lr._ctag_pool = Mock()
self.lr._ctag_pool.get_id.return_value = sentinel.ctagid
self.assertEquals(self.lr._generate_ctag(), "zctag-%s" % str(sentinel.ctagid))
def test__return_ctag(self):
self.lr._ctag_pool = Mock()
self.lr._return_ctag("m-5")
self.lr._ctag_pool.release_id.assert_called_once_with(5)
def test__generate_and_return_ctag(self):
ctag = self.lr._generate_ctag()
ctagnum = int(ctag.split("-")[-1])
self.assertIn(ctagnum, self.lr._ctag_pool._ids_in_use)
self.lr._return_ctag(ctag)
self.assertIn(ctagnum, self.lr._ctag_pool._ids_free)
self.assertNotIn(ctagnum, self.lr._ctag_pool._ids_in_use)
def test__generate_dtag(self):
dtag = self.lr._generate_dtag(sentinel.ctag, sentinel.cnt)
self.assertIn(str(sentinel.ctag), dtag)
self.assertIn(str(sentinel.cnt), dtag)
def test_ack(self):
self.lr._unacked[sentinel.dtag] = True
self.lr.ack(sentinel.dtag)
self.assertEquals(len(self.lr._unacked), 0)
def test_reject(self):
self.lr._unacked[sentinel.dtag] = (None, None, None)
self.lr.reject(sentinel.dtag)
self.assertEquals(len(self.lr._unacked), 0)
def test_reject_requeue(self):
q = Mock()
self.lr._queues[sentinel.queue] = q
self.lr._unacked[sentinel.dtag] = (None, sentinel.queue, sentinel.m)
self.lr.reject(sentinel.dtag, requeue=True)
self.assertEquals(len(self.lr._unacked), 0)
q.put.assert_called_once_with(sentinel.m)
def test_transport_close(self):
# no body in localrouter method
pass
def test_get_stats(self):
self.lr.declare_queue(sentinel.queue)
self.lr._queues[sentinel.queue].put(sentinel.m)
mc, cc = self.lr.get_stats(sentinel.queue)
self.assertEquals((mc, cc), (1, 0))
def test_get_stats_with_consumers(self):
self.lr.declare_queue(sentinel.queue)
self.lr._queues[sentinel.queue].put(sentinel.m)
self.lr._consumers[sentinel.queue] = [sentinel.ctag]
mc, cc = self.lr.get_stats(sentinel.queue)
self.assertEquals((mc, cc), (1, 1))
def test_purge(self):
self.lr.declare_queue(sentinel.queue)
self.lr._queues[sentinel.queue].put(sentinel.m)
self.assertEquals(self.lr._queues[sentinel.queue].qsize(), 1)
self.lr.purge(sentinel.queue)
self.assertEquals(self.lr._queues[sentinel.queue].qsize(), 0)
class Container(BaseContainerAgent):
"""
The Capability Container. Its purpose is to spawn/monitor processes and services
that do the bulk of the work in the ION system. It also manages connections to the Exchange
and the various forms of datastores in the systems.
"""
# Singleton static variables
#node = None
id = None
name = None
pidfile = None
instance = None
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set container id and cc_agent name (as they are set in base class call)
self.id = get_default_container_id()
self.name = "cc_agent_%s" % self.id
self._capabilities = []
bootstrap.container_instance = self
Container.instance = self
log.debug("Container (sysname=%s) initializing ..." %
bootstrap.get_sys_name())
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# TODO: Do not start a capability here. Symmetric start/stop
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Instantiate Directory
self.directory = Directory()
# internal router
self.local_router = None
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
# publisher, initialized in start()
self.event_pub = None
# context-local storage
self.context = LocalContextMixin()
log.debug("Container initialized, OK.")
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError(
"Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s"
% self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {
'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name()
}
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit(
) # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
# set up greenlet debugging signal handler
gevent.signal(signal.SIGUSR2, self._handle_sigusr2)
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects",
DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# internal router for local transports
self.local_router = LocalRouter(bootstrap.get_sys_name())
self.local_router.start()
self.local_router.ready.wait(timeout=2)
self._capabilities.append("LOCAL_ROUTER")
# Start ExchangeManager, which starts the node (broker connection)
self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.get_safe('container.sflow.enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node,
from_name=self.name,
service=self,
process=self)
cleanup = lambda _: self.proc_manager._cleanup_method(self.name, rsvc)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(name=self.name,
listeners=[rsvc],
service=self,
cleanup_method=cleanup)
self.proc_manager.proc_sup.ensure_ready(proc)
proc.start_listeners()
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id,
origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container (%s) started, OK.", self.id)
def _handle_sigusr2(self): #, signum, frame):
"""
Handles SIGUSR2, prints debugging greenlet information.
"""
gls = GreenletLeak.get_greenlets()
allgls = []
for gl in gls:
status = GreenletLeak.format_greenlet(gl)
# build formatted output:
# Greenlet at 0xdeadbeef
# self: <EndpointUnit at 0x1ffcceef>
# func: bound, EndpointUnit.some_func
status[0].insert(
0, "%s at %s:" % (gl.__class__.__name__, hex(id(gl))))
# indent anything in status a second time
prefmt = [s.replace("\t", "\t\t") for s in status[0]]
prefmt.append("traceback:")
for line in status[1]:
for subline in line.split("\n")[0:2]:
prefmt.append(subline)
glstr = "\n\t".join(prefmt)
allgls.append(glstr)
# print it out!
print >> sys.stderr, "\n\n".join(allgls)
with open("gls-%s" % os.getpid(), "w") as f:
f.write("\n\n".join(allgls))
@property
def node(self):
"""
Returns the active/default Node that should be used for most communication in the system.
Defers to exchange manager, but only if it has been started, otherwise returns None.
"""
if "EXCHANGE_MANAGER" in self._capabilities:
return self.ex_manager.default_node
return None
@contextmanager
def _push_status(self, new_status):
"""
Temporarily sets the internal status flag.
Use this as a decorator or in a with-statement before calling a temporary status changing
method, like start_rel_from_url.
"""
curstatus = self._status
self._status = new_status
try:
yield
finally:
self._status = curstatus
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# serve forever short-circuits if immediate is on and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if not (immediate
and num_procs == 1): # only spawned greenlet is the CC-Agent
# print a warning just in case
if immediate and num_procs != 1:
log.warn(
"CFG.system.immediate=True but number of spawned processes is not 1 (%d)",
num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info(
'Received a kill signal, shutting down the container.')
if hasattr(self, 'gl_parent_watch'
) and self.gl_parent_watch is not None:
self.gl_parent_watch.kill()
except:
log.exception('Unhandled error! Forcing container shutdown')
else:
log.debug(
"Container.serve_forever short-circuiting due to CFG.system.immediate"
)
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def status(self):
"""
Returns the internal status.
"""
return self._status
def _cleanup_pid(self):
if self.pidfile:
log.debug("Cleanup pidfile: %s", self.pidfile)
try:
os.remove(self.pidfile)
except Exception, e:
log.warn("Pidfile could not be deleted: %s" % str(e))
self.pidfile = None
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError(
"Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s"
% self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {
'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name()
}
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit(
) # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
# set up greenlet debugging signal handler
gevent.signal(signal.SIGUSR2, self._handle_sigusr2)
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects",
DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# internal router for local transports
self.local_router = LocalRouter(bootstrap.get_sys_name())
self.local_router.start()
self.local_router.ready.wait(timeout=2)
self._capabilities.append("LOCAL_ROUTER")
# Start ExchangeManager, which starts the node (broker connection)
self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.get_safe('container.sflow.enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node,
from_name=self.name,
service=self,
process=self)
cleanup = lambda _: self.proc_manager._cleanup_method(self.name, rsvc)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(name=self.name,
listeners=[rsvc],
service=self,
cleanup_method=cleanup)
self.proc_manager.proc_sup.ensure_ready(proc)
proc.start_listeners()
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id,
origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container (%s) started, OK.", self.id)
