def test_directory(self):
directory_service = Directory(DatastoreManager())
root = directory_service.lookup("/")
self.assertEquals(root, None)
self.assertEquals(directory_service.register("/","temp"), None)
# Create a node
root = directory_service.lookup("/temp")
self.assertEquals(root, {} )
# The create case
entry_old = directory_service.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory_service.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory_service.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo":"awesome"})
# The delete case
entry_old = directory_service.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo":"ingenious"})
entry_new = directory_service.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
def load_to_datastore (self, path, key, content):
'''
Load data to datastore
'''
log.debug(' Adding ' + key +' to the directory')
dir = Directory()
if not dir.lookup('/' + path):
dir.register('/',path)
dir.register ('/' + path, key, data=content)
new_entry = dir.lookup ('/' + path + '/' + key)
if content == new_entry['data']:
log.debug(key + ' has been added to the directory')
else:
print '\n\nError adding: '+ key +' to the directory'
log.error('Error adding: '+ key +' to the directory')
def load_to_datastore(self, path, key, content):
'''
Load data to datastore
'''
log.debug(' Adding ' + key + ' to the directory')
dir = Directory()
if not dir.lookup('/' + path):
dir.register('/', path)
dir.register('/' + path, key, data=content)
new_entry = dir.lookup('/' + path + '/' + key)
if content == new_entry['data']:
log.debug(key + ' has been added to the directory')
else:
print '\n\nError adding: ' + key + ' to the directory'
log.error('Error adding: ' + key + ' to the directory')
class Container(BaseContainerAgent):
"""
The Capability Container. Its purpose is to spawn/monitor processes and services
that do the bulk of the work in the ION system. It also manages connections to the Exchange
and the various forms of datastores in the systems.
"""
# Singleton static variables
#node = None
id = None
name = None
pidfile = None
instance = None
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
self._capabilities = []
# set container id and cc_agent name (as they are set in base class call)
self.id = get_default_container_id()
self.name = "cc_agent_%s" % self.id
Container.instance = self
from pyon.core import bootstrap
bootstrap.container_instance = self
log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Instantiate Directory and self-register
# Has the additional side effect of either
# bootstrapping the configuration into the
# directory or read the configuration based
# in the value of the auto_bootstrap setting
self.directory = Directory()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
log.debug("Container initialized, OK.")
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError("Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s" % self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name() }
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit() # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Self-register with Directory
self.directory.register("/Containers", self.id, cc_agent=self.name)
self.directory.register("/Containers/%s" % self.id, "Processes")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects", DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# Start ExchangeManager, which starts the node (broker connection)
self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.container.get('sflow', {}).get('enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node, from_name=self.name, service=self, process=self)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(name=self.name, listeners=[rsvc], service=self)
self.proc_manager.proc_sup.ensure_ready(proc)
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container started, OK.")
@property
def node(self):
"""
Returns the active/default Node that should be used for most communication in the system.
Defers to exchange manager, but only if it has been started, otherwise returns None.
"""
if "EXCHANGE_MANAGER" in self._capabilities:
return self.ex_manager.default_node
return None
@contextmanager
def _push_status(self, new_status):
"""
Temporarily sets the internal status flag.
Use this as a decorator or in a with-statement before calling a temporary status changing
method, like start_rel_from_url.
"""
curstatus = self._status
self._status = new_status
try:
yield
finally:
self._status = curstatus
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# serve forever short-circuits if immediate is on and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if not (immediate and num_procs == 1): # only spawned greenlet is the CC-Agent
# print a warning just in case
if immediate and num_procs != 1:
log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info('Received a kill signal, shutting down the container.')
watch_parent = CFG.system.get('watch_parent', None)
if watch_parent:
watch_parent.kill()
except:
log.exception('Unhandled error! Forcing container shutdown')
else:
log.debug("Container.serve_forever short-circuiting due to CFG.system.immediate")
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def status(self):
"""
Returns the internal status.
"""
return self._status
def _cleanup_pid(self):
if self.pidfile:
log.debug("Cleanup pidfile: %s", self.pidfile)
try:
os.remove(self.pidfile)
except Exception, e:
log.warn("Pidfile could not be deleted: %s" % str(e))
self.pidfile = None
class Container(BaseContainerAgent):
"""
The Capability Container. Its purpose is to spawn/monitor processes and services
that do the bulk of the work in the ION system.
"""
# Singleton static variables
node = None
id = None
name = None
pidfile = None
instance = None
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set id and name (as they are set in base class call)
self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
self.name = "cc_agent_%s" % self.id
Container.instance = self
# TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
dict_merge(CFG, kwargs, inplace=True)
from pyon.core import bootstrap
bootstrap.container_instance = self
bootstrap.assert_configuration(CFG)
bootstrap.sys_name = CFG.system.name or bootstrap.sys_name
log.debug("Container (sysname=%s) initializing ..." % bootstrap.sys_name)
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Load object and service registry etc.
bootstrap_pyon()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Coordinates the container start
self._is_started = False
self._capabilities = []
log.debug("Container initialized, OK.")
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError("Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s" % self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
from pyon.core.bootstrap import get_sys_name
pid_contents = {'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': get_sys_name() }
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit() # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
self._capabilities.append("EXCHANGE_CONNECTION")
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Instantiate Directory and self-register
self.directory = Directory()
self.directory.register("/Containers", self.id, cc_agent=self.name)
self._capabilities.append("DIRECTORY")
# Create other repositories to make sure they are there and clean if needed
self.datastore_manager.get_datastore("resources", DataStore.DS_PROFILE.RESOURCES)
self.datastore_manager.get_datastore("objects", DataStore.DS_PROFILE.OBJECTS)
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
self.event_repository = EventRepository()
self._capabilities.append("EVENT_REPOSITORY")
# Start ExchangeManager. In particular establish broker connection
self.ex_manager.start()
# TODO: Move this in ExchangeManager - but there is an error
self.node, self.ioloop = messaging.make_node() # TODO: shortcut hack
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node, name=self.name, service=self, process=self)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc)
self.proc_manager.proc_sup.ensure_ready(proc)
self._capabilities.append("CONTAINER_AGENT")
self._is_started = True
log.info("Container started, OK.")
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info('Received a kill signal, shutting down the container.')
except:
log.exception('Unhandled error! Forcing container shutdown')
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def _cleanup_pid(self):
if self.pidfile:
log.debug("Cleanup pidfile: %s", self.pidfile)
try:
os.remove(self.pidfile)
except Exception, e:
log.warn("Pidfile could not be deleted: %s" % str(e))
self.pidfile = None
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources")
ds.delete_datastore()
ds.create_datastore()
directory = Directory(datastore_manager=dsm)
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
self.assert_("_design/directory" in objs)
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/", "temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {})
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo": "awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo": "awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo": "ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m",
direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
directory.close()
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources")
ds.delete_datastore()
ds.create_datastore()
directory = Directory(datastore_manager=dsm)
directory.start()
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
self.assert_("_design/directory" in objs)
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/","temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {} )
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo":"awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo":"ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m", direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
directory.stop()
def test_directory(self):
dsm = DatastoreManager()
directory = Directory(dsm)
root = directory.lookup("/")
self.assert_(root is not None)
self.assertEquals(directory.register("/","temp"), None)
# Create a node
root = directory.lookup("/temp")
self.assertEquals(root, {} )
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo":"awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo":"ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0][0], "ION/BranchA/Z")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 1)
directory.close()
class Container(BaseContainerAgent):
"""
The Capability Container. Its purpose is to spawn/monitor processes and services
that do the bulk of the work in the ION system.
"""
# Singleton static variables
node = None
id = None
name = None
pidfile = None
instance = None
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set id and name (as they are set in base class call)
self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
self.name = "cc_agent_%s" % self.id
Container.instance = self
# TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
dict_merge(CFG, kwargs, inplace=True)
from pyon.core import bootstrap
bootstrap.container_instance = self
bootstrap.assert_configuration(CFG)
log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Load object and service registry etc.
bootstrap_pyon()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._is_started = False
self._capabilities = []
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
log.debug("Container initialized, OK.")
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError("Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s" % self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name() }
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit() # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
self._capabilities.append("EXCHANGE_CONNECTION")
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Instantiate Directory and self-register
self.directory = Directory()
self.directory.register("/Containers", self.id, cc_agent=self.name)
self.directory.register("/Containers/%s" % self.id, "Processes")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects", DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# Start ExchangeManager, which starts the node (broker connection)
self.node, self.ioloop = self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.container.get('sflow', {}).get('enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node, from_name=self.name, service=self, process=self)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn((CFG.cc.proctype or 'green', None), listener=rsvc)
self.proc_manager.proc_sup.ensure_ready(proc)
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container started, OK.")
@contextmanager
def _push_status(self, new_status):
"""
Temporarily sets the internal status flag.
Use this as a decorator or in a with-statement before calling a temporary status changing
method, like start_rel_from_url.
"""
curstatus = self._status
self._status = new_status
try:
yield
finally:
self._status = curstatus
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# serve forever short-circuits if immediate is on and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if not (immediate and num_procs == 1): # only spawned greenlet is the CC-Agent
# print a warning just in case
if immediate and num_procs != 1:
log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info('Received a kill signal, shutting down the container.')
watch_parent = CFG.system.get('watch_parent', None)
if watch_parent:
watch_parent.kill()
except:
log.exception('Unhandled error! Forcing container shutdown')
else:
log.debug("Container.serve_forever short-circuiting due to CFG.system.immediate")
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def status(self):
"""
Returns the internal status.
"""
return self._status
def _cleanup_pid(self):
if self.pidfile:
log.debug("Cleanup pidfile: %s", self.pidfile)
try:
os.remove(self.pidfile)
except Exception, e:
log.warn("Pidfile could not be deleted: %s" % str(e))
self.pidfile = None
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources", "DIRECTORY")
ds.delete_datastore()
ds.create_datastore()
self.patch_cfg('pyon.ion.directory.CFG', {'service': {'directory': {'publish_events': False}}})
directory = Directory(datastore_manager=dsm)
directory.start()
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/", "temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {} )
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo": "awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo": "ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m", direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
entry_list = directory.lookup_mult("/BranchA", ["X", "Z"])
self.assertEquals(len(entry_list), 2)
self.assertEquals(entry_list[0]["resource_id"], "rid1")
self.assertEquals(entry_list[1]["resource_id"], "rid3")
entry_list = directory.lookup_mult("/BranchA", ["Y", "FOO"])
self.assertEquals(len(entry_list), 2)
self.assertEquals(entry_list[0]["resource_id"], "rid2")
self.assertEquals(entry_list[1], None)
# Test prevent duplicate entries
directory.register("/some", "dupentry", foo="ingenious")
de = directory.lookup("/some/dupentry", return_entry=True)
de1_attrs = de.__dict__.copy()
del de1_attrs["_id"]
del de1_attrs["_rev"]
del de1_attrs["type_"]
de1 = DirEntry(**de1_attrs)
with self.assertRaises(BadRequest) as ex:
de_id1,_ = directory.dir_store.create(de1)
self.assertTrue(ex.message.startswith("DirEntry already exists"))
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(1, len(res_list))
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources", "DIRECTORY")
ds.delete_datastore()
ds.create_datastore()
directory = Directory(datastore_manager=dsm)
directory.start()
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
if CFG.get_safe("container.datastore.default_server", "couchdb").startswith("couch"):
self.assert_("_design/directory" in objs)
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/","temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {} )
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo":"awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo":"ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m", direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
# Test _cleanup_outdated_entries
directory.register("/some", "dupentry", foo="ingenious")
de = directory.lookup("/some/dupentry", return_entry=True)
de1_attrs = de.__dict__.copy()
del de1_attrs["_id"]
del de1_attrs["_rev"]
del de1_attrs["type_"]
de1 = DirEntry(**de1_attrs)
de_id1,_ = directory.dir_store.create(de1)
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(2, len(res_list))
de = directory.lookup("/some/dupentry", return_entry=True)
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(1, len(res_list))
de1_attrs = de.__dict__.copy()
del de1_attrs["_id"]
del de1_attrs["_rev"]
del de1_attrs["type_"]
de1_attrs["ts_updated"] = str(int(de1_attrs["ts_updated"]) + 10)
de1_attrs["attributes"]["unique"] = "NEW"
de1 = DirEntry(**de1_attrs)
de_id1,_ = directory.dir_store.create(de1)
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(2, len(res_list))
de = directory.lookup("/some/dupentry", return_entry=True)
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(1, len(res_list))
self.assertEquals("NEW", res_list[0].attributes["unique"])
directory.stop()
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources", "DIRECTORY")
ds.delete_datastore()
ds.create_datastore()
self.patch_cfg('pyon.ion.directory.CFG',
{'service': {
'directory': {
'publish_events': False
}
}})
directory = Directory(datastore_manager=dsm)
directory.start()
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/", "temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {})
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo": "awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo": "awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo": "ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m",
direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
entry_list = directory.lookup_mult("/BranchA", ["X", "Z"])
self.assertEquals(len(entry_list), 2)
self.assertEquals(entry_list[0]["resource_id"], "rid1")
self.assertEquals(entry_list[1]["resource_id"], "rid3")
entry_list = directory.lookup_mult("/BranchA", ["Y", "FOO"])
self.assertEquals(len(entry_list), 2)
self.assertEquals(entry_list[0]["resource_id"], "rid2")
self.assertEquals(entry_list[1], None)
# Test prevent duplicate entries
directory.register("/some", "dupentry", foo="ingenious")
de = directory.lookup("/some/dupentry", return_entry=True)
de1_attrs = de.__dict__.copy()
del de1_attrs["_id"]
del de1_attrs["_rev"]
del de1_attrs["type_"]
de1 = DirEntry(**de1_attrs)
with self.assertRaises(BadRequest) as ex:
de_id1, _ = directory.dir_store.create(de1)
self.assertTrue(ex.message.startswith("DirEntry already exists"))
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(1, len(res_list))
def test_directory(self):
dsm = DatastoreManager()
directory = Directory(dsm)
self.addCleanup(directory.dir_store.delete_datastore)
root = directory.lookup("/")
self.assert_(root is not None)
self.assertEquals(directory.register("/","temp"), None)
# Create a node
root = directory.lookup("/temp")
self.assertEquals(root, {} )
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo":"awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo":"ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0][0], "ION/BranchA/Z")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 1)
directory.close()