def test_directory(self):
dsm = DatastoreManager()
directory = Directory(dsm)
self.addCleanup(directory.dir_store.delete_datastore)
root = directory.lookup("/")
self.assert_(root is not None)
self.assertEquals(directory.register("/","temp"), None)
# Create a node
root = directory.lookup("/temp")
self.assertEquals(root, {} )
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo":"awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo":"awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo":"ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0][0], "ION/BranchA/Z")
res_list = directory.find_by_value("/", attribute="resource_id", value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k", attribute="resource_id", value="rid7")
self.assertEquals(len(res_list), 1)
directory.close()
def create_org(self, org=None):
"""Creates an Org based on the provided object. The id string returned
is the internal id by which Org will be identified in the data store.
"""
# Only allow one root ION Org in the system
self._validate_resource_obj("org",
org,
RT.Org,
checks="noid,name,unique")
# If this governance identifier is not set, then set to a safe version of the org name.
if not org.org_governance_name:
org.org_governance_name = create_basic_identifier(org.name)
if not is_basic_identifier(org.org_governance_name):
raise BadRequest(
"The Org org_governance_name '%s' contains invalid characters"
% org.org_governance_name)
org_id, _ = self.rr.create(org)
# Instantiate a Directory for this Org
directory = Directory(orgname=org.name)
# Instantiate initial set of User Roles for this Org
self._create_org_roles(org_id)
return org_id
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
self._capabilities = []
# set container id and cc_agent name (as they are set in base class call)
self.id = get_default_container_id()
self.name = "cc_agent_%s" % self.id
Container.instance = self
from pyon.core import bootstrap
bootstrap.container_instance = self
log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Instantiate Directory and self-register
# Has the additional side effect of either
# bootstrapping the configuration into the
# directory or read the configuration based
# in the value of the auto_bootstrap setting
self.directory = Directory()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
log.debug("Container initialized, OK.")
def create_org(self, org=None):
"""Persists the provided Org object. The id string returned
is the internal id by which Org will be identified in the data store.
@param org Org
@retval org_id str
@throws BadRequest if object passed has _id or _rev attribute
"""
if not org:
raise BadRequest("The org parameter is missing")
#Only allow one root ION Org in the system
if org.name == ROOT_ION_ORG_NAME:
res_list, _ = self.clients.resource_registry.find_resources(
restype=RT.Org, name=ROOT_ION_ORG_NAME)
if len(res_list) > 0:
raise BadRequest('There can only be one Org named %s' %
ROOT_ION_ORG_NAME)
if not is_basic_identifier(org.name):
raise BadRequest(
"The Org name '%s' can only contain alphanumeric and underscore characters"
% org.name)
org_id, org_rev = self.clients.resource_registry.create(org)
org._id = org_id
org._rev = org_rev
#Instantiate a Directory for this Org
directory = Directory(orgname=org.name)
#Instantiate initial set of User Roles for this Org
manager_role = IonObject(RT.UserRole,
name=MANAGER_ROLE,
label='Org Manager',
description='Org Manager')
self.add_user_role(org_id, manager_role)
member_role = IonObject(RT.UserRole,
name=MEMBER_ROLE,
label='Org Member',
description='Org Member')
self.add_user_role(org_id, member_role)
return org_id
def load_to_datastore(self, path, key, content):
'''
Load data to datastore
'''
log.debug(' Adding ' + key + ' to the directory')
dir = Directory()
if not dir.lookup('/' + path):
dir.register('/', path)
dir.register('/' + path, key, data=content)
new_entry = dir.lookup('/' + path + '/' + key)
if content == new_entry['data']:
log.debug(key + ' has been added to the directory')
else:
print '\n\nError adding: ' + key + ' to the directory'
log.error('Error adding: ' + key + ' to the directory')
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources")
ds.delete_datastore()
ds.create_datastore()
directory = Directory(datastore_manager=dsm)
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
self.assert_("_design/directory" in objs)
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/", "temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {})
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo": "awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo": "awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo": "ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m",
direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
directory.close()
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError(
"Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s"
% self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {
'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name()
}
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit(
) # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
self._capabilities.append("EXCHANGE_CONNECTION")
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Instantiate Directory and self-register
self.directory = Directory()
self.directory.register("/Containers", self.id, cc_agent=self.name)
self.directory.register("/Containers/%s" % self.id, "Processes")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects",
DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# Start ExchangeManager, which starts the node (broker connection)
self.node, self.ioloop = self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.container.get('sflow', {}).get('enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node,
from_name=self.name,
service=self,
process=self)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(
(CFG.cc.proctype or 'green', None), listener=rsvc)
self.proc_manager.proc_sup.ensure_ready(proc)
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id,
origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container started, OK.")
def test_directory(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources", "DIRECTORY")
ds.delete_datastore()
ds.create_datastore()
self.patch_cfg('pyon.ion.directory.CFG',
{'service': {
'directory': {
'publish_events': False
}
}})
directory = Directory(datastore_manager=dsm)
directory.start()
#self.addCleanup(directory.dir_store.delete_datastore)
objs = directory.dir_store.list_objects()
root = directory.lookup("/DIR")
self.assert_(root is not None)
entry = directory.lookup("/temp")
self.assert_(entry is None)
entry_old = directory.register("/", "temp")
self.assertEquals(entry_old, None)
# Create a node
entry = directory.lookup("/temp")
self.assertEquals(entry, {})
# The create case
entry_old = directory.register("/temp", "entry1", foo="awesome")
self.assertEquals(entry_old, None)
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, {"foo": "awesome"})
# The update case
entry_old = directory.register("/temp", "entry1", foo="ingenious")
self.assertEquals(entry_old, {"foo": "awesome"})
# The delete case
entry_old = directory.unregister("/temp", "entry1")
self.assertEquals(entry_old, {"foo": "ingenious"})
entry_new = directory.lookup("/temp/entry1")
self.assertEquals(entry_new, None)
directory.register("/BranchA", "X", resource_id="rid1")
directory.register("/BranchA", "Y", resource_id="rid2")
directory.register("/BranchA", "Z", resource_id="rid3")
directory.register("/BranchA/X", "a", resource_id="rid4")
directory.register("/BranchA/X", "b", resource_id="rid5")
directory.register("/BranchB", "k", resource_id="rid6")
directory.register("/BranchB", "l", resource_id="rid7")
directory.register("/BranchB/k", "m", resource_id="rid7")
directory.register("/BranchB/k", "X")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid3")
self.assertEquals(len(res_list), 1)
self.assertEquals(res_list[0].org, "ION")
self.assertEquals(res_list[0].parent, "/BranchA")
self.assertEquals(res_list[0].key, "Z")
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid34")
self.assertEquals(len(res_list), 0)
res_list = directory.find_by_value("/",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/Branch",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_value("/BranchB/k",
attribute="resource_id",
value="rid7")
self.assertEquals(len(res_list), 1)
res_list = directory.find_child_entries("/BranchB/k/m")
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB")
self.assertEquals(len(res_list), 2)
res_list = directory.find_child_entries("/BranchB/k/m",
direct_only=False)
self.assertEquals(len(res_list), 0)
res_list = directory.find_child_entries("/BranchB", direct_only=False)
self.assertEquals(len(res_list), 4)
res_list = directory.find_by_key("X")
self.assertEquals(len(res_list), 2)
res_list = directory.find_by_key("X", parent="/BranchB")
self.assertEquals(len(res_list), 1)
entry_list = directory.lookup_mult("/BranchA", ["X", "Z"])
self.assertEquals(len(entry_list), 2)
self.assertEquals(entry_list[0]["resource_id"], "rid1")
self.assertEquals(entry_list[1]["resource_id"], "rid3")
entry_list = directory.lookup_mult("/BranchA", ["Y", "FOO"])
self.assertEquals(len(entry_list), 2)
self.assertEquals(entry_list[0]["resource_id"], "rid2")
self.assertEquals(entry_list[1], None)
# Test prevent duplicate entries
directory.register("/some", "dupentry", foo="ingenious")
de = directory.lookup("/some/dupentry", return_entry=True)
de1_attrs = de.__dict__.copy()
del de1_attrs["_id"]
del de1_attrs["_rev"]
del de1_attrs["type_"]
de1 = DirEntry(**de1_attrs)
with self.assertRaises(BadRequest) as ex:
de_id1, _ = directory.dir_store.create(de1)
self.assertTrue(ex.message.startswith("DirEntry already exists"))
res_list = directory.find_by_key("dupentry", parent="/some")
self.assertEquals(1, len(res_list))
def test_directory_lock(self):
dsm = DatastoreManager()
ds = dsm.get_datastore("resources", "DIRECTORY")
ds.delete_datastore()
ds.create_datastore()
self.patch_cfg('pyon.ion.directory.CFG',
{'service': {
'directory': {
'publish_events': False
}
}})
directory = Directory(datastore_manager=dsm)
directory.start()
lock1 = directory.acquire_lock("LOCK1",
lock_info=dict(process="proc1"))
self.assertEquals(lock1, True)
lock2 = directory.acquire_lock("LOCK1",
lock_info=dict(process="proc2"))
self.assertEquals(lock2, False)
with self.assertRaises(BadRequest):
directory.acquire_lock("LOCK/SOME")
with self.assertRaises(BadRequest):
directory.release_lock("LOCK/SOME")
with self.assertRaises(NotFound):
directory.release_lock("LOCK2")
directory.release_lock("LOCK1")
lock1 = directory.acquire_lock("LOCK1",
lock_info=dict(process="proc3"))
self.assertEquals(lock1, True)
# TEST: With lock holders
lock5 = directory.acquire_lock("LOCK5", lock_holder="proc1")
self.assertEquals(lock5, True)
lock5 = directory.acquire_lock("LOCK5", lock_holder="proc1")
self.assertEquals(lock5, True)
lock5 = directory.acquire_lock("LOCK5", lock_holder="proc2")
self.assertEquals(lock5, False)
directory.release_lock("LOCK5")
# TEST: Timeout
lock5 = directory.acquire_lock("LOCK5",
lock_holder="proc1",
timeout=0.1)
self.assertEquals(lock5, True)
lock5 = directory.acquire_lock("LOCK5", lock_holder="proc2")
self.assertEquals(lock5, False)
res = directory.is_locked("LOCK5")
self.assertEquals(res, True)
gevent.sleep(0.15)
res = directory.is_locked("LOCK5")
self.assertEquals(res, False)
lock5 = directory.acquire_lock("LOCK5",
lock_holder="proc2",
timeout=0.1)
self.assertEquals(lock5, True)
gevent.sleep(0.15)
# TEST: Holder self renew
lock5 = directory.acquire_lock("LOCK5",
lock_holder="proc2",
timeout=0.1)
self.assertEquals(lock5, True)
directory.stop()
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
# set container id and cc_agent name (as they are set in base class call)
self.id = get_default_container_id()
self.name = "cc_agent_%s" % self.id
self._capabilities = []
bootstrap.container_instance = self
Container.instance = self
log.debug("Container (sysname=%s) initializing ..." %
bootstrap.get_sys_name())
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
# TODO: Do not start a capability here. Symmetric start/stop
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Instantiate Directory
self.directory = Directory()
# internal router
self.local_router = None
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
# publisher, initialized in start()
self.event_pub = None
# context-local storage
self.context = LocalContextMixin()
log.debug("Container initialized, OK.")
