class InterfaceAdminTest(IonIntegrationTestCase):
def setUp(self):
self.dir = DirectoryStandalone(sysname=bootstrap.get_sys_name(),
config=bootstrap.CFG)
self.rr = ResourceRegistryStandalone(sysname=bootstrap.get_sys_name(),
config=bootstrap.CFG)
self.iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap.CFG)
self.addCleanup(self.iadm.close)
def test_store_core(self):
# Store system CFG properties
ion_config = config.read_standard_configuration()
self.iadm.store_config(ion_config)
# Validate the CFG entries are stored in DB
entries = self.dir.lookup('/Config/CFG')
self.assertTrue(entries)
def test_store_interfaces(self):
self.iadm.store_interfaces()
# Validate there are entries for ObjectType
entries = self.rr.find_by_type("ObjectType", id_only=True)
self.assertGreater(len(entries), 30)
# Validate there are entries for ServiceInterfaces
entries = self.rr.find_by_type("ServiceDefinition", id_only=True)
self.assertGreater(len(entries), 30)
def __init__(self, sysname, config=None):
self.sysname = sysname
self.config = config
self.dir = DirectoryStandalone(sysname=self.sysname, config=config)
self.rr = ResourceRegistryStandalone(sysname=self.sysname,
config=config)
self._closed = False
class InterfaceAdminTest(IonIntegrationTestCase):
def setUp(self):
self.dir = DirectoryStandalone(sysname=bootstrap.get_sys_name(), config=bootstrap.CFG)
self.rr = ResourceRegistryStandalone(sysname=bootstrap.get_sys_name(), config=bootstrap.CFG)
self.iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap.CFG)
self.addCleanup(self.iadm.close)
def test_store_core(self):
# Store system CFG properties
ion_config = config.read_standard_configuration()
self.iadm.store_config(ion_config)
# Validate the CFG entries are stored in DB
entries = self.dir.lookup('/Config/CFG')
self.assertTrue(entries)
def test_store_interfaces(self):
self.iadm.store_interfaces()
# Validate there are entries for ObjectType
entries = self.rr.find_by_type("ObjectType", id_only=True)
self.assertGreater(len(entries), 30)
# Validate there are entries for ServiceInterfaces
entries = self.rr.find_by_type("ServiceDefinition", id_only=True)
self.assertGreater(len(entries), 30)
def setUp(self):
self.dir = DirectoryStandalone(sysname=bootstrap.get_sys_name(),
config=bootstrap.CFG)
self.rr = ResourceRegistryStandalone(sysname=bootstrap.get_sys_name(),
config=bootstrap.CFG)
self.iadm = InterfaceAdmin(bootstrap.get_sys_name(),
config=bootstrap.CFG)
self.addCleanup(self.iadm.close)
def get_service_definition_from_datastore(sysname):
fragments = []
rr = ResourceRegistryStandalone(sysname=sysname)
svc_defs = rr.find_by_type('ServiceDefinition')
if not svc_defs:
return ''
for item in svc_defs:
try:
fragments.append(item['definition'])
except:
return ''
full_definition = "\n".join(fragments)
return full_definition
def get_service_definition_from_datastore(sysname):
fragments = []
rr = ResourceRegistryStandalone(sysname=sysname)
svc_defs = rr.find_by_type('ServiceDefinition')
if not svc_defs:
return ''
for item in svc_defs:
try:
fragments.append(item['definition'])
except:
return ''
full_definition = "\n".join(fragments)
return full_definition
def get_object_definition_from_datastore(sysname):
fragments = []
rr = ResourceRegistryStandalone(sysname=sysname)
obj_types = rr.find_by_type('ObjectType')
if not obj_types:
return ''
for item in obj_types:
try:
fragments.append((item['definition_order'], item['definition']))
except:
return ''
fragments = [item for ordinal, item in sorted(fragments)]
full_definition = "\n".join(fragments)
return full_definition
def get_object_definition_from_datastore(sysname):
fragments = []
rr = ResourceRegistryStandalone(sysname=sysname)
obj_types = rr.find_by_type('ObjectType')
if not obj_types:
return ''
for item in obj_types:
try:
fragments.append((item['definition_order'],item['definition']))
except:
return ''
fragments = [item for ordinal, item in sorted(fragments)]
full_definition = "\n".join(fragments)
return full_definition
def __init__(self, sysname, config=None):
self._closed = False
self.sysname = sysname
self.config = config
self.dir = DirectoryStandalone(sysname=self.sysname, config=self.config)
self.rr = ResourceRegistryStandalone(sysname=self.sysname, config=self.config)
class InterfaceAdmin:
"""
Administrates work with interfaces in the datastore
"""
DIR_RESFILES_PATH = "/Config"
DIR_CONFIG_PATH = "/Config"
def __init__(self, sysname, config=None):
self._closed = False
self.sysname = sysname
self.config = config
self.dir = DirectoryStandalone(sysname=self.sysname, config=self.config)
self.rr = ResourceRegistryStandalone(sysname=self.sysname, config=self.config)
def close(self):
self.dir.close()
self.dir = None
self.rr.close()
self.rr = None
self._closed = True
def __del__(self):
if not self._closed:
print "WARNING: Call close() on InterfaceAdmin (and datastores)"
def create_core_datastores(self):
"""
Main entry point into creating core datastores
"""
ds = DatastoreFactory.get_datastore(config=self.config, scope=self.sysname, variant=DatastoreFactory.DS_BASE)
datastores = [
('resources', 'RESOURCES'),
('events', 'EVENTS'),
('state', 'STATE'),
('objects', 'OBJECTS'),
]
count = 0
ds_created = []
for local_dsn, profile in datastores:
if not ds.datastore_exists(local_dsn):
ds.create_datastore(datastore_name=local_dsn, profile=profile)
count += 1
ds_created.append(local_dsn)
print "store_interfaces: Created %s datastores: %s" % (count, ds_created)
def store_config(self, system_cfg):
"""
Main entry point into storing system config
"""
de = self.dir.lookup(self.DIR_CONFIG_PATH + "/CFG")
if de:
#print "store_interfaces: WARN: Config already exists. Overwrite"
return
print "store_interfaces: Storing system config in directory..."
self.dir.register(self.DIR_CONFIG_PATH, "CFG", **deepcopy(system_cfg))
def store_interfaces(self, object_definition_file=None,
service_definition_file=None, idempotent=False):
"""
Main entry point into storing interfaces
"""
self.idempotent = idempotent
self.bulk_entries = {}
self.bulk_resources = []
self.serial_num = 1
if object_definition_file:
self.store_object_interfaces(file=object_definition_file)
elif service_definition_file:
self.store_service_interfaces(file=service_definition_file)
else:
if self.idempotent:
de = self.rr.find_by_type("ServiceDefinition", id_only=True)
if de:
#print "store_interfaces: Interfaces already stored. Ignoring"
return
# load all files
self.store_object_interfaces()
self.store_service_interfaces()
self.store_config_files()
self._register_bulk()
def store_object_interfaces(self, file=None):
#print "\nStoring object interfaces in datastore..."
if file and os.path.exists(file):
self._load_object_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
data_yaml_filenames = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self._load_object_files(data_yaml_filenames)
def _load_object_files(self, filenames):
serial_num = 1
for file_path in filenames:
objs = OrderedDict()
class_content = ''
# Open and read data model definition file
with open(file_path, 'r') as f:
file_content_str = f.read()
first_time = True
for line in file_content_str.split('\n'):
if len(line) > 0 and (line[0].isalpha()):
if not first_time:
objs[classname] = class_content
class_content = ""
else:
first_time = False
classname = line.split(':')[0]
class_content += line + "\n"
else:
class_content += line + "\n"
objs[classname] = class_content
for key in objs.keys():
obj_type = self._create_object_type(key, objs[key], serial_num)
self.bulk_resources.append(obj_type)
serial_num += 1
def store_service_interfaces(self, file=None):
#print "\nStoring service interfaces in datastore..."
if file and os.path.exists(file):
self._load_service_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
service_yaml_filenames = list_files_recursive('obj/services', '*.yml')
self._load_service_files(service_yaml_filenames)
def _load_service_files(self, filenames):
for file_path in filenames:
objs = {}
with open(file_path, 'r') as f:
file_content_str = f.read()
for line in file_content_str.split('\n'):
if line[:5] == "name:":
key = line.split()[1]
objs[key] = file_content_str
if not objs:
print "\n\n=====ERROR===== Can't find object name for: ", file_path
for key in objs.keys():
svc_def = self._create_service_definition(key, objs[key], file_path)
self.bulk_resources.append(svc_def)
def store_config_files(self):
#print "\nStoring system res files in datastore..."
resource_filenames = list_files_recursive('res/config', '*.yml')
self._load_config_files(resource_filenames)
def _load_config_files(self, filenames):
for file_path in filenames:
with open(file_path, 'r') as f:
file_content_str = f.read()
objs = {}
objs[os.path.basename(file_path)] = file_content_str
for key in objs.keys():
self.bulk_entries[(self.DIR_RESFILES_PATH, key)] = dict(file_path=file_path, definition=objs[key])
def _create_object_type(self, name, definition, definition_order):
return dict(type_="ObjectType", name=name, definition=definition, definition_order=definition_order)
def _create_service_definition(self, name, definition, namespace):
return dict(type_="ServiceDefinition", name=name, definition=definition, namespace=namespace)
def _register_bulk(self):
print "store_interfaces: Storing %s entries in directory..." % len(self.bulk_entries)
entries = [(path, key, attrs) for ((path, key), attrs) in self.bulk_entries.iteritems()]
res = self.dir.register_mult(entries)
self.bulk_entries = {}
print "store_interfaces: Storing %s resources in registry..." % len(self.bulk_resources)
res = self.rr.create_mult(self.bulk_resources)
self.bulk_resources = []
print "store_interfaces: Storing interfaces successful"
def setUp(self):
self.dir = DirectoryStandalone(sysname=bootstrap.get_sys_name(), config=bootstrap.CFG)
self.rr = ResourceRegistryStandalone(sysname=bootstrap.get_sys_name(), config=bootstrap.CFG)
self.iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap.CFG)
self.addCleanup(self.iadm.close)
class InterfaceAdmin:
"""
Administrates work with interfaces in the datastore
"""
DIR_RESFILES_PATH = "/Config"
DIR_CONFIG_PATH = "/Config"
def __init__(self, sysname, config=None):
self.sysname = sysname
self.config = config
self.dir = DirectoryStandalone(sysname=self.sysname, config=config)
self.rr = ResourceRegistryStandalone(sysname=self.sysname,
config=config)
self._closed = False
def close(self):
self.dir.close()
self.dir = None
self.rr.close()
self.rr = None
self._closed = True
def __del__(self):
if not self._closed:
print "WARNING: Call close() on InterfaceAdmin (and datastores)"
def create_core_datastores(self):
"""
Main entry point into creating core datastores
"""
ds = CouchDataStore(config=self.config, scope=self.sysname)
datastores = ['resources']
for local_dsn in datastores:
if not ds.exists_datastore(local_dsn):
ds.create_datastore(local_dsn)
# NOTE: Views and other datastores are created by containers' DatastoreManager
def store_config(self, system_cfg):
"""
Main entry point into storing system config
"""
de = self.dir.lookup(self.DIR_CONFIG_PATH + "/CFG")
if de:
#print "store_interfaces: WARN: Config already exists. Overwrite"
return
print "store_interfaces: Storing system config in directory..."
self.dir.register(self.DIR_CONFIG_PATH, "CFG", **deepcopy(system_cfg))
def store_interfaces(self,
object_definition_file=None,
service_definition_file=None,
idempotent=False):
"""
Main entry point into storing interfaces
"""
self.idempotent = idempotent
self.bulk_entries = {}
self.bulk_resources = []
self.serial_num = 1
if object_definition_file:
self.store_object_interfaces(file=object_definition_file)
elif service_definition_file:
self.store_service_interfaces(file=service_definition_file)
else:
if self.idempotent:
de = self.rr.find_by_type("ServiceDefinition", id_only=True)
if de:
#print "store_interfaces: Interfaces already stored. Ignoring"
return
# load all files
self.store_object_interfaces()
self.store_service_interfaces()
self.store_config_files()
self._register_bulk()
def store_object_interfaces(self, file=None):
#print "\nStoring object interfaces in datastore..."
if file and os.path.exists(file):
self._load_object_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
data_yaml_filenames = list_files_recursive(
'obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self._load_object_files(data_yaml_filenames)
def _load_object_files(self, filenames):
serial_num = 1
for file_path in filenames:
objs = OrderedDict()
class_content = ''
# Open and read data model definition file
with open(file_path, 'r') as f:
file_content_str = f.read()
first_time = True
for line in file_content_str.split('\n'):
if len(line) > 0 and (line[0].isalpha()):
if not first_time:
objs[classname] = class_content
class_content = ""
else:
first_time = False
classname = line.split(':')[0]
class_content += line + "\n"
else:
class_content += line + "\n"
objs[classname] = class_content
for key in objs.keys():
obj_type = self._create_object_type(key, objs[key], serial_num)
self.bulk_resources.append(obj_type)
serial_num += 1
def store_service_interfaces(self, file=None):
#print "\nStoring service interfaces in datastore..."
if file and os.path.exists(file):
self._load_service_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
service_yaml_filenames = list_files_recursive(
'obj/services', '*.yml')
self._load_service_files(service_yaml_filenames)
def _load_service_files(self, filenames):
for file_path in filenames:
objs = {}
with open(file_path, 'r') as f:
file_content_str = f.read()
for line in file_content_str.split('\n'):
if line[:5] == "name:":
key = line.split()[1]
objs[key] = file_content_str
if not objs:
print "\n\n=====ERROR===== Can't find object name for: ", file_path
for key in objs.keys():
svc_def = self._create_service_definition(
key, objs[key], file_path)
self.bulk_resources.append(svc_def)
def store_config_files(self):
#print "\nStoring system res files in datastore..."
resource_filenames = list_files_recursive('res/config', '*.yml')
self._load_config_files(resource_filenames)
def _load_config_files(self, filenames):
for file_path in filenames:
with open(file_path, 'r') as f:
file_content_str = f.read()
objs = {}
objs[os.path.basename(file_path)] = file_content_str
for key in objs.keys():
self.bulk_entries[(self.DIR_RESFILES_PATH,
key)] = dict(file_path=file_path,
definition=objs[key])
def _create_object_type(self, name, definition, definition_order):
return dict(type_="ObjectType",
name=name,
definition=definition,
definition_order=definition_order)
def _create_service_definition(self, name, definition, namespace):
return dict(type_="ServiceDefinition",
name=name,
definition=definition,
namespace=namespace)
def _register_bulk(self):
print "store_interfaces: Storing %s entries in directory..." % len(
self.bulk_entries)
entries = [(path, key, attrs)
for ((path, key), attrs) in self.bulk_entries.iteritems()]
res = self.dir.register_mult(entries)
self.bulk_entries = {}
print "store_interfaces: Storing %s resources in registry..." % len(
self.bulk_resources)
res = self.rr.create_mult(self.bulk_resources)
self.bulk_resources = []
print "store_interfaces: Storing interfaces successful"
class InterfaceAdmin:
"""
Administrates work with interfaces in the datastore
"""
DIR_RESFILES_PATH = "/Config"
DIR_CONFIG_PATH = "/Config"
def __init__(self, sysname, config=None):
self._closed = False
self.sysname = sysname
self.config = config
self.dir = DirectoryStandalone(sysname=self.sysname,
config=self.config)
self.rr = ResourceRegistryStandalone(sysname=self.sysname,
config=self.config)
def close(self):
self.dir.close()
self.dir = None
self.rr.close()
self.rr = None
self._closed = True
def __del__(self):
if not self._closed:
log.warn(
"WARNING: Call close() on InterfaceAdmin (and datastores)")
def set_config(self, config):
self.config = config
def system_data_exists(self):
""" Returns True if there is persistent system data in the database - this may be a restart. """
actors = self.rr.find_by_type("ActorIdentity", id_only=True, limit=1)
return bool(actors)
def create_core_datastores(self):
""" Create core datastores, so that they exist when containers start concurrently. """
ds = DatastoreFactory.get_datastore(config=self.config,
scope=self.sysname,
variant=DatastoreFactory.DS_BASE)
datastores = [
('resources', 'RESOURCES'),
('events', 'EVENTS'),
('state', 'STATE'),
('objects', 'OBJECTS'),
]
count = 0
ds_created = []
for local_dsn, profile in datastores:
if not ds.datastore_exists(local_dsn):
ds.create_datastore(datastore_name=local_dsn, profile=profile)
count += 1
ds_created.append(local_dsn)
if count:
log.info("store_interfaces: Created %s datastores: %s" %
(count, ds_created))
def store_config(self, system_cfg):
"""
Store system config in directory and setup basic directory structure, so that everything is
prepared when containers start concurrently.
"""
# Register some default keys
self.dir.register_safe("/",
"DIR",
description="Directory root",
sys_name=self.sysname)
self.dir.register_safe("/",
"Agents",
description="Running agents",
create_only=True)
self.dir.register_safe("/",
"Config",
description="System configuration",
create_only=True)
self.dir.register_safe("/",
"System",
description="System management information",
create_only=True)
self.dir.register_safe("/System",
"Locks",
description="System exclusive locks",
create_only=True)
if get_safe(system_cfg, "bootstrap.store_config") is True:
de = self.dir.lookup(self.DIR_CONFIG_PATH + "/CFG")
if de:
log.debug(
"store_interfaces: Updating system config in directory...")
else:
log.info(
"store_interfaces: Storing system config in directory...")
self.dir.register(self.DIR_CONFIG_PATH, "CFG",
**deepcopy(system_cfg))
def store_interfaces(self,
object_definition_file=None,
service_definition_file=None,
idempotent=True):
"""
Store system interfaces (services, objects, config files), so that everything is
prepared when containers start concurrently.
"""
self.idempotent = idempotent
self.bulk_entries = {}
self.bulk_resources = []
self.serial_num = 1
if object_definition_file:
# Update single object definition
self.store_object_interfaces(file=object_definition_file)
elif service_definition_file:
# Update single service definition
self.store_service_interfaces(file=service_definition_file)
else:
# Update all interfaces
if self.idempotent:
de = self.rr.find_by_type("ServiceDefinition", id_only=True)
if de:
log.debug(
"store_interfaces: Interfaces already stored. Not updating."
)
return
# load all files
self.store_object_interfaces()
self.store_service_interfaces()
#self.store_config_files()
self._register_bulk()
def store_object_interfaces(self, file=None):
# log.info("Storing object interfaces in datastore...")
if file and os.path.exists(file):
self._load_object_files([file])
elif file:
log.warn("store_interfaces: Error couldn't find the file path")
else:
data_yaml_filenames = list_files_recursive(
'obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self._load_object_files(data_yaml_filenames)
def _load_object_files(self, filenames):
serial_num = 1
for file_path in filenames:
objs = OrderedDict()
class_content = ''
# Open and read data model definition file
with open(file_path, 'r') as f:
file_content_str = f.read()
first_time = True
for line in file_content_str.split('\n'):
if len(line) > 0 and (line[0].isalpha()):
if not first_time:
objs[classname] = class_content
class_content = ""
else:
first_time = False
classname = line.split(':')[0]
class_content += line + "\n"
else:
class_content += line + "\n"
objs[classname] = class_content
for key in objs.keys():
obj_type = self._create_object_type(key, objs[key], serial_num)
#self.bulk_resources.append(obj_type)
serial_num += 1
def store_service_interfaces(self, file=None):
# log.info("Storing service interfaces in datastore...")
if file and os.path.exists(file):
self._load_service_files([file])
elif file:
log.warn("store_interfaces: Error couldn't find the file path")
else:
service_yaml_filenames = list_files_recursive(
'obj/services', '*.yml')
self._load_service_files(service_yaml_filenames)
def _load_service_files(self, filenames):
for file_path in filenames:
objs = {}
with open(file_path, 'r') as f:
file_content_str = f.read()
for line in file_content_str.split('\n'):
if line[:5] == "name:":
key = line.split()[1]
objs[key] = file_content_str
if not objs:
log.error("=====ERROR===== Can't find object name for: %s",
file_path)
for key in objs.keys():
svc_def = self._create_service_definition(
key, objs[key], file_path)
self.bulk_resources.append(svc_def)
def store_config_files(self):
# log.info("Storing system res files in datastore...")
resource_filenames = list_files_recursive('res/config', '*.yml')
self._load_config_files(resource_filenames)
def _load_config_files(self, filenames):
for file_path in filenames:
with open(file_path, 'r') as f:
file_content_str = f.read()
objs = {}
objs[os.path.basename(file_path)] = file_content_str
for key in objs.keys():
self.bulk_entries[(self.DIR_RESFILES_PATH,
key)] = dict(file_path=file_path,
definition=objs[key])
def _create_object_type(self, name, definition, definition_order):
return dict(type_="ObjectType",
name=name,
definition=definition,
definition_order=definition_order)
def _create_service_definition(self, name, definition, namespace):
return dict(type_="ServiceDefinition",
name=name,
definition=definition,
namespace=namespace)
def _register_bulk(self):
if self.bulk_entries:
log.info("store_interfaces: Storing %s entries in directory..." %
len(self.bulk_entries))
entries = [(path, key, attrs)
for ((path, key),
attrs) in self.bulk_entries.iteritems()]
res = self.dir.register_mult(entries)
self.bulk_entries = {}
if self.bulk_resources:
log.info("store_interfaces: Storing %s resources in registry..." %
len(self.bulk_resources))
res = self.rr.create_mult(self.bulk_resources)
self.bulk_resources = []
log.debug("store_interfaces: Storing interfaces successful")
def declare_core_exchange_resources(self):
"""
Create important messaging resources (queues, bindings, etc), so that they are ready before
the first containers start and processes get spawned concurrently.
"""
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname)
# Event persister queue
rabbit_util.declare_exchange("events")
rabbit_util.declare_queue("events", "event_persister")
rabbit_util.bind_queue("events", "event_persister", "#")
class InterfaceAdmin:
"""
Administrates work with interfaces in the datastore
"""
DIR_RESFILES_PATH = "/Config"
DIR_CONFIG_PATH = "/Config"
def __init__(self, sysname, config=None):
self._closed = False
self.sysname = sysname
self.config = config
self.dir = DirectoryStandalone(sysname=self.sysname, config=self.config)
self.rr = ResourceRegistryStandalone(sysname=self.sysname, config=self.config)
def close(self):
self.dir.close()
self.dir = None
self.rr.close()
self.rr = None
self._closed = True
def __del__(self):
if not self._closed:
log.warn("WARNING: Call close() on InterfaceAdmin (and datastores)")
def set_config(self, config):
self.config = config
def system_data_exists(self):
""" Returns True if there is persistent system data in the database - this may be a restart. """
actors = self.rr.find_by_type("ActorIdentity", id_only=True, limit=1)
return bool(actors)
def create_core_datastores(self):
""" Create core datastores, so that they exist when containers start concurrently. """
ds = DatastoreFactory.get_datastore(config=self.config, scope=self.sysname, variant=DatastoreFactory.DS_BASE)
datastores = [
('resources', 'RESOURCES'),
('events', 'EVENTS'),
('state', 'STATE'),
('objects', 'OBJECTS'),
]
count = 0
ds_created = []
for local_dsn, profile in datastores:
if not ds.datastore_exists(local_dsn):
ds.create_datastore(datastore_name=local_dsn, profile=profile)
count += 1
ds_created.append(local_dsn)
if count:
log.info("store_interfaces: Created %s datastores: %s" % (count, ds_created))
def store_config(self, system_cfg):
"""
Store system config in directory and setup basic directory structure, so that everything is
prepared when containers start concurrently.
"""
# Register some default keys
self.dir.register_safe("/", "DIR", description="Directory root", sys_name=self.sysname)
self.dir.register_safe("/", "Agents", description="Running agents", create_only=True)
self.dir.register_safe("/", "Config", description="System configuration", create_only=True)
self.dir.register_safe("/", "System", description="System management information", create_only=True)
self.dir.register_safe("/System", "Locks", description="System exclusive locks", create_only=True)
if get_safe(system_cfg, "bootstrap.store_config") is True:
de = self.dir.lookup(self.DIR_CONFIG_PATH + "/CFG")
if de:
log.debug("store_interfaces: Updating system config in directory...")
else:
log.info("store_interfaces: Storing system config in directory...")
self.dir.register(self.DIR_CONFIG_PATH, "CFG", **deepcopy(system_cfg))
def store_interfaces(self, object_definition_file=None, service_definition_file=None, idempotent=True):
"""
Store system interfaces (services, objects, config files), so that everything is
prepared when containers start concurrently.
"""
self.idempotent = idempotent
self.bulk_entries = {}
self.bulk_resources = []
self.serial_num = 1
if object_definition_file:
# Update single object definition
self.store_object_interfaces(file=object_definition_file)
elif service_definition_file:
# Update single service definition
self.store_service_interfaces(file=service_definition_file)
else:
# Update all interfaces
if self.idempotent:
de = self.rr.find_by_type("ServiceDefinition", id_only=True)
if de:
log.debug("store_interfaces: Interfaces already stored. Not updating.")
return
# load all files
self.store_object_interfaces()
self.store_service_interfaces()
#self.store_config_files()
self._register_bulk()
def store_object_interfaces(self, file=None):
# log.info("Storing object interfaces in datastore...")
if file and os.path.exists(file):
self._load_object_files([file])
elif file:
log.warn("store_interfaces: Error couldn't find the file path")
else:
data_yaml_filenames = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self._load_object_files(data_yaml_filenames)
def _load_object_files(self, filenames):
serial_num = 1
for file_path in filenames:
objs = OrderedDict()
class_content = ''
# Open and read data model definition file
with open(file_path, 'r') as f:
file_content_str = f.read()
first_time = True
for line in file_content_str.split('\n'):
if len(line) > 0 and (line[0].isalpha()):
if not first_time:
objs[classname] = class_content
class_content = ""
else:
first_time = False
classname = line.split(':')[0]
class_content += line + "\n"
else:
class_content += line + "\n"
objs[classname] = class_content
for key in objs.keys():
obj_type = self._create_object_type(key, objs[key], serial_num)
#self.bulk_resources.append(obj_type)
serial_num += 1
def store_service_interfaces(self, file=None):
# log.info("Storing service interfaces in datastore...")
if file and os.path.exists(file):
self._load_service_files([file])
elif file:
log.warn("store_interfaces: Error couldn't find the file path")
else:
service_yaml_filenames = list_files_recursive('obj/services', '*.yml')
self._load_service_files(service_yaml_filenames)
def _load_service_files(self, filenames):
for file_path in filenames:
objs = {}
with open(file_path, 'r') as f:
file_content_str = f.read()
for line in file_content_str.split('\n'):
if line[:5] == "name:":
key = line.split()[1]
objs[key] = file_content_str
if not objs:
log.error("=====ERROR===== Can't find object name for: %s", file_path)
for key in objs.keys():
svc_def = self._create_service_definition(key, objs[key], file_path)
self.bulk_resources.append(svc_def)
def store_config_files(self):
# log.info("Storing system res files in datastore...")
resource_filenames = list_files_recursive('res/config', '*.yml')
self._load_config_files(resource_filenames)
def _load_config_files(self, filenames):
for file_path in filenames:
with open(file_path, 'r') as f:
file_content_str = f.read()
objs = {}
objs[os.path.basename(file_path)] = file_content_str
for key in objs.keys():
self.bulk_entries[(self.DIR_RESFILES_PATH, key)] = dict(file_path=file_path, definition=objs[key])
def _create_object_type(self, name, definition, definition_order):
return dict(type_="ObjectType", name=name, definition=definition, definition_order=definition_order)
def _create_service_definition(self, name, definition, namespace):
return dict(type_="ServiceDefinition", name=name, definition=definition, namespace=namespace)
def _register_bulk(self):
if self.bulk_entries:
log.info("store_interfaces: Storing %s entries in directory..." % len(self.bulk_entries))
entries = [(path, key, attrs) for ((path, key), attrs) in self.bulk_entries.iteritems()]
res = self.dir.register_mult(entries)
self.bulk_entries = {}
if self.bulk_resources:
log.info("store_interfaces: Storing %s resources in registry..." % len(self.bulk_resources))
res = self.rr.create_mult(self.bulk_resources)
self.bulk_resources = []
log.debug("store_interfaces: Storing interfaces successful")
def declare_core_exchange_resources(self):
"""
Create important messaging resources (queues, bindings, etc), so that they are ready before
the first containers start and processes get spawned concurrently.
"""
from putil.rabbitmq.rabbit_util import RabbitManagementUtil
rabbit_util = RabbitManagementUtil(self.config, sysname=self.sysname)
# Event persister queue
rabbit_util.declare_exchange("events")
rabbit_util.declare_queue("events", "event_persister")
rabbit_util.bind_queue("events", "event_persister", "#")
