def add_flavors_internal(session, exception_when_existing=True):
configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR)
for config in configs:
logging.info('add config %s to flavor', config)
adapter = utils.get_db_object(
session, models.Adapter,
name=config['ADAPTER_NAME']
)
for flavor_dict in config['FLAVORS']:
flavor = utils.add_db_object(
session, models.AdapterFlavor,
exception_when_existing, flavor_dict['flavor'], adapter.id,
display_name=flavor_dict.get('display_name', None),
template=flavor_dict.get('template', None)
)
role_names = flavor_dict.get('roles', [])
for role_name in role_names:
role = utils.get_db_object(
session, models.AdapterRole,
name=role_name, adapter_id=adapter.id
)
utils.add_db_object(
session, models.AdapterFlavorRole,
exception_when_existing, flavor.id, role.id
)
utils.update_db_object(
session, flavor,
patched_ordered_flavor_roles=[role_name]
)
def add_os_adapters_internal(session):
parents = {}
configs = util.load_configs(setting.OS_ADAPTER_DIR)
with session.begin(subtransactions=True):
for config in configs:
if 'OS' in config:
os = utils.get_db_object(
session, models.OperatingSystem,
name=config['OS']
)
else:
os = None
if 'INSTALLER' in config:
installer = utils.get_db_object(
session, models.OSInstaller,
name=config['INSTALLER']
)
else:
installer = None
object = utils.add_db_object(
session, models.OSAdapter,
True, config['NAME'], os=os, installer=installer
)
parents[config['NAME']] = (object, config.get('PARENT', None))
for name, (object, parent_name) in parents.items():
if parent_name:
parent, _ = parents[parent_name]
else:
parent = None
utils.update_db_object(
session, object, parent=parent
)
_complement_os_adapters(session)
def _get_os_fields_from_configuration():
"""Get os fields from os field config dir."""
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(setting.OS_FIELD_DIR, env_locals=env_locals)
return _get_field_from_configuration(configs)
def _get_oses_metadata_from_configuration():
"""Get os metadata from os metadata config dir."""
oses_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.OS_METADATA_DIR,
env_locals=env_locals
)
for config in configs:
os_name = config['OS']
os_metadata = oses_metadata.setdefault(os_name, {})
for key, value in config['METADATA'].items():
os_metadata[key] = _get_metadata_from_configuration(
key, key, value, OS_FIELDS
)
oses = adapter_api.OSES
parents = {}
for os_name, os in oses.items():
parent = os.get('parent', None)
parents[os_name] = parent
for os_name, os in oses.items():
oses_metadata[os_name] = util.recursive_merge_dict(
os_name, oses_metadata, parents
)
return oses_metadata
def test_load_confs(self):
loaded = util.load_configs(self.TEST_UTIL_HOME +
'/data/test_load_confs')
expected = [{'TEST': True, 'PROD': False}, {'UTIL_TEST': 'unittest'}]
loaded.sort()
expected.sort()
self.assertTrue(loaded, expected)
def _get_flavors_metadata_from_configuration():
"""Get flavor metadata from flavor metadata config dir."""
flavors_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(setting.FLAVOR_METADATA_DIR,
env_locals=env_locals)
for config in configs:
adapter_name = config['ADAPTER']
flavor_name = config['FLAVOR']
flavor_metadata = flavors_metadata.setdefault(adapter_name,
{}).setdefault(
flavor_name, {})
for key, value in config['METADATA'].items():
flavor_metadata[key] = _get_metadata_from_configuration(
key, key, value, FLAVOR_FIELDS)
packages_metadata = PACKAGES_METADATA
adapters_flavors = adapter_api.ADAPTERS_FLAVORS
for adapter_name, adapter_flavors in adapters_flavors.items():
package_metadata = packages_metadata.get(adapter_name, {})
for flavor_name, flavor in adapter_flavors.items():
flavor_metadata = flavors_metadata.setdefault(adapter_name,
{}).setdefault(
flavor_name, {})
util.merge_dict(flavor_metadata, package_metadata, override=False)
return flavors_metadata
def _get_oses_from_configuration():
"""Get all os configs from os configuration dir.
Example: {
<os_name>: {
'name': <os_name>,
'id': <os_name>,
'os_id': <os_name>,
'deployable': True
}
}
"""
configs = util.load_configs(setting.OS_DIR)
systems = {}
for config in configs:
logging.info('get config %s', config)
system_name = config['NAME']
parent_name = config.get('PARENT', None)
system = {
'name': system_name,
'id': system_name,
'os_id': system_name,
'parent': parent_name,
'parent_id': parent_name,
'deployable': config.get('DEPLOYABLE', False)
}
systems[system_name] = system
parents = {}
for name, system in systems.items():
parent = system.get('parent', None)
parents[name] = parent
for name, system in systems.items():
util.recursive_merge_dict(name, systems, parents)
return systems
def _get_flavors_metadata_from_configuration():
"""Get flavor metadata from flavor metadata config dir."""
flavors_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.FLAVOR_METADATA_DIR,
env_locals=env_locals
)
for config in configs:
adapter_name = config['ADAPTER']
flavor_name = config['FLAVOR']
flavor_metadata = flavors_metadata.setdefault(
adapter_name, {}
).setdefault(flavor_name, {})
for key, value in config['METADATA'].items():
flavor_metadata[key] = _get_metadata_from_configuration(
key, key, value, FLAVOR_FIELDS
)
packages_metadata = PACKAGES_METADATA
adapters_flavors = adapter_api.ADAPTERS_FLAVORS
for adapter_name, adapter_flavors in adapters_flavors.items():
package_metadata = packages_metadata.get(adapter_name, {})
for flavor_name, flavor in adapter_flavors.items():
flavor_metadata = flavors_metadata.setdefault(
adapter_name, {}
).setdefault(flavor_name, {})
util.merge_dict(flavor_metadata, package_metadata, override=False)
return flavors_metadata
def _add_roles_from_configuration():
"""Get roles from roles config dir and update to adapters."""
configs = util.load_configs(setting.ADAPTER_ROLE_DIR)
for config in configs:
logging.info('add config %s to role', config)
adapter_name = config['ADAPTER_NAME']
adapter = ADAPTERS[adapter_name]
adapter_roles = ADAPTERS_ROLES.setdefault(adapter_name, {})
for role_dict in config['ROLES']:
role_name = role_dict['role']
display_name = role_dict.get('display_name', role_name)
adapter_roles[role_name] = {
'name': role_name,
'id': '%s:%s' % (adapter_name, role_name),
'adapter_id': adapter_name,
'adapter_name': adapter_name,
'display_name': display_name,
'description': role_dict.get('description', display_name),
'optional': role_dict.get('optional', False)
}
parents = {}
for name, adapter in ADAPTERS.items():
parent = adapter.get('parent', None)
parents[name] = parent
for adapter_name, adapter_roles in ADAPTERS_ROLES.items():
util.recursive_merge_dict(adapter_name, ADAPTERS_ROLES, parents)
for adapter_name, adapter_roles in ADAPTERS_ROLES.items():
adapter = ADAPTERS[adapter_name]
adapter['roles'] = adapter_roles.values()
def _get_packages_metadata_from_configuration():
"""Get package metadata from package metadata config dir."""
packages_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.PACKAGE_METADATA_DIR,
env_locals=env_locals
)
for config in configs:
adapter_name = config['ADAPTER']
package_metadata = packages_metadata.setdefault(adapter_name, {})
for key, value in config['METADATA'].items():
package_metadata[key] = _get_metadata_from_configuration(
key, key, value, PACKAGE_FIELDS
)
adapters = adapter_api.ADAPTERS
parents = {}
for adapter_name, adapter in adapters.items():
parent = adapter.get('parent', None)
parents[adapter_name] = parent
for adapter_name, adapter in adapters.items():
packages_metadata[adapter_name] = util.recursive_merge_dict(
adapter_name, packages_metadata, parents
)
return packages_metadata
def _get_oses_from_configuration():
"""Get all os configs from os configuration dir.
Example: {
<os_name>: {
'name': <os_name>,
'id': <os_name>,
'os_id': <os_name>,
'deployable': True
}
}
"""
configs = util.load_configs(setting.OS_DIR)
systems = {}
for config in configs:
logging.info('get config %s', config)
system_name = config['NAME']
parent_name = config.get('PARENT', None)
system = {
'name': system_name,
'id': system_name,
'os_id': system_name,
'parent': parent_name,
'parent_id': parent_name,
'deployable': config.get('DEPLOYABLE', False)
}
systems[system_name] = system
parents = {}
for name, system in systems.items():
parent = system.get('parent', None)
parents[name] = parent
for name, system in systems.items():
util.recursive_merge_dict(name, systems, parents)
return systems
def _add_roles_from_configuration():
"""Get roles from roles config dir and update to adapters."""
configs = util.load_configs(setting.ADAPTER_ROLE_DIR)
for config in configs:
logging.info(
'add config %s to role', config
)
adapter_name = config['ADAPTER_NAME']
adapter = ADAPTERS[adapter_name]
adapter_roles = ADAPTERS_ROLES.setdefault(adapter_name, {})
for role_dict in config['ROLES']:
role_name = role_dict['role']
display_name = role_dict.get('display_name', role_name)
adapter_roles[role_name] = {
'name': role_name,
'id': '%s:%s' % (adapter_name, role_name),
'adapter_id': adapter_name,
'adapter_name': adapter_name,
'display_name': display_name,
'description': role_dict.get('description', display_name),
'optional': role_dict.get('optional', False)
}
parents = {}
for name, adapter in ADAPTERS.items():
parent = adapter.get('parent', None)
parents[name] = parent
for adapter_name, adapter_roles in ADAPTERS_ROLES.items():
util.recursive_merge_dict(adapter_name, ADAPTERS_ROLES, parents)
for adapter_name, adapter_roles in ADAPTERS_ROLES.items():
adapter = ADAPTERS[adapter_name]
adapter['roles'] = adapter_roles.values()
def add_flavor_field_internal(session):
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.FLAVOR_FIELD_DIR,
env_locals=env_locals
)
return _add_field_internal(
session, models.FlavorConfigField, configs
)
def _get_adapters_from_configuration():
"""Get adapters from adapter config dir."""
configs = util.load_configs(setting.ADAPTER_DIR)
adapters = {}
for config in configs:
logging.info('add config %s to adapter', config)
if 'OS_INSTALLER' in config:
os_installer = OS_INSTALLERS[config['OS_INSTALLER']]
else:
os_installer = None
if 'PACKAGE_INSTALLER' in config:
package_installer = PACKAGE_INSTALLERS[
config['PACKAGE_INSTALLER']
]
else:
package_installer = None
adapter_name = config['NAME']
parent_name = config.get('PARENT', None)
adapter = {
'name': adapter_name,
'id': adapter_name,
'parent': parent_name,
'parent_id': parent_name,
'display_name': config.get('DISPLAY_NAME', adapter_name),
'os_installer': os_installer,
'package_installer': package_installer,
'deployable': config.get('DEPLOYABLE', False),
'health_check_cmd': config.get('HEALTH_CHECK_COMMAND', None),
'supported_oses': [],
'roles': [],
'flavors': []
}
supported_os_patterns = [
re.compile(supported_os_pattern)
for supported_os_pattern in config.get('SUPPORTED_OS_PATTERNS', [])
]
for os_name, os in OSES.items():
if not os.get('deployable', False):
continue
for supported_os_pattern in supported_os_patterns:
if supported_os_pattern.match(os_name):
adapter['supported_oses'].append(os)
break
adapters[adapter_name] = adapter
parents = {}
for name, adapter in adapters.items():
parent = adapter.get('parent', None)
parents[name] = parent
for name, adapter in adapters.items():
util.recursive_merge_dict(name, adapters, parents)
return adapters
def _get_flavor_fields_from_configuration():
"""Get flavor fields from flavor field config dir."""
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.FLAVOR_FIELD_DIR,
env_locals=env_locals
)
return _get_field_from_configuration(
configs
)
def scan(self):
"""Implemnets the scan method in BasePlugin class.
.. note::
Dummy scan function for compass appliance.
Returns fixed mac addresses.
"""
mac_lists = util.load_configs(setting.MAC_LIST_DIR)
mac_list = None
for item in mac_lists:
mac_list = item['MAC_LIST']
return mac_list
def _get_adapters_from_configuration():
"""Get adapters from adapter config dir."""
configs = util.load_configs(setting.ADAPTER_DIR)
adapters = {}
for config in configs:
logging.info('add config %s to adapter', config)
if 'OS_INSTALLER' in config:
os_installer = OS_INSTALLERS[config['OS_INSTALLER']]
else:
os_installer = None
if 'PACKAGE_INSTALLER' in config:
package_installer = PACKAGE_INSTALLERS[config['PACKAGE_INSTALLER']]
else:
package_installer = None
adapter_name = config['NAME']
parent_name = config.get('PARENT', None)
adapter = {
'name': adapter_name,
'id': adapter_name,
'parent': parent_name,
'parent_id': parent_name,
'display_name': config.get('DISPLAY_NAME', adapter_name),
'os_installer': os_installer,
'package_installer': package_installer,
'deployable': config.get('DEPLOYABLE', False),
'health_check_cmd': config.get('HEALTH_CHECK_COMMAND', None),
'supported_oses': [],
'roles': [],
'flavors': []
}
supported_os_patterns = [
re.compile(supported_os_pattern)
for supported_os_pattern in config.get('SUPPORTED_OS_PATTERNS', [])
]
for os_name, os in OSES.items():
if not os.get('deployable', False):
continue
for supported_os_pattern in supported_os_patterns:
if supported_os_pattern.match(os_name):
adapter['supported_oses'].append(os)
break
adapters[adapter_name] = adapter
parents = {}
for name, adapter in adapters.items():
parent = adapter.get('parent', None)
parents[name] = parent
for name, adapter in adapters.items():
util.recursive_merge_dict(name, adapters, parents)
return adapters
def _add_flavors_from_configuration():
"""Get flavors from flavor config dir and update to adapters."""
configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR)
for config in configs:
logging.info('add config %s to flavor', config)
adapter_name = config['ADAPTER_NAME']
adapter = ADAPTERS[adapter_name]
adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {})
adapter_roles = ADAPTERS_ROLES[adapter_name]
for flavor_dict in config['FLAVORS']:
flavor_name = flavor_dict['flavor']
flavor_id = '%s:%s' % (adapter_name, flavor_name)
flavor = {
'name': flavor_name,
'id': flavor_id,
'adapter_id': adapter_name,
'adapter_name': adapter_name,
'display_name': flavor_dict.get('display_name', flavor_name),
'template': flavor_dict.get('template', None)
}
flavor_roles = flavor_dict.get('roles', [])
roles_in_flavor = []
for flavor_role in flavor_roles:
if isinstance(flavor_role, basestring):
role_name = flavor_role
role_in_flavor = {
'name': role_name,
'flavor_id': flavor_id
}
else:
role_in_flavor = flavor_role
role_in_flavor['flavor_id'] = flavor_id
if 'role' in role_in_flavor:
role_in_flavor['name'] = role_in_flavor['role']
del role_in_flavor['role']
role_name = role_in_flavor['name']
role = adapter_roles[role_name]
util.merge_dict(role_in_flavor, role, override=False)
roles_in_flavor.append(role_in_flavor)
flavor['roles'] = roles_in_flavor
adapter_flavors[flavor_name] = flavor
parents = {}
for name, adapter in ADAPTERS.items():
parent = adapter.get('parent', None)
parents[name] = parent
for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items():
util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents)
for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items():
adapter = ADAPTERS[adapter_name]
adapter['flavors'] = adapter_flavors.values()
def _add_flavors_from_configuration():
"""Get flavors from flavor config dir and update to adapters."""
configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR)
for config in configs:
logging.info('add config %s to flavor', config)
adapter_name = config['ADAPTER_NAME']
adapter = ADAPTERS[adapter_name]
adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {})
adapter_roles = ADAPTERS_ROLES[adapter_name]
for flavor_dict in config['FLAVORS']:
flavor_name = flavor_dict['flavor']
flavor_id = '%s:%s' % (adapter_name, flavor_name)
flavor = {
'name': flavor_name,
'id': flavor_id,
'adapter_id': adapter_name,
'adapter_name': adapter_name,
'display_name': flavor_dict.get('display_name', flavor_name),
'template': flavor_dict.get('template', None)
}
flavor_roles = flavor_dict.get('roles', [])
roles_in_flavor = []
for flavor_role in flavor_roles:
if isinstance(flavor_role, basestring):
role_name = flavor_role
role_in_flavor = {
'name': role_name,
'flavor_id': flavor_id
}
else:
role_in_flavor = flavor_role
role_in_flavor['flavor_id'] = flavor_id
if 'role' in role_in_flavor:
role_in_flavor['name'] = role_in_flavor['role']
del role_in_flavor['role']
role_name = role_in_flavor['name']
role = adapter_roles[role_name]
util.merge_dict(role_in_flavor, role, override=False)
roles_in_flavor.append(role_in_flavor)
flavor['roles'] = roles_in_flavor
adapter_flavors[flavor_name] = flavor
parents = {}
for name, adapter in ADAPTERS.items():
parent = adapter.get('parent', None)
parents[name] = parent
for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items():
util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents)
for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items():
adapter = ADAPTERS[adapter_name]
adapter['flavors'] = adapter_flavors.values()
def test_load_confs_local_env_no_override(self):
loaded = util.load_configs(self.TEST_UTIL_HOME +
'/data/test_load_confs',
env_globals={'TEST': False})
expected = [{
'TEST': False,
'UTIL_TEST': 'unittest'
}, {
'TEST': True,
'PROD': False
}]
loaded.sort()
expected.sort()
self.assertTrue(loaded, expected)
def scan(self):
"""Implemnets the scan method in BasePlugin class.
.. note::
Dummy scan function for compass appliance.
Returns fixed mac addresses.
"""
mac_list = None
machine_lists = util.load_configs(setting.MACHINE_LIST_DIR)
for items in machine_lists:
for item in items['MACHINE_LIST']:
for k, v in item.items():
if k == self.host:
mac_list = v
return mac_list
def add_roles_internal(session):
configs = util.load_configs(setting.PACKAGE_ROLE_DIR)
with session.begin(subtransactions=True):
for config in configs:
package_adapter = utils.get_db_object(
session, models.PackageAdapter,
name=config['ADAPTER_NAME']
)
for role_dict in config['ROLES']:
utils.add_db_object(
session, models.PackageAdapterRole,
True, role_dict['role'], package_adapter.id,
description=role_dict['description'],
optional=role_dict.get('optional', False)
)
def add_os_metadata_internal(session):
os_metadatas = []
configs = util.load_configs(setting.OS_METADATA_DIR, env_locals=validator.VALIDATOR_LOCALS)
with session.begin(subtransactions=True):
for config in configs:
adapter = utils.get_db_object(
session, models.OSAdapter, name=config['ADAPTER']
)
for key, value in config['METADATA'].items():
os_metadatas.append(_add_metadata(
session, models.OSConfigField,
models.OSConfigMetadata,
key, value, parent=None,
adapter=adapter
))
return os_metadatas
def test_load_confs(self):
loaded = util.load_configs(
self.TEST_UTIL_HOME + '/data/test_load_confs'
)
expected = [
{
'TEST': True,
'PROD': False
},
{
'UTIL_TEST': 'unittest'
}
]
loaded.sort()
expected.sort()
self.assertTrue(loaded, expected)
def test_load_confs_local_env(self):
loaded = util.load_configs(
self.TEST_UTIL_HOME + '/data/test_load_confs',
env_globals={'TEST': True}
)
expected = [
{
'TEST': True,
'UTIL_TEST': 'unittest'
},
{
'TEST': True,
'PROD': False
}]
loaded.sort()
expected.sort()
self.assertTrue(loaded, expected)
def add_roles_internal(session, exception_when_existing=True):
configs = util.load_configs(setting.ADAPTER_ROLE_DIR)
for config in configs:
logging.info(
'add config %s to role', config
)
adapter = utils.get_db_object(
session, models.Adapter,
name=config['ADAPTER_NAME']
)
for role_dict in config['ROLES']:
utils.add_db_object(
session, models.AdapterRole,
exception_when_existing, role_dict['role'], adapter.id,
display_name=role_dict.get('display_name', None),
description=role_dict.get('description', None),
optional=role_dict.get('optional', False)
)
def _get_flavors_metadata_ui_converters_from_configuration():
"""Get flavor metadata ui converters from flavor mapping config dir."""
flavors_metadata_ui_converters = {}
configs = util.load_configs(setting.FLAVOR_MAPPING_DIR)
for config in configs:
adapter_name = config['ADAPTER']
flavor_name = config['FLAVOR']
flavors_metadata_ui_converters.setdefault(
adapter_name, {})[flavor_name] = config.get('CONFIG_MAPPING', {})
adapters = adapter_api.ADAPTERS
parents = {}
for adapter_name, adapter in adapters.items():
parent = adapter.get('parent', None)
parents[adapter_name] = parent
for adapter_name, adapter in adapters.items():
flavors_metadata_ui_converters[adapter_name] = (
util.recursive_merge_dict(adapter_name,
flavors_metadata_ui_converters, parents))
return flavors_metadata_ui_converters
def _get_oses_metadata_ui_converters_from_configuration():
"""Get os metadata ui converters from os metadata mapping config dir.
os metadata ui converter is used to convert os metadata to
the format UI can understand and show.
"""
oses_metadata_ui_converters = {}
configs = util.load_configs(setting.OS_MAPPING_DIR)
for config in configs:
os_name = config['OS']
oses_metadata_ui_converters[os_name] = config.get('CONFIG_MAPPING', {})
oses = adapter_api.OSES
parents = {}
for os_name, os in oses.items():
parent = os.get('parent', None)
parents[os_name] = parent
for os_name, os in oses.items():
oses_metadata_ui_converters[os_name] = util.recursive_merge_dict(
os_name, oses_metadata_ui_converters, parents)
return oses_metadata_ui_converters
def add_os_metadata_internal(session, exception_when_existing=True):
os_metadatas = []
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.OS_METADATA_DIR,
env_locals=env_locals
)
for config in configs:
os = utils.get_db_object(
session, models.OperatingSystem, name=config['OS']
)
for key, value in config['METADATA'].items():
os_metadatas.append(_add_metadata(
session, models.OSConfigField,
models.OSConfigMetadata,
os.id, key, key, value,
exception_when_existing=exception_when_existing,
parent=None
))
return os_metadatas
def add_flavor_metadata_internal(session, exception_when_existing=True):
flavor_metadatas = []
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.FLAVOR_METADATA_DIR,
env_locals=env_locals
)
for config in configs:
flavor = utils.get_db_object(
session, models.AdapterFlavor, name=config['FLAVOR']
)
for key, value in config['METADATA'].items():
flavor_metadatas.append(_add_metadata(
session, models.FlavorConfigField,
models.FlavorConfigMetadata,
flavor.id, key, key, value,
exception_when_existing=exception_when_existing,
parent=None
))
return flavor_metadatas
def _get_oses_metadata_ui_converters_from_configuration():
"""Get os metadata ui converters from os metadata mapping config dir.
os metadata ui converter is used to convert os metadata to
the format UI can understand and show.
"""
oses_metadata_ui_converters = {}
configs = util.load_configs(setting.OS_MAPPING_DIR)
for config in configs:
os_name = config['OS']
oses_metadata_ui_converters[os_name] = config.get('CONFIG_MAPPING', {})
oses = adapter_api.OSES
parents = {}
for os_name, os in oses.items():
parent = os.get('parent', None)
parents[os_name] = parent
for os_name, os in oses.items():
oses_metadata_ui_converters[os_name] = util.recursive_merge_dict(
os_name, oses_metadata_ui_converters, parents
)
return oses_metadata_ui_converters
def _get_flavors_metadata_ui_converters_from_configuration():
"""Get flavor metadata ui converters from flavor mapping config dir."""
flavors_metadata_ui_converters = {}
configs = util.load_configs(setting.FLAVOR_MAPPING_DIR)
for config in configs:
adapter_name = config['ADAPTER']
flavor_name = config['FLAVOR']
flavors_metadata_ui_converters.setdefault(
adapter_name, {}
)[flavor_name] = config.get('CONFIG_MAPPING', {})
adapters = adapter_api.ADAPTERS
parents = {}
for adapter_name, adapter in adapters.items():
parent = adapter.get('parent', None)
parents[adapter_name] = parent
for adapter_name, adapter in adapters.items():
flavors_metadata_ui_converters[adapter_name] = (
util.recursive_merge_dict(
adapter_name, flavors_metadata_ui_converters, parents
)
)
return flavors_metadata_ui_converters
def _get_oses_metadata_from_configuration():
"""Get os metadata from os metadata config dir."""
oses_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(setting.OS_METADATA_DIR, env_locals=env_locals)
for config in configs:
os_name = config['OS']
os_metadata = oses_metadata.setdefault(os_name, {})
for key, value in config['METADATA'].items():
os_metadata[key] = _get_metadata_from_configuration(
key, key, value, OS_FIELDS)
oses = adapter_api.OSES
parents = {}
for os_name, os in oses.items():
parent = os.get('parent', None)
parents[os_name] = parent
for os_name, os in oses.items():
oses_metadata[os_name] = util.recursive_merge_dict(
os_name, oses_metadata, parents)
return oses_metadata
def _get_packages_metadata_from_configuration():
"""Get package metadata from package metadata config dir."""
packages_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(setting.PACKAGE_METADATA_DIR,
env_locals=env_locals)
for config in configs:
adapter_name = config['ADAPTER']
package_metadata = packages_metadata.setdefault(adapter_name, {})
for key, value in config['METADATA'].items():
package_metadata[key] = _get_metadata_from_configuration(
key, key, value, PACKAGE_FIELDS)
adapters = adapter_api.ADAPTERS
parents = {}
for adapter_name, adapter in adapters.items():
parent = adapter.get('parent', None)
parents[adapter_name] = parent
for adapter_name, adapter in adapters.items():
packages_metadata[adapter_name] = util.recursive_merge_dict(
adapter_name, packages_metadata, parents)
return packages_metadata
def add_package_adapters_internal(session):
parents = {}
configs = util.load_configs(setting.PACKAGE_ADAPTER_DIR)
with session.begin(subtransactions=True):
for config in configs:
if 'DISTRIBUTED_SYSTEM' in config:
distributed_system = utils.get_db_object(
session, models.DistributedSystem,
name=config['DISTRIBUTED_SYSTEM']
)
else:
distributed_system = None
if 'INSTALLER' in config:
installer = utils.get_db_object(
session, models.PackageInstaller,
name=config['INSTALLER']
)
else:
installer = None
object = utils.add_db_object(
session, models.PackageAdapter,
True,
config['NAME'],
distributed_system=distributed_system,
installer=installer,
support_os_patterns=config.get('SUPPORT_OS_PATTERNS', [])
)
parents[config['NAME']] = (object, config.get('PARENT', None))
for name, (object, parent_name) in parents.items():
if parent_name:
parent, _ = parents[parent_name]
else:
parent = None
utils.update_db_object(session, object, parent=parent)
_complement_distributed_system_adapters(session)
def is_valid_percentage(name, percentage, **kwargs):
"""Valid the percentage."""
if 0 <= percentage <= 100:
return True
logging.debug('%s invalid percentage %s', name, percentage)
def is_valid_port(name, port, **kwargs):
"""Valid the format of port."""
if 0 < port < 65536:
return True
logging.debug('%s invalid port %s', name, port)
def is_valid_size(name, size, **kwargs):
if re.match(r'^(\d+)(K|M|G|T)$', size):
return True
logging.debug('%s invalid size %s', name, size)
return False
VALIDATOR_GLOBALS = globals()
VALIDATOR_LOCALS = locals()
VALIDATOR_CONFIGS = util.load_configs(setting.VALIDATOR_DIR,
config_name_suffix='.py',
env_globals=VALIDATOR_GLOBALS,
env_locals=VALIDATOR_LOCALS)
for validator_config in VALIDATOR_CONFIGS:
VALIDATOR_LOCALS.update(validator_config)
def _load_calculator_configurations(force=False):
global PROGRESS_CALCULATOR_CONFIGURATIONS
if force or PROGRESS_CALCULATOR_CONFIGURATIONS is None:
env_locals = {}
env_locals.update(ENV_GLOBALS)
env_locals.update(ENV_LOCALS)
PROGRESS_CALCULATOR_CONFIGURATIONS = util.load_configs(
setting.PROGRESS_CALCULATOR_DIR,
env_locals=env_locals
)
if not PROGRESS_CALCULATOR_CONFIGURATIONS:
logging.info('No configuration found for progress calculator.')
global OS_ADAPTER_CONFIGURATIONS
if force or OS_ADAPTER_CONFIGURATIONS is None:
OS_ADAPTER_CONFIGURATIONS = []
for progress_calculator_configuration in (
PROGRESS_CALCULATOR_CONFIGURATIONS
):
if 'OS_LOG_CONFIGURATIONS' in (
progress_calculator_configuration
):
os_installer_configurations = (
progress_calculator_configuration['OS_LOG_CONFIGURATIONS']
)
for os_installer_configuration in os_installer_configurations:
OS_ADAPTER_CONFIGURATIONS.append(OSMatcher(
os_installer_name=(
os_installer_configuration['os_installer_name']
),
os_pattern=os_installer_configuration['os_pattern'],
item_matcher=(
os_installer_configuration['item_matcher']
),
file_reader_factory=FileReaderFactory(
os_installer_configuration['logdir']
)
))
if not OS_ADAPTER_CONFIGURATIONS:
logging.info(
'no OS_LOG_CONFIGURATIONS section found '
'in progress calculator.'
)
else:
logging.debug(
'OS_ADAPTER_CONFIGURATIONS is\n%s',
OS_ADAPTER_CONFIGURATIONS
)
global PACKAGE_ADAPTER_CONFIGURATIONS
if force or PACKAGE_ADAPTER_CONFIGURATIONS is None:
PACKAGE_ADAPTER_CONFIGURATIONS = []
for progress_calculator_configuration in (
PROGRESS_CALCULATOR_CONFIGURATIONS
):
if 'ADAPTER_LOG_CONFIGURATIONS' in (
progress_calculator_configuration
):
package_installer_configurations = (
progress_calculator_configuration[
'ADAPTER_LOG_CONFIGURATIONS'
]
)
for package_installer_configuration in (
package_installer_configurations
):
PACKAGE_ADAPTER_CONFIGURATIONS.append(PackageMatcher(
package_installer_name=(
package_installer_configuration[
'package_installer_name'
]
),
adapter_pattern=(
package_installer_configuration['adapter_pattern']
),
item_matcher=(
package_installer_configuration['item_matcher']
),
file_reader_factory=FileReaderFactory(
package_installer_configuration['logdir']
)
))
if not PACKAGE_ADAPTER_CONFIGURATIONS:
logging.info(
'no PACKAGE_LOG_CONFIGURATIONS section found '
'in progress calculator.'
)
else:
logging.debug(
'PACKAGE_ADAPTER_CONFIGURATIONS is\n%s',
PACKAGE_ADAPTER_CONFIGURATIONS
)
def add_distributed_systems_internal(session, exception_when_existing=True):
configs = util.load_configs(setting.DISTRIBUTED_SYSTEM_DIR)
_add_system(
session, models.DistributedSystem, configs,
exception_when_existing=exception_when_existing
)
def add_adapters_internal(session, exception_when_existing=True):
parents = {}
configs = util.load_configs(setting.ADAPTER_DIR)
for config in configs:
logging.info('add config %s to adapter', config)
if 'DISTRIBUTED_SYSTEM' in config:
distributed_system = utils.get_db_object(
session, models.DistributedSystem,
name=config['DISTRIBUTED_SYSTEM']
)
else:
distributed_system = None
if 'OS_INSTALLER' in config:
os_installer = utils.get_db_object(
session, models.OSInstaller,
alias=config['OS_INSTALLER']
)
else:
os_installer = None
if 'PACKAGE_INSTALLER' in config:
package_installer = utils.get_db_object(
session, models.PackageInstaller,
alias=config['PACKAGE_INSTALLER']
)
else:
package_installer = None
adapter = utils.add_db_object(
session, models.Adapter,
exception_when_existing,
config['NAME'],
display_name=config.get('DISPLAY_NAME', None),
distributed_system=distributed_system,
os_installer=os_installer,
package_installer=package_installer,
deployable=config.get('DEPLOYABLE', False)
)
supported_os_patterns = [
re.compile(supported_os_pattern)
for supported_os_pattern in config.get('SUPPORTED_OS_PATTERNS', [])
]
oses = utils.list_db_objects(
session, models.OperatingSystem
)
for os in oses:
if not os.deployable:
continue
os_name = os.name
for supported_os_pattern in supported_os_patterns:
if supported_os_pattern.match(os_name):
utils.add_db_object(
session, models.AdapterOS,
exception_when_existing,
os.id, adapter.id
)
break
parents[config['NAME']] = (adapter, config.get('PARENT', None))
for name, (adapter, parent_name) in parents.items():
if parent_name:
parent, _ = parents[parent_name]
else:
parent = None
utils.update_db_object(session, adapter, parent=parent)
# limitations under the License.
"""Metadata Callback methods."""
import logging
import netaddr
import random
import re
import socket
from compass.db import exception
from compass.utils import setting_wrapper as setting
from compass.utils import util
CALLBACK_GLOBALS = globals()
CALLBACK_LOCALS = locals()
CALLBACK_CONFIGS = util.load_configs(setting.CALLBACK_DIR,
config_name_suffix='.py',
env_globals=CALLBACK_GLOBALS,
env_locals=CALLBACK_LOCALS)
for callback_config in CALLBACK_CONFIGS:
CALLBACK_LOCALS.update(callback_config)
def default_proxy(name, **kwargs):
return setting.COMPASS_SUPPORTED_PROXY
def proxy_options(name, **kwargs):
return [setting.COMPASS_SUPPORTED_PROXY]
def default_noproxy(name, **kwargs):
return setting.COMPASS_SUPPORTED_DEFAULT_NOPROXY
def add_oses_internal(session, exception_when_existing=True):
configs = util.load_configs(setting.OS_DIR)
_add_system(
session, models.OperatingSystem, configs,
exception_when_existing=exception_when_existing
)
def test_load_conf_error(self):
err_dir = 'non-exist/dir'
loaded = util.load_configs(err_dir)
self.assertEqual([], loaded)
def _get_package_installers_from_configuration():
"""Get package installers from package installer config dir."""
configs = util.load_configs(setting.PACKAGE_INSTALLER_DIR)
return _get_installers_from_configuration(configs)
def get_vendor(self, host, credential):
"""Check and get vendor of the switch.
:param host: switch ip:
:param credential: credential to access switch
:return a tuple (vendor, switch_state, error)
"""
switch_lists = util.load_configs(setting.MACHINE_LIST_DIR)
switch_list = []
for items in switch_lists:
for item in items['MACHINE_LIST']:
for k, v in item.items():
switch_list.append(k)
if host in switch_list:
return ("appliance", "Found", "")
# TODO(grace): Why do we need to have valid IP?
# a hostname should also work.
if not utils.valid_ip_format(host):
logging.error("host '%s' is not valid IP address!", host)
return (None, ERROR, "Invalid IP address %s!" % host)
if not utils.is_valid_snmp_v2_credential(credential):
logging.debug("******The credential %s of host %s cannot "
"be used for either SNMP v2 or SSH*****",
credential, host)
return (None, ERROR, "Invalid credential")
sys_info, err = self.get_sys_info(host, credential)
if not sys_info:
return (None, UNREACHABLE, err)
# List all vendors in vendors directory -- a directory but hidden
# under ../vendors
all_vendors = [o for o in os.listdir(self.vendors_dir)
if os.path.isdir(os.path.join(self.vendors_dir, o))
and re.match(r'^[^\.]', o)]
logging.debug("[get_vendor][available vendors]: %s ", all_vendors)
logging.debug("[get_vendor] System Information is [%s]", sys_info)
# TODO(grace): should not conver to lower. The vendor impl can choose
# to do case-insensitive match
# sys_info = sys_info.lower()
vendor = None
for vname in all_vendors:
vpath = os.path.join(self.vendors_dir, vname)
instance = utils.load_module(vname, vpath)
if not instance:
logging.error('no instance %s load from %s', vname, vpath)
continue
if instance.is_this_vendor(sys_info):
logging.info("[get_vendor]****Found vendor '%s'****", vname)
vendor = vname
break
if not vendor:
logging.debug("[get_vendor] No vendor found! <==================")
return (None, NOTSUPPORTED, "Not supported switch vendor!")
return (vendor, REPOLLING, "")
def _get_os_installers_from_configuration():
"""Get os installers from os installer config dir."""
configs = util.load_configs(setting.OS_INSTALLER_DIR)
return _get_installers_from_configuration(configs)
def test_load_conf(self):
loaded = util.load_configs(
self.TEST_UTIL_HOME + '/data/test_load_conf'
)
expected = [{'TEST': True, 'PROD': False}]
self.assertEqual(loaded, expected)
def test_load_no_suffix(self):
loaded = util.load_configs(
self.TEST_UTIL_HOME + '/data/test_no_suffix'
)
expected = []
self.assertEqual(loaded, expected)
def test_load_no_suffix(self):
loaded = util.load_configs(self.TEST_UTIL_HOME +
'/data/test_no_suffix')
expected = []
self.assertEqual(loaded, expected)
def test_load_conf_error(self):
err_dir = 'non-exist/dir'
loaded = util.load_configs(err_dir)
self.assertEqual([], loaded)
def test_load_conf(self):
loaded = util.load_configs(self.TEST_UTIL_HOME +
'/data/test_load_conf')
expected = [{'TEST': True, 'PROD': False}]
self.assertEqual(loaded, expected)