def __init__(self, region_name=None):
# Check if objects are cached and try to use those
self.os_clients_dict = collections.defaultdict(dict)
self._token = None
self.region_name = region_name
if ('keystone' in self.os_clients_dict) and self._is_token_valid():
self.keystone_client = self.os_clients_dict['keystone']
else:
self.keystone_client = KeystoneClient()
self.os_clients_dict['keystone'] = self.keystone_client
if region_name in self.os_clients_dict and \
self._is_token_valid():
LOG.info('Using cached OS client objects %s' % region_name)
self.sysinv_client = self.os_clients_dict[region_name]['sysinv']
else:
# Create new objects and cache them
LOG.debug("Creating fresh OS Clients objects %s" % region_name)
self.os_clients_dict[region_name] = collections.defaultdict(dict)
try:
self.sysinv_client = SysinvClient(region_name,
self.keystone_client.session)
self.os_clients_dict[region_name][
'sysinv'] = self.sysinv_client
except Exception as exception:
LOG.error('sysinv_client region %s error: %s' %
(region_name, exception.message))
def get_all_regions_for_project(self, project_id):
try:
# Retrieve regions based on endpoint filter for the project.
region_lists = self._get_filtered_regions(project_id)
if not region_lists:
# If endpoint filter is not used for the project, then
# return all regions
region_lists = \
KeystoneClient().endpoint_cache.get_all_regions()
# nova, cinder, and neutron have no endpoints in consts.CLOUD_0
if consts.CLOUD_0 in region_lists:
region_lists.remove(consts.CLOUD_0)
return region_lists
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
raise
def __init__(self, region_name=consts.VIRTUAL_MASTER_CLOUD, auth_url=None):
# Check if objects are cached and try to use those
self.os_clients_dict = collections.defaultdict(dict)
self._identity_tokens = {}
self.region_name = region_name
if ((region_name in self.os_clients_dict)
and ('keystone' in self.os_clients_dict[region_name])
and self._is_token_valid(region_name)):
self.keystone_client = \
self.os_clients_dict[region_name]['keystone']
else:
LOG.info("get new keystone client for subcloud %s", region_name)
self.keystone_client = KeystoneClient(region_name, auth_url)
self.os_clients_dict[region_name]['keystone'] = \
self.keystone_client
if ((region_name in self.os_clients_dict)
and ('sysinv' in self.os_clients_dict[region_name])
and self._is_token_valid(region_name)):
LOG.info('Using cached OS client objects %s' % region_name)
self.sysinv_client = self.os_clients_dict[region_name]['sysinv']
else:
# Create new objects and cache them
LOG.debug("Creating fresh OS Clients objects %s" % region_name)
self.os_clients_dict[region_name] = collections.defaultdict(dict)
try:
self.sysinv_client = SysinvClient(region_name,
self.keystone_client.session)
self.os_clients_dict[region_name][
'sysinv'] = self.sysinv_client
except Exception as exception:
LOG.error('sysinv_client region %s error: %s' %
(region_name, exception.message))
def _delete_subcloud_routes(self, context, subcloud):
"""Delete the routes to this subcloud"""
keystone_client = KeystoneClient()
# Delete subcloud's identity endpoints
keystone_client.delete_endpoints(subcloud.name)
# Delete the route to this subcloud on the management interface on
# both controllers.
management_subnet = netaddr.IPNetwork(subcloud.management_subnet)
session = keystone_client.endpoint_cache.get_session_from_token(
context.auth_token, context.project)
sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session)
controllers = sysinv_client.get_controller_hosts()
for controller in controllers:
management_interface = sysinv_client.get_management_interface(
controller.hostname)
if management_interface is not None:
sysinv_client.delete_route(
management_interface.uuid, str(management_subnet.ip),
management_subnet.prefixlen,
str(netaddr.IPAddress(
subcloud.systemcontroller_gateway_ip)), 1)
def __init__(self, region_name=consts.VIRTUAL_MASTER_CLOUD, auth_url=None):
# Check if objects are cached and try to use those
self.region_name = region_name
if (region_name in OpenStackDriver._identity_tokens
and (region_name in OpenStackDriver.os_clients_dict) and
('keystone' in OpenStackDriver.os_clients_dict[region_name])
and self._is_token_valid(self.region_name)):
self.keystone_client = \
OpenStackDriver.os_clients_dict[region_name]['keystone']
else:
LOG.info("get new keystone client for %s" % region_name)
self.keystone_client = KeystoneClient(region_name, auth_url)
OpenStackDriver.os_clients_dict[region_name]['keystone'] = \
self.keystone_client
self.disabled_quotas = self._get_disabled_quotas(region_name)
if region_name in OpenStackDriver.os_clients_dict and \
self._is_token_valid(region_name):
LOG.info('Using cached OS client objects %s' % region_name)
self.sysinv_client = OpenStackDriver.os_clients_dict[region_name][
'sysinv']
self.nova_client = OpenStackDriver.os_clients_dict[region_name][
'nova']
self.cinder_client = OpenStackDriver.os_clients_dict[region_name][
'cinder']
self.neutron_client = OpenStackDriver.os_clients_dict[region_name][
'neutron']
self.fm_client = OpenStackDriver.os_clients_dict[region_name]['fm']
else:
# Create new objects and cache them
LOG.info("Creating fresh OS Clients objects %s" % region_name)
OpenStackDriver.os_clients_dict[
region_name] = collections.defaultdict(dict)
try:
self.sysinv_client = SysinvClient(region_name,
self.keystone_client.session)
OpenStackDriver.os_clients_dict[region_name][
'sysinv'] = self.sysinv_client
except Exception as exception:
LOG.error('sysinv_client region %s error: %s' %
(region_name, exception.message))
try:
self.neutron_client = NeutronClient(
region_name,
self.disabled_quotas,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT)
OpenStackDriver.os_clients_dict[region_name][
'neutron'] = self.neutron_client
except Exception as exception:
LOG.error('neutron_client region %s error: %s' %
(region_name, exception.message))
try:
self.nova_client = NovaClient(
region_name,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT,
disabled_quotas=self.disabled_quotas)
OpenStackDriver.os_clients_dict[region_name][
'nova'] = self.nova_client
except Exception as exception:
LOG.error('nova_client region %s error: %s' %
(region_name, exception.message))
try:
self.cinder_client = CinderClient(
region_name,
self.disabled_quotas,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT)
OpenStackDriver.os_clients_dict[region_name][
'cinder'] = self.cinder_client
except Exception as exception:
LOG.error('cinder_client region %s error: %s' %
(region_name, exception.message))
try:
self.fm_client = FmClient(
region_name,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT)
OpenStackDriver.os_clients_dict[region_name][
'fm'] = self.fm_client
except Exception as exception:
LOG.error('fm_client region %s error: %s' %
(region_name, exception.message))
class OpenStackDriver(object):
os_clients_dict = collections.defaultdict(dict)
_identity_tokens = {}
@lockutils.synchronized('dcorch-openstackdriver')
def __init__(self, region_name=consts.VIRTUAL_MASTER_CLOUD, auth_url=None):
# Check if objects are cached and try to use those
self.region_name = region_name
if (region_name in OpenStackDriver._identity_tokens
and (region_name in OpenStackDriver.os_clients_dict) and
('keystone' in OpenStackDriver.os_clients_dict[region_name])
and self._is_token_valid(self.region_name)):
self.keystone_client = \
OpenStackDriver.os_clients_dict[region_name]['keystone']
else:
LOG.info("get new keystone client for %s" % region_name)
self.keystone_client = KeystoneClient(region_name, auth_url)
OpenStackDriver.os_clients_dict[region_name]['keystone'] = \
self.keystone_client
self.disabled_quotas = self._get_disabled_quotas(region_name)
if region_name in OpenStackDriver.os_clients_dict and \
self._is_token_valid(region_name):
LOG.info('Using cached OS client objects %s' % region_name)
self.sysinv_client = OpenStackDriver.os_clients_dict[region_name][
'sysinv']
self.nova_client = OpenStackDriver.os_clients_dict[region_name][
'nova']
self.cinder_client = OpenStackDriver.os_clients_dict[region_name][
'cinder']
self.neutron_client = OpenStackDriver.os_clients_dict[region_name][
'neutron']
self.fm_client = OpenStackDriver.os_clients_dict[region_name]['fm']
else:
# Create new objects and cache them
LOG.info("Creating fresh OS Clients objects %s" % region_name)
OpenStackDriver.os_clients_dict[
region_name] = collections.defaultdict(dict)
try:
self.sysinv_client = SysinvClient(region_name,
self.keystone_client.session)
OpenStackDriver.os_clients_dict[region_name][
'sysinv'] = self.sysinv_client
except Exception as exception:
LOG.error('sysinv_client region %s error: %s' %
(region_name, exception.message))
try:
self.neutron_client = NeutronClient(
region_name,
self.disabled_quotas,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT)
OpenStackDriver.os_clients_dict[region_name][
'neutron'] = self.neutron_client
except Exception as exception:
LOG.error('neutron_client region %s error: %s' %
(region_name, exception.message))
try:
self.nova_client = NovaClient(
region_name,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT,
disabled_quotas=self.disabled_quotas)
OpenStackDriver.os_clients_dict[region_name][
'nova'] = self.nova_client
except Exception as exception:
LOG.error('nova_client region %s error: %s' %
(region_name, exception.message))
try:
self.cinder_client = CinderClient(
region_name,
self.disabled_quotas,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT)
OpenStackDriver.os_clients_dict[region_name][
'cinder'] = self.cinder_client
except Exception as exception:
LOG.error('cinder_client region %s error: %s' %
(region_name, exception.message))
try:
self.fm_client = FmClient(
region_name,
self.keystone_client.session,
endpoint_type=consts.KS_ENDPOINT_DEFAULT)
OpenStackDriver.os_clients_dict[region_name][
'fm'] = self.fm_client
except Exception as exception:
LOG.error('fm_client region %s error: %s' %
(region_name, exception.message))
@classmethod
@lockutils.synchronized('dcorch-openstackdriver')
def delete_region_clients(cls, region_name, clear_token=False):
LOG.warn("delete_region_clients=%s, clear_token=%s" %
(region_name, clear_token))
if region_name in cls.os_clients_dict:
del cls.os_clients_dict[region_name]
if clear_token:
OpenStackDriver._identity_tokens[region_name] = None
def get_enabled_projects(self, id_only=True):
try:
return self.keystone_client.get_enabled_projects(id_only)
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
def get_project_by_name(self, projectname):
try:
return self.keystone_client.get_project_by_name(projectname)
except Exception as exception:
LOG.error('Error Occurred : %s', exception.message)
def get_project_by_id(self, projectid):
try:
return self.keystone_client.get_project_by_id(projectid)
except Exception as exception:
LOG.error('Error Occurred : %s', exception.message)
def get_enabled_users(self, id_only=True):
try:
return self.keystone_client.get_enabled_users(id_only)
except Exception as exception:
LOG.error('Error Occurred : %s', exception.message)
def get_user_by_name(self, username):
try:
return self.keystone_client.get_user_by_name(username)
except Exception as exception:
LOG.error('Error Occurred : %s', exception.message)
def get_user_by_id(self, userid):
try:
return self.keystone_client.get_user_by_id(userid)
except Exception as exception:
LOG.error('Error Occurred : %s', exception.message)
def get_resource_usages(self, project_id, user_id):
# If one of the resources is unavailable we still want to return
# any usage information we have for the others.
nova_usages = {}
neutron_usages = {}
cinder_usages = {}
try:
nova_usages = self.nova_client.get_resource_usages(
project_id, user_id)
if user_id is None:
# neutron/cinder don't do per-user quotas/usage
neutron_usages = self.neutron_client.get_resource_usages(
project_id)
cinder_usages = self.cinder_client.get_resource_usages(
project_id)
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut) as ex:
# Delete the cached objects for that region
LOG.error('Error Occurred: %s', ex.message)
del OpenStackDriver.os_clients_dict[self.region_name]
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
return nova_usages, neutron_usages, cinder_usages
def get_quota_limits(self, project_id, user_id):
# If one of the resources is unavailable we still want to return
# any limit information we have for the others.
nova_limits = {}
neutron_limits = {}
cinder_limits = {}
try:
nova_limits = self.nova_client.get_quota_limits(
project_id, user_id)
if user_id is None:
# neutron/cinder don't do per-user quotas/usage
neutron_limits = self.neutron_client.get_quota_limits(
project_id)
cinder_limits = self.cinder_client.get_quota_limits(project_id)
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut) as ex:
LOG.error('Error Occurred: %s', ex.message)
# Delete the cached objects for that region
del OpenStackDriver.os_clients_dict[self.region_name]
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
return nova_limits, neutron_limits, cinder_limits
def write_quota_limits(self, project_id, user_id, limits_to_write):
try:
self.nova_client.update_quota_limits(project_id, user_id,
**limits_to_write['nova'])
# Only nova supports per-user quotas.
if user_id is None:
self.cinder_client.update_quota_limits(
project_id, **limits_to_write['cinder'])
self.neutron_client.update_quota_limits(
project_id, limits_to_write['neutron'])
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut) as ex:
LOG.error('Error Occurred: %s', ex.message)
# Delete the cached objects for that region
del OpenStackDriver.os_clients_dict[self.region_name]
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
def delete_quota_limits(self, project_id):
try:
self.nova_client.delete_quota_limits(project_id)
self.neutron_client.delete_quota_limits(project_id)
self.cinder_client.delete_quota_limits(project_id)
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut):
# Delete the cached objects for that region
del OpenStackDriver.os_clients_dict[self.region_name]
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
def _get_disabled_quotas(self, region):
disabled_quotas = []
if not self.keystone_client.is_service_enabled('volume') and \
not self.keystone_client.is_service_enabled('volumev2'):
disabled_quotas.extend(consts.CINDER_QUOTA_FIELDS)
# Neutron
if not self.keystone_client.is_service_enabled('network'):
disabled_quotas.extend(consts.NEUTRON_QUOTA_FIELDS)
else:
disabled_quotas.extend(['floating_ips', 'fixed_ips'])
disabled_quotas.extend(['security_groups', 'security_group_rules'])
return disabled_quotas
def get_all_regions_for_project(self, project_id):
try:
# Retrieve regions based on endpoint filter for the project.
region_lists = self._get_filtered_regions(project_id)
if not region_lists:
# If endpoint filter is not used for the project, then
# return all regions
region_lists = \
KeystoneClient().endpoint_cache.get_all_regions()
# nova, cinder, and neutron have no endpoints in consts.CLOUD_0
if consts.CLOUD_0 in region_lists:
region_lists.remove(consts.CLOUD_0)
return region_lists
except Exception as exception:
LOG.error('Error Occurred: %s', exception.message)
raise
def _get_filtered_regions(self, project_id):
return self.keystone_client.get_filtered_region(project_id)
def _is_token_valid(self, region_name):
try:
keystone = \
self.os_clients_dict[region_name]['keystone'].keystone_client
if (not OpenStackDriver._identity_tokens
or region_name not in OpenStackDriver._identity_tokens
or not OpenStackDriver._identity_tokens[region_name]):
identity_token = \
keystone.tokens.validate(keystone.session.get_token())
OpenStackDriver._identity_tokens[region_name] = identity_token
LOG.info("Got new token for subcloud %s, expires_at=%s" %
(region_name, identity_token['expires_at']))
# Reset the cached dictionary
OpenStackDriver.os_clients_dict[region_name] = \
collections.defaultdict(dict)
return False
keystone.tokens.validate(
OpenStackDriver._identity_tokens[region_name])
except Exception as exception:
LOG.info('_is_token_valid handle: %s', exception.message)
# Reset the cached dictionary
OpenStackDriver.os_clients_dict[region_name] = \
collections.defaultdict(dict)
OpenStackDriver._identity_tokens[region_name] = None
return False
identity_token = OpenStackDriver._identity_tokens[region_name]
expiry_time = timeutils.normalize_time(
timeutils.parse_isotime(identity_token['expires_at']))
if timeutils.is_soon(expiry_time, STALE_TOKEN_DURATION):
LOG.info("The cached keystone token for subcloud %s will "
"expire soon %s" %
(region_name, identity_token['expires_at']))
# Reset the cached dictionary
OpenStackDriver.os_clients_dict[region_name] = \
collections.defaultdict(dict)
OpenStackDriver._identity_tokens[region_name] = None
return False
else:
return True
def _get_management_address_pool(self, context):
"""Get the system controller's management address pool"""
session = KeystoneClient().endpoint_cache.get_session_from_token(
context.auth_token, context.project)
sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session)
return sysinv_client.get_management_address_pool()
def _periodic_subcloud_audit_loop(self):
"""Audit availability of subclouds loop."""
# We will be running in our own green thread here.
LOG.info('Triggered subcloud audit.')
# For each subcloud, if at least one service is active in
# each service of servicegroup-list then declare the subcloud online.
for subcloud in db_api.subcloud_get_all(self.context):
subcloud_name = subcloud.name
subcloud_id = subcloud.id
management_state = subcloud.management_state
avail_status_current = subcloud.availability_status
audit_fail_count = subcloud.audit_fail_count
# Set defaults to None and disabled so we will still set disabled
# status if we encounter an error.
sysinv_client = None
svc_groups = None
avail_to_set = consts.AVAILABILITY_OFFLINE
try:
ks_client = KeystoneClient(subcloud_name)
sysinv_client = SysinvClient(subcloud_name, ks_client.session)
except (keystone_exceptions.EndpointNotFound,
keystone_exceptions.ConnectFailure, IndexError) as e:
if avail_status_current == consts.AVAILABILITY_OFFLINE:
LOG.info("Identity or Platform endpoint for %s not "
"found, ignoring for offline "
"subcloud." % subcloud_name)
continue
else:
LOG.error("Identity or Platform endpoint for online "
"subcloud: %s not found." % subcloud_name)
except Exception as e:
LOG.exception(e)
if sysinv_client:
try:
svc_groups = sysinv_client.get_service_groups()
except Exception as e:
svc_groups = None
LOG.warn('Cannot retrieve service groups for '
'subcloud:%s, %s' % (subcloud_name, e))
if svc_groups:
active_sgs = []
inactive_sgs = []
# Build 2 lists, 1 of active service groups,
# one with non-active.
for sg in svc_groups:
if sg.state != consts.SERVICE_GROUP_STATUS_ACTIVE:
inactive_sgs.append(sg.service_group_name)
else:
active_sgs.append(sg.service_group_name)
# Create a list of service groups that are only present
# in non-active list
inactive_only = [
sg for sg in inactive_sgs if sg not in active_sgs
]
# An empty inactive only list and a non-empty active list
# means we're good to go.
if not inactive_only and active_sgs:
avail_to_set = \
consts.AVAILABILITY_ONLINE
else:
LOG.info("Subcloud:%s has non-active "
"service groups: %s" %
(subcloud_name, inactive_only))
if avail_to_set == consts.AVAILABILITY_OFFLINE:
if audit_fail_count < consts.AVAIL_FAIL_COUNT_MAX:
audit_fail_count = audit_fail_count + 1
if (avail_status_current == consts.AVAILABILITY_ONLINE) and \
(audit_fail_count < consts.AVAIL_FAIL_COUNT_TO_ALARM):
# Do not set offline until we have failed audit
# the requisite number of times
avail_to_set = consts.AVAILABILITY_ONLINE
else:
# In the case of a one off blip, we may need to set the
# fail count back to 0
audit_fail_count = 0
if avail_to_set != avail_status_current:
if avail_to_set == consts.AVAILABILITY_ONLINE:
audit_fail_count = 0
LOG.info('Setting new availability status: %s '
'on subcloud: %s' % (avail_to_set, subcloud_name))
entity_instance_id = "subcloud=%s" % subcloud_name
fault = self.fm_api.get_fault(
fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
entity_instance_id)
if fault and (avail_to_set == consts.AVAILABILITY_ONLINE):
try:
self.fm_api.clear_fault(
fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
entity_instance_id)
except Exception as e:
LOG.exception(e)
elif not fault and \
(avail_to_set == consts.AVAILABILITY_OFFLINE):
try:
fault = fm_api.Fault(
alarm_id=fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE,
alarm_state=fm_const.FM_ALARM_STATE_SET,
entity_type_id=fm_const.FM_ENTITY_TYPE_SUBCLOUD,
entity_instance_id=entity_instance_id,
severity=fm_const.FM_ALARM_SEVERITY_CRITICAL,
reason_text=('%s is offline' % subcloud_name),
alarm_type=fm_const.FM_ALARM_TYPE_0,
probable_cause=fm_const.ALARM_PROBABLE_CAUSE_29,
proposed_repair_action="Wait for subcloud to "
"become online; if "
"problem persists contact "
"next level of support.",
service_affecting=True)
self.fm_api.set_fault(fault)
except Exception as e:
LOG.exception(e)
try:
db_api.subcloud_update(self.context,
subcloud_id,
management_state=None,
availability_status=avail_to_set,
software_version=None,
description=None,
location=None,
audit_fail_count=audit_fail_count)
except exceptions.SubcloudNotFound:
# slim possibility subcloud could have been deleted since
# we found it in db, ignore this benign error.
LOG.info('Ignoring SubcloudNotFound when attempting state'
' update: %s' % subcloud_name)
continue
try:
self.dcorch_rpc_client.\
update_subcloud_states(self.context,
subcloud_name,
management_state,
avail_to_set)
LOG.info('Notifying dcorch, subcloud:%s management: %s, '
'availability:%s' %
(subcloud_name, management_state, avail_to_set))
except Exception as e:
LOG.exception(e)
LOG.warn('Problem informing dcorch of subcloud '
'state change, subcloud: %s' % subcloud_name)
if avail_to_set == consts.AVAILABILITY_OFFLINE:
# Subcloud is going offline, set all endpoint statuses to
# unknown.
try:
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud_name,
endpoint_type=None,
sync_status=consts.SYNC_STATUS_UNKNOWN)
except exceptions.SubcloudNotFound:
LOG.info('Ignoring SubcloudNotFound when attempting '
'sync_status update: %s' % subcloud_name)
continue
elif audit_fail_count != subcloud.audit_fail_count:
try:
db_api.subcloud_update(self.context,
subcloud_id,
management_state=None,
availability_status=None,
software_version=None,
description=None,
location=None,
audit_fail_count=audit_fail_count)
except exceptions.SubcloudNotFound:
# slim possibility subcloud could have been deleted since
# we found it in db, ignore this benign error.
LOG.info('Ignoring SubcloudNotFound when attempting '
'audit_fail_count update: %s' % subcloud_name)
continue
def add_subcloud(self, context, payload):
"""Add subcloud and notify orchestrators.
:param context: request context object
:param name: name of subcloud to add
:param payload: subcloud configuration
"""
LOG.info("Adding subcloud %s." % payload['name'])
try:
subcloud = db_api.subcloud_get_by_name(context, payload['name'])
except exceptions.SubcloudNameNotFound:
pass
else:
raise exceptions.BadRequest(
resource='subcloud',
msg='Subcloud with that name already exists')
# Subcloud is added with software version that matches system
# controller.
software_version = SW_VERSION
try:
subcloud = db_api.subcloud_create(
context, payload['name'], payload.get('description'),
payload.get('location'), software_version,
payload['management-subnet'], payload['management-gateway-ip'],
payload['management-start-ip'], payload['management-end-ip'],
payload['systemcontroller-gateway-ip'])
except Exception as e:
LOG.exception(e)
raise e
# Populate the subcloud status table with all endpoints
for endpoint in dcorch_consts.ENDPOINT_TYPES_LIST:
db_api.subcloud_status_create(context, subcloud.id, endpoint)
try:
# Create a new route to this subcloud on the management interface
# on both controllers.
m_ks_client = KeystoneClient()
subcloud_subnet = netaddr.IPNetwork(payload['management-subnet'])
session = m_ks_client.endpoint_cache.get_session_from_token(
context.auth_token, context.project)
sysinv_client = SysinvClient(consts.DEFAULT_REGION_NAME, session)
controllers = sysinv_client.get_controller_hosts()
for controller in controllers:
management_interface = sysinv_client.get_management_interface(
controller.hostname)
if management_interface is not None:
sysinv_client.create_route(
management_interface.uuid, str(subcloud_subnet.ip),
subcloud_subnet.prefixlen,
payload['systemcontroller-gateway-ip'], 1)
# Create identity endpoints to this subcloud on the
# management-start-ip of the subcloud which will be allocated
# as the floating Management IP of the Subcloud if the
# Address Pool is not shared. Incase the endpoint entry
# is incorrect, or the management IP of the subcloud is changed
# in the future, it will not go managed or will show up as
# out of sync. To fix this use Openstack endpoint commands
# on the SystemController to change the subcloud endpoint
ks_service_id = None
for service in m_ks_client.services_list:
if service.type == dcorch_consts.ENDPOINT_TYPE_IDENTITY:
ks_service_id = service.id
break
else:
raise exceptions.BadRequest(
resource='subcloud',
msg='No Identity service found on SystemController')
identity_endpoint_ip = payload['management-start-ip']
if netaddr.IPAddress(identity_endpoint_ip).version == 6:
identity_endpoint_url = \
"http://[{}]:5000/v3".format(identity_endpoint_ip)
else:
identity_endpoint_url = \
"http://{}:5000/v3".format(identity_endpoint_ip)
for iface in ['internal', 'admin']:
m_ks_client.keystone_client.endpoints.create(
ks_service_id,
identity_endpoint_url,
interface=iface,
region=subcloud.name)
# Inform orchestrator that subcloud has been added
self.dcorch_rpc_client.add_subcloud(context, subcloud.name,
subcloud.software_version)
# Regenerate the addn_hosts_dc file
self._create_addn_hosts_dc(context)
return db_api.subcloud_db_model_to_dict(subcloud)
except Exception as e:
LOG.exception(e)
# If we failed to create the subcloud, clean up anything we may
# have done.
self._delete_subcloud_routes(context, subcloud)
db_api.subcloud_destroy(context, subcloud.id)
raise e
def delete_subcloud(self, context, subcloud_id):
"""Delete subcloud and notify orchestrators.
:param context: request context object.
:param subcloud_id: id of subcloud to delete
"""
LOG.info("Deleting subcloud %s." % subcloud_id)
# Retrieve the subcloud details from the database
subcloud = db_api.subcloud_get(context, subcloud_id)
# Semantic checking
if subcloud.management_state != consts.MANAGEMENT_UNMANAGED:
raise exceptions.SubcloudNotUnmanaged()
if subcloud.availability_status == \
consts.AVAILABILITY_ONLINE:
raise exceptions.SubcloudNotOffline()
# Inform orchestrators that subcloud has been deleted
try:
self.dcorch_rpc_client.del_subcloud(context, subcloud.name)
except RemoteError as e:
if "SubcloudNotFound" in e:
pass
# We only delete subcloud endpoints, region and user information
# in the Central Region. The subcloud is already unmanaged and powered
# down so is not accessible. Therefore set up a session with the
# Central Region Keystone ONLY.
keystone_client = KeystoneClient()
# Delete keystone endpoints for subcloud
keystone_client.delete_endpoints(subcloud.name)
keystone_client.delete_region(subcloud.name)
# Delete the routes to this subcloud
self._delete_subcloud_routes(context, subcloud)
# Remove the subcloud from the database
try:
db_api.subcloud_destroy(context, subcloud_id)
except Exception as e:
LOG.exception(e)
raise e
# Clear the offline fault associated with this subcloud as we
# are deleting it. Note that endpoint out-of-sync alarms should
# have been cleared when the subcloud was unmanaged and the endpoint
# sync statuses were set to unknown.
entity_instance_id = "subcloud=%s" % subcloud.name
try:
subcloud_offline = fm_const.FM_ALARM_ID_DC_SUBCLOUD_OFFLINE
fault = self.fm_api.get_fault(subcloud_offline, entity_instance_id)
if fault:
self.fm_api.clear_fault(subcloud_offline, entity_instance_id)
except Exception as e:
LOG.info("Problem clearing offline fault for "
"subcloud %s" % subcloud.name)
LOG.exception(e)
# Regenerate the addn_hosts_dc file
self._create_addn_hosts_dc(context)
def _periodic_patch_audit_loop(self):
"""Audit patch status of subclouds loop."""
# We are running in our own green thread here.
LOG.info('Triggered patch audit.')
try:
ks_client = KeystoneClient()
except Exception:
LOG.warn('Failure initializing KeystoneClient, exiting audit.')
return
# First query RegionOne to determine what patches should be applied
# to the system.
patching_client = PatchingClient(consts.DEFAULT_REGION_NAME,
ks_client.session)
regionone_patches = patching_client.query()
LOG.debug("regionone_patches: %s" % regionone_patches)
# Build lists of patches that should be applied or committed in all
# subclouds, based on their state in RegionOne. Check repostate
# (not patchstate) as we only care if the patch has been applied to
# the repo (not whether it is installed on the hosts).
applied_patch_ids = list()
committed_patch_ids = list()
for patch_id in regionone_patches.keys():
if regionone_patches[patch_id]['repostate'] == \
patching_v1.PATCH_STATE_APPLIED:
applied_patch_ids.append(patch_id)
elif regionone_patches[patch_id]['repostate'] == \
patching_v1.PATCH_STATE_COMMITTED:
committed_patch_ids.append(patch_id)
LOG.debug("RegionOne applied_patch_ids: %s" % applied_patch_ids)
LOG.debug("RegionOne committed_patch_ids: %s" % committed_patch_ids)
# For each subcloud, check whether the patches match the target.
for subcloud in db_api.subcloud_get_all(self.context):
# Only audit patching on subclouds that are managed and online
if (subcloud.management_state != consts.MANAGEMENT_MANAGED
or subcloud.availability_status !=
consts.AVAILABILITY_ONLINE):
continue
try:
sc_ks_client = KeystoneClient(subcloud.name)
except (keystone_exceptions.EndpointNotFound, IndexError) as e:
LOG.warn(
"Identity endpoint for online subcloud % not found. %" %
(subcloud.name, e))
continue
try:
patching_client = PatchingClient(subcloud.name,
sc_ks_client.session)
except keystone_exceptions.EndpointNotFound:
LOG.warn(
"Patching endpoint for online subcloud %s not found." %
subcloud.name)
continue
try:
sysinv_client = SysinvClient(subcloud.name,
sc_ks_client.session)
except keystone_exceptions.EndpointNotFound:
LOG.warn("Sysinv endpoint for online subcloud %s not found." %
subcloud.name)
continue
# Retrieve all the patches that are present in this subcloud.
try:
subcloud_patches = patching_client.query()
LOG.debug("Patches for subcloud %s: %s" %
(subcloud.name, subcloud_patches))
except Exception:
LOG.warn('Cannot retrieve patches for subcloud: %s' %
subcloud.name)
continue
# Determine which loads are present in this subcloud. During an
# upgrade, there will be more than one load installed.
installed_loads = list()
try:
loads = sysinv_client.get_loads()
except Exception:
LOG.warn('Cannot retrieve loads for subcloud: %s' %
subcloud.name)
continue
for load in loads:
installed_loads.append(load.software_version)
out_of_sync = False
# Check that all patches in this subcloud are in the correct
# state, based on the state of the patch in RegionOne. For the
# subcloud, we use the patchstate because we care whether the
# patch is installed on the hosts.
for patch_id in subcloud_patches.keys():
if subcloud_patches[patch_id]['patchstate'] == \
patching_v1.PATCH_STATE_APPLIED:
if patch_id not in applied_patch_ids:
if patch_id not in committed_patch_ids:
LOG.debug("Patch %s should not be applied in %s" %
(patch_id, subcloud.name))
else:
LOG.debug("Patch %s should be committed in %s" %
(patch_id, subcloud.name))
out_of_sync = True
elif subcloud_patches[patch_id]['patchstate'] == \
patching_v1.PATCH_STATE_COMMITTED:
if patch_id not in committed_patch_ids:
LOG.warn("Patch %s should not be committed in %s" %
(patch_id, subcloud.name))
out_of_sync = True
else:
# In steady state, all patches should either be applied
# or committed in each subcloud. Patches in other
# states mean a sync is required.
out_of_sync = True
# Check that all applied or committed patches in RegionOne are
# present in the subcloud.
for patch_id in applied_patch_ids:
if regionone_patches[patch_id]['sw_version'] in \
installed_loads and patch_id not in subcloud_patches:
LOG.debug("Patch %s missing from %s" %
(patch_id, subcloud.name))
out_of_sync = True
for patch_id in committed_patch_ids:
if regionone_patches[patch_id]['sw_version'] in \
installed_loads and patch_id not in subcloud_patches:
LOG.debug("Patch %s missing from %s" %
(patch_id, subcloud.name))
out_of_sync = True
if out_of_sync:
LOG.debug("Subcloud %s is out-of-sync for patching" %
subcloud.name)
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud.name,
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_OUT_OF_SYNC)
else:
LOG.debug("Subcloud %s is in-sync for patching" %
subcloud.name)
self.subcloud_manager.update_subcloud_endpoint_status(
self.context,
subcloud_name=subcloud.name,
endpoint_type=dcorch_consts.ENDPOINT_TYPE_PATCHING,
sync_status=consts.SYNC_STATUS_IN_SYNC)