def _nfvi_host_get_callback(nfvi_host_uuid, nfvi_host_name):
"""
NFVI Host get callback
"""
DLOG.debug("Host get, nfvi_host_uuid=%s, nfvi_host_name=%s." %
(nfvi_host_uuid, nfvi_host_name))
instances = 0
instances_failed = 0
instances_stopped = 0
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is not None:
instance_table = tables.tables_get_instance_table()
for instance in instance_table.on_host(host.name):
if instance.is_deleting() or instance.is_deleted():
continue
if instance.is_failed():
instances_failed += 1
if instance.is_locked():
instances_stopped += 1
instances += 1
DLOG.info("Host %s has %s instances, failed=%s, stopped=%s." %
(host.name, instances, instances_failed, instances_stopped))
return True, instances, instances_failed, instances_stopped
def _audit_nfvi_instance_callback(instance_uuid):
"""
Audit Instance
"""
global _nfvi_instance_outstanding
response = (yield)
if instance_uuid in _nfvi_instance_outstanding:
del _nfvi_instance_outstanding[instance_uuid]
if response['completed']:
nfvi_instance = response['result-data']
DLOG.info("Audit-Instance callback for %s" % nfvi_instance.uuid)
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance.uuid, None)
if instance is None:
if nfvi.objects.v1.INSTANCE_AVAIL_STATUS.DELETED \
not in nfvi_instance.avail_status:
instance = objects.Instance(nfvi_instance)
instance_table[instance.uuid] = instance
instance.nfvi_instance_update(nfvi_instance)
else:
if instance.nfvi_instance_audit_in_progress:
# Show that the audit is complete
instance.nfvi_instance_audit_in_progress = False
instance.nfvi_instance_update(nfvi_instance)
else:
DLOG.info("Ignoring stale audit reply for %s" %
nfvi_instance.uuid)
else:
DLOG.error("Audit-Instance callback, not completed, response=%s." %
response)
def _nfvi_instance_delete_callback(nfvi_instance_uuid):
"""
NFVI Instance delete callback
"""
DLOG.info("Instance delete, nfvi_instance_uuid=%s." % nfvi_instance_uuid)
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is not None:
instance.nfvi_instance_delete()
def _nfvi_guest_services_query_callback(nfvi_host_uuid, nfvi_instance_uuid):
"""
NFVI Guest Services query callback
"""
DLOG.debug("Guest-Services query, nfvi_host_uuid=%s, "
"nfvi_instance_uuid=%s." % (nfvi_host_uuid, nfvi_instance_uuid))
# scope of instance
if nfvi_instance_uuid is not None:
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is None:
return False, None
result = dict()
result['uuid'] = instance.uuid
result['hostname'] = instance.host_name
result['services'] = instance.guest_services.get_nfvi_guest_services()
return True, result
# scope of host
host_table = tables.tables_get_host_table()
host = host_table.get_by_uuid(nfvi_host_uuid)
if host is None:
return False, None
instance_table = tables.tables_get_instance_table()
instances = list()
for instance in instance_table.on_host(host.name):
guest_services = instance.guest_services
if guest_services.are_provisioned():
result = dict()
result['uuid'] = instance.uuid
result['hostname'] = instance.host_name
result['services'] = guest_services.get_nfvi_guest_services()
instances.append(result)
results = dict()
results['instances'] = instances
return True, results
def manage_alarms(self):
"""
Manage alarms
"""
from nfv_vim import tables
instance_table = tables.tables_get_instance_table()
for member_uuid in self.member_uuids:
member = instance_table.get(member_uuid, None)
if member is None or member.is_deleted():
continue
for peer_member_uuid in self.member_uuids:
peer_member = instance_table.get(peer_member_uuid, None)
if peer_member is None or peer_member.is_deleted():
continue
if peer_member.uuid == member.uuid:
continue
if INSTANCE_GROUP_POLICY.AFFINITY_BEST_EFFORT in self.policies:
if peer_member.host_name != member.host_name:
if not self._alarms:
additional_text = (
", some instances are not on the same host")
self._alarms = \
alarm.raise_instance_group_policy_alarm(
self,
INSTANCE_GROUP_POLICY.AFFINITY_BEST_EFFORT,
alarm.ALARM_TYPE.
INSTANCE_GROUP_POLICY_CONFLICT,
additional_text=additional_text)
return
elif INSTANCE_GROUP_POLICY.ANTI_AFFINITY_BEST_EFFORT in \
self.policies:
if peer_member.host_name == member.host_name:
if not self._alarms:
additional_text = (
", some instances are on the same host")
self._alarms = \
alarm.raise_instance_group_policy_alarm(
self,
INSTANCE_GROUP_POLICY.
ANTI_AFFINITY_BEST_EFFORT,
alarm.ALARM_TYPE.
INSTANCE_GROUP_POLICY_CONFLICT,
additional_text=additional_text)
return
# No policy conflicts were detected
self.clear_alarms()
def _audit_alarms():
"""
Audit Alarms. This is being done to allow hold off times to be supported
for instance alarms. By auditing the instance alarms every 10 seconds, it
allows a held off alarm to be raised within 10 seconds of the hold off
time expiring.
"""
while True:
timer_id = (yield)
DLOG.verbose("Audit alarms called, timer_id=%s." % timer_id)
instance_table = tables.tables_get_instance_table()
for instance in instance_table.itervalues():
if not instance.is_deleted():
alarm.instance_manage_alarms(instance)
def _nfvi_instance_action_callback(nfvi_instance_uuid, nfvi_action_data):
"""
NFVI Instance action callback
"""
DLOG.debug("Instance action, uuid=%s, nfvi_action_data=%s" %
(nfvi_instance_uuid, nfvi_action_data))
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is not None:
instance.nfvi_instance_action_update(nfvi_action_data)
return True
else:
DLOG.error("Instance %s is not found" % nfvi_instance_uuid)
return False
def _nfvi_guest_services_state_notify_callback(nfvi_instance_uuid,
nfvi_host_name,
nfvi_guest_services):
"""
NFVI Guest Services notify callback for service type event
"""
DLOG.debug("Guest-Services state notify, instance_uuid=%s, "
"host_name=%s guest_services=%s." %
(nfvi_instance_uuid, nfvi_host_name, nfvi_guest_services))
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is not None:
instance.nfvi_guest_services_update(nfvi_guest_services,
nfvi_host_name)
def _audit_nfvi_guest_services():
"""
Audit NFVI Guest Services
"""
while True:
timer_id = (yield)
DLOG.verbose("Audit guest services called, timer_id=%s." % timer_id)
instance_table = tables.tables_get_instance_table()
for instance_uuid in instance_table.keys():
instance = instance_table[instance_uuid]
if instance is not None:
if not instance.is_deleted():
if instance.guest_services.are_provisioned():
nfvi.nfvi_guest_services_query(
instance_uuid, _audit_nfvi_guest_services_callback())
def vim_instance_api_start_instance(connection, msg):
"""
Handle Start-Instance API request
"""
DLOG.verbose("Start instance %s." % msg.uuid)
instance_table = tables.tables_get_instance_table()
response = rpc.APIResponseStartInstance()
instance = instance_table.get(msg.uuid, None)
if instance is not None:
instance.do_action(objects.INSTANCE_ACTION_TYPE.START)
response.uuid = msg.uuid
else:
response.result = rpc.RPC_MSG_RESULT.NOT_FOUND
connection.send(response.serialize())
DLOG.verbose("Sent response=%s" % response)
connection.close()
def _nfvi_instance_state_change_callback(nfvi_instance):
"""
NFVI Instance state change callback
"""
DLOG.debug("Instance state-change, nfvi_instance=%s." % nfvi_instance)
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance.uuid, None)
if instance is None:
if nfvi.objects.v1.INSTANCE_AVAIL_STATUS.DELETED \
not in nfvi_instance.avail_status:
nfvi.nfvi_get_instance(nfvi_instance.uuid,
_query_nfvi_instance_callback())
else:
# We are handling a notification from nova, which will not have all
# the data for the nfvi_instance we store as part of our instance
# object. As part of the processing of the notification, we are going
# to save the nfvi_instance to the database, so make sure we don't
# overwrite data that did not come in the notification, by retrieving
# it from the instance object. Yes - this is ugly. No - I'm not going
# to rewrite this now.
if nfvi_instance.tenant_id is None:
nfvi_instance.tenant_id = instance.tenant_uuid
if nfvi_instance.instance_type is None:
nfvi_instance.instance_type = instance._nfvi_instance.instance_type
if nfvi_instance.image_uuid is None:
nfvi_instance.image_uuid = instance.image_uuid
if not nfvi_instance.attached_volumes:
nfvi_instance.attached_volumes = instance.attached_volumes
if nfvi_instance.recovery_priority is None:
nfvi_instance.recovery_priority = \
instance._nfvi_instance.recovery_priority
if nfvi_instance.live_migration_timeout is None:
nfvi_instance.live_migration_timeout = \
instance._nfvi_instance.live_migration_timeout
instance.nfvi_instance_update(nfvi_instance)
if nfvi.objects.v1.INSTANCE_AVAIL_STATUS.DELETED \
in nfvi_instance.avail_status:
instance.nfvi_instance_deleted()
return True
def vim_instance_api_delete_instance(connection, msg):
"""
Handle Delete-Instance API request
"""
DLOG.verbose("Delete instance %s." % msg.uuid)
instance_table = tables.tables_get_instance_table()
response = rpc.APIResponseDeleteInstance()
instance = instance_table.get(msg.uuid, None)
if instance is not None:
instance_director = directors.get_instance_director()
instance_director.delete_instance(instance)
response.uuid = msg.uuid
else:
response.result = rpc.RPC_MSG_RESULT.NOT_FOUND
connection.send(response.serialize())
DLOG.verbose("Sent response=%s" % response)
connection.close()
def _create_instance_callback(success, instance_name, instance_uuid):
"""
Handle Create-Instance callback
"""
DLOG.verbose("Create instance callback, name=%s." % instance_name)
connection = _instance_create_operations.get(instance_name, None)
if connection is not None:
instance_table = tables.tables_get_instance_table()
response = rpc.APIResponseCreateInstance()
if success:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
response.uuid = instance.uuid
response.name = instance.name
response.admin_state = instance.admin_state
response.oper_state = instance.oper_state
response.avail_status = instance.avail_status
response.action = instance.action
response.host_name = instance.host_name
response.instance_type_original_name \
= instance.instance_type_original_name
response.image_uuid = instance.image_uuid
response.vcpus = instance.vcpus
response.memory_mb = instance.memory_mb
response.disk_gb = instance.disk_gb
response.ephemeral_gb = instance.ephemeral_gb
response.swap_gb = instance.swap_gb
response.auto_recovery = instance.auto_recovery
response.live_migration_timeout \
= instance.max_live_migrate_wait_in_secs
response.live_migration_max_downtime \
= instance.max_live_migration_downtime_in_ms
if instance.host_name is not None:
host_table = tables.tables_get_host_table()
host = host_table.get(instance.host_name, None)
if host is not None:
response.host_uuid = host.uuid
else:
response.result = rpc.RPC_MSG_RESULT.FAILED
else:
response.result = rpc.RPC_MSG_RESULT.FAILED
connection.send(response.serialize())
connection.close()
DLOG.info("Sent response=%s" % response)
del _instance_create_operations[instance_name]
def _audit_nfvi_instance():
"""
Audit NFVI for Instance Details
"""
global _nfvi_instances_to_audit, _nfvi_instance_outstanding
while True:
timer_id = (yield)
DLOG.verbose("Audit instance called, timer_id=%s." % timer_id)
instance_table = tables.tables_get_instance_table()
for instance_uuid in _nfvi_instance_outstanding.keys():
instance = instance_table.get(instance_uuid, None)
if instance is None:
del _nfvi_instance_outstanding[instance_uuid]
if 0 < len(_nfvi_instance_outstanding):
DLOG.info(
"Audit instance queries still outstanding, outstanding=%s" %
_nfvi_instance_outstanding)
_audit_dump_debug_info()
else:
_audit_dump_debug_info(do_dump=False)
for instance_uuid in _nfvi_instances_to_audit.keys():
if 4 <= len(_nfvi_instance_outstanding):
break
do_audit = True
instance = instance_table.get(instance_uuid, None)
if instance is not None:
if instance.nfvi_instance_is_deleted():
do_audit = False
else:
# Indicate that audit is in progress
instance.nfvi_instance_audit_in_progress = True
if do_audit:
DLOG.info("Auditing instance %s." % instance_uuid)
nfvi.nfvi_get_instance(
instance_uuid,
_audit_nfvi_instance_callback(instance_uuid))
_nfvi_instance_outstanding[instance_uuid] \
= _nfvi_instances_to_audit[instance_uuid]
del _nfvi_instances_to_audit[instance_uuid]
def _nfvi_instance_action_change_callback(nfvi_instance_uuid,
nfvi_action_type,
nfvi_action_state,
reason=""):
"""
NFVI Instance action change callback
"""
DLOG.debug(
"Instance action-change, uuid=%s, nfvi_action=%s, "
"nfvi_action_state=%s, reason=%s." %
(nfvi_instance_uuid, nfvi_action_type, nfvi_action_state, reason))
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is not None:
instance.nfvi_instance_action_change(nfvi_action_type,
nfvi_action_state, reason)
return True
def _nfvi_guest_services_action_notify_callback(nfvi_instance_uuid,
nfvi_action_type,
nfvi_action_state, reason):
"""
NFVI Guest Services notify callback for action type event
"""
DLOG.debug(
"Guest-Services action notify, instance_uuid=%s, "
"nfvi_action_type=%s, nfvi_action_state=%s, reason=%s." %
(nfvi_instance_uuid, nfvi_action_type, nfvi_action_state, reason))
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is None:
return False
instance.nfvi_instance_action_change(nfvi_action_type, nfvi_action_state,
reason)
return True
def vim_instance_api_get_instance(connection, msg):
"""
Handle Get-Instance API request
"""
DLOG.verbose("Get instance, filter_by_uuid=%s." % msg.filter_by_uuid)
instance_table = tables.tables_get_instance_table()
response = rpc.APIResponseGetInstance()
instance = instance_table.get(msg.filter_by_uuid, None)
if instance is not None:
response.uuid = instance.uuid
response.name = instance.name
response.admin_state = instance.admin_state
response.oper_state = instance.oper_state
response.avail_status = instance.avail_status
response.action = instance.action
response.host_name = instance.host_name
response.instance_type_original_name \
= instance.instance_type_original_name
response.image_uuid = instance.image_uuid
response.vcpus = instance.vcpus
response.memory_mb = instance.memory_mb
response.disk_gb = instance.disk_gb
response.ephemeral_gb = instance.ephemeral_gb
response.swap_gb = instance.swap_gb
response.auto_recovery = instance.auto_recovery
response.live_migration_timeout \
= instance.max_live_migrate_wait_in_secs
response.live_migration_max_downtime \
= instance.max_live_migration_downtime_in_ms
if instance.host_name is not None:
host_table = tables.tables_get_host_table()
host = host_table.get(instance.host_name, None)
if host is not None:
response.host_uuid = host.uuid
else:
response.result = rpc.RPC_MSG_RESULT.NOT_FOUND
connection.send(response.serialize())
DLOG.verbose("Sent response=%s" % response)
connection.close()
def _audit_nfvi_guest_services_callback():
"""
Audit Guest Services
"""
response = (yield)
DLOG.verbose("Audit-Guest-Services callback, response=%s." % response)
if response['completed']:
result_data = response.get('result-data', None)
if result_data is not None:
instance_uuid = result_data['instance_uuid']
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(instance_uuid, None)
if instance is not None:
host_name = result_data.get('host_name', None)
nfvi_guest_services = result_data.get('services', list())
instance.nfvi_guest_services_update(nfvi_guest_services,
host_name)
else:
DLOG.error("Audit-Guest-Services callback, not completed, "
"response=%s." % response)
def _query_nfvi_instance_callback():
"""
Query Instance
"""
response = (yield)
DLOG.verbose("Query-Instance callback, response=%s." % response)
if response['completed']:
nfvi_instance = response['result-data']
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance.uuid, None)
if instance is None:
if nfvi.objects.v1.INSTANCE_AVAIL_STATUS.DELETED \
not in nfvi_instance.avail_status:
instance = objects.Instance(nfvi_instance)
instance_table[instance.uuid] = instance
instance.nfvi_instance_update(nfvi_instance)
else:
instance.nfvi_instance_update(nfvi_instance)
else:
DLOG.error("Query-Instance callback, not completed, responses=%s." %
response)
def _nfvi_guest_services_alarm_notify_callback(nfvi_instance_uuid,
nfvi_avail_status,
nfvi_repair_action):
"""
NFVI Guest Services notify callback for alarm type event
"""
DLOG.debug("Guest-Services alarm notify, instance_uuid=%s, "
"avail_status=%s, repair_action=%s." %
(nfvi_instance_uuid, nfvi_avail_status, nfvi_repair_action))
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is None:
return False
if nfvi.objects.v1.INSTANCE_AVAIL_STATUS.FAILED == nfvi_avail_status:
if nfvi.objects.v1.INSTANCE_ACTION_TYPE.REBOOT == nfvi_repair_action:
instance.guest_services_failed(do_soft_reboot=True)
elif nfvi.objects.v1.INSTANCE_ACTION_TYPE.STOP == nfvi_repair_action:
instance.guest_services_failed(do_stop=True)
else:
instance.guest_services_failed()
elif nfvi.objects.v1.INSTANCE_AVAIL_STATUS.UNHEALTHY == nfvi_avail_status:
if nfvi.objects.v1.INSTANCE_ACTION_TYPE.REBOOT == nfvi_repair_action:
instance.guest_services_failed(do_soft_reboot=True,
health_check_failed_only=True)
elif nfvi.objects.v1.INSTANCE_ACTION_TYPE.STOP == nfvi_repair_action:
instance.guest_services_failed(do_stop=True,
health_check_failed_only=True)
else:
instance.guest_services_failed(health_check_failed_only=True)
result = dict()
result['uuid'] = instance.uuid
result['hostname'] = instance.host_name
result['services'] = instance.guest_services.get_nfvi_guest_services()
return True, result
def _nfvi_host_notification_callback(host_ip, nfvi_notify_type,
nfvi_notify_data):
"""
NFVI Host notification callback
"""
instance_count = 0
status = 'error'
if nfvi.objects.v1.HOST_NOTIFICATIONS.BOOTING == nfvi_notify_type:
host_name = nfvi_notify_data.get('hostname', None)
if host_name is not None:
DLOG.info(
"Booting notification received for host %s, host-ip=%s." %
(host_name, host_ip))
instance_table = tables.tables_get_instance_table()
for instance in instance_table.on_host(host_name):
if not (instance.is_deleting() or instance.is_deleted()
or instance.is_failed() or instance.is_locked()
or instance.is_rebuilding() or instance.is_migrating()
or instance.is_rebooting()):
DLOG.info(
"Failing instance %s because host %s is booting, "
"host-ip=%s." % (instance.name, host_name, host_ip))
instance.fail('host booting')
instance_count += 1
if 0 == instance_count:
status = 'okay'
else:
status = 'accepted'
else:
DLOG.error("Unknown host notification received, type=%s, data=%s" %
(nfvi_notify_type, nfvi_notify_data))
return status
def _audit_nfvi_instances_callback(timer_id):
"""
Audit Instances
"""
global _main_audit_inprogress
global _deletable_instances, _nfvi_instances_paging
global _nfvi_instances_to_audit, _nfvi_instance_outstanding
response = (yield)
DLOG.verbose("Audit-Instances callback, response=%s." % response)
trigger_recovery = False
if response['completed']:
if response[
'page-request-id'] == _nfvi_instances_paging.page_request_id:
instance_table = tables.tables_get_instance_table()
if _deletable_instances is None:
_deletable_instances = list(instance_table)
for instance_uuid, instance_name in response['result-data']:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
if instance.uuid in _deletable_instances:
_deletable_instances.remove(instance.uuid)
if instance_uuid not in _nfvi_instances_to_audit:
_nfvi_instances_to_audit[instance_uuid] = instance_name
if _nfvi_instances_paging.done:
for instance_uuid in _deletable_instances:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
DLOG.info("Deleting instance %s, audit mismatch" %
instance_uuid)
instance.nfvi_instance_deleted()
if instance.is_deleted():
trigger_recovery = True
del instance_table[instance_uuid]
if instance_uuid in _nfvi_instances_to_audit:
del _nfvi_instances_to_audit[instance_uuid]
if instance_uuid in _nfvi_instance_outstanding:
del _nfvi_instance_outstanding[instance_uuid]
_deletable_instances = list(instance_table)
_nfvi_instances_paging.first_page()
else:
DLOG.verbose("Paging is not done for instances.")
else:
DLOG.error("Audit-Instances callback, page-request-id mismatch, "
"responses=%s, page-request-id=%s." %
(response, _nfvi_instances_paging.page_request_id))
instance_table = tables.tables_get_instance_table()
_deletable_instances = list(instance_table)
_nfvi_instances_paging.first_page()
else:
DLOG.error("Audit-Instances callback, not completed, responses=%s." %
response)
instance_table = tables.tables_get_instance_table()
_deletable_instances = list(instance_table)
_nfvi_instances_paging.first_page()
_nfvi_instances_paging.set_page_request_id()
_main_audit_inprogress = False
timers.timers_reschedule_timer(timer_id, 2) # 2 seconds later
if trigger_recovery:
# Resources have been freed, there is potential to recover instances.
DLOG.info("Recover-Instances-Audit triggered by instance deletion.")
instance_director = directors.get_instance_director()
instance_director.recover_instances()