def swact_hosts(self, host_names):
"""
Swact a list of hosts
"""
DLOG.info("Swact hosts: %s" % host_names)
host_operation = Operation(OPERATION_TYPE.SWACT_HOSTS)
if self._host_operation is not None:
DLOG.debug("Canceling previous host operation %s, before "
"continuing with host operation %s." %
(self._host_operation.operation_type,
host_operation.operation_type))
self._host_operation = None
host_table = tables.tables_get_host_table()
for host_name in host_names:
host = host_table.get(host_name, None)
if host is None:
reason = "Unknown host %s given." % host_name
DLOG.info(reason)
host_operation.set_failed(reason)
return host_operation
host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS)
self._nfvi_swact_host(host.uuid, host.name)
if host_operation.is_inprogress():
self._host_operation = host_operation
return host_operation
def _nfvi_host_get_callback(nfvi_host_uuid, nfvi_host_name):
"""
NFVI Host get callback
"""
DLOG.debug("Host get, nfvi_host_uuid=%s, nfvi_host_name=%s." %
(nfvi_host_uuid, nfvi_host_name))
instances = 0
instances_failed = 0
instances_stopped = 0
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is not None:
instance_table = tables.tables_get_instance_table()
for instance in instance_table.on_host(host.name):
if instance.is_deleting() or instance.is_deleted():
continue
if instance.is_failed():
instances_failed += 1
if instance.is_locked():
instances_stopped += 1
instances += 1
DLOG.info("Host %s has %s instances, failed=%s, stopped=%s." %
(host.name, instances, instances_failed, instances_stopped))
return True, instances, instances_failed, instances_stopped
def _nfvi_host_action_callback(nfvi_host_uuid, nfvi_host_name, do_action):
"""
NFVI host action callback
"""
DLOG.debug("Host action, host_uuid=%s, host_name=%s, do_action=%s." %
(nfvi_host_uuid, nfvi_host_name, do_action))
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is not None:
if nfvi.objects.v1.HOST_ACTION.UNLOCK == do_action:
host.unlock()
elif nfvi.objects.v1.HOST_ACTION.LOCK == do_action:
host.lock()
elif nfvi.objects.v1.HOST_ACTION.LOCK_FORCE == do_action:
host.lock(force=True)
elif nfvi.objects.v1.HOST_ACTION.DELETE == do_action:
host.nfvi_host_delete()
else:
DLOG.info("Unknown action %s received for %s." %
(do_action, nfvi_host_name))
return True
def _nfvi_disable_host_services_callback(self):
"""
NFVI Disable Host Services Callback
"""
from nfv_vim import directors
response = (yield)
DLOG.verbose("NFVI Disable Host Services callback response=%s." %
response)
if not response['completed']:
DLOG.info(
"Disable of host services on host %s failed, reason=%s." %
(response['host_name'], response['reason']))
host_table = tables.tables_get_host_table()
host = host_table.get(response['host_name'], None)
if host is None:
DLOG.verbose("Host %s does not exist." % response['host_name'])
return
if self._host_operation is None:
DLOG.verbose("No host %s operation inprogress." % host.name)
return
if OPERATION_TYPE.DISABLE_HOST_SERVICES != \
self._host_operation.operation_type:
DLOG.verbose("Unexpected host %s operation %s, ignoring." %
(host.name, self._host_operation.operation_type))
return
sw_mgmt_director = directors.get_sw_mgmt_director()
sw_mgmt_director.disable_host_services_failed(host)
def _nfvi_lock_host_callback(self):
"""
NFVI Lock Host Callback
"""
from nfv_vim import directors
response = (yield)
DLOG.verbose("NFVI Lock Host callback response=%s." % response)
if not response['completed']:
DLOG.info("Lock of host %s failed, reason=%s." %
(response['host_name'], response['reason']))
host_table = tables.tables_get_host_table()
host = host_table.get(response['host_name'], None)
if host is None:
DLOG.verbose("Host %s does not exist." % response['host_name'])
return
if self._host_operation is None:
DLOG.verbose("No host %s operation inprogress." % host.name)
return
if OPERATION_TYPE.LOCK_HOSTS != self._host_operation.operation_type:
DLOG.verbose("Unexpected host %s operation %s, ignoring." %
(host.name, self._host_operation.operation_type))
return
sw_mgmt_director = directors.get_sw_mgmt_director()
sw_mgmt_director.host_lock_failed(host)
def _nfvi_periodic_timer_event():
"""
Periodic timer for hosts
"""
while True:
timer_id = (yield)
DLOG.verbose("NFVI periodic timer called, timer_id=%s." % timer_id)
host_table = tables.tables_get_host_table()
for host in host_table.values():
host.periodic_timer()
def _nfvi_host_upgrade_callback(nfvi_host_uuid, nfvi_host_name,
upgrade_inprogress, recover_instances):
"""
NFVI Host upgrade callback
"""
DLOG.debug("Host upgrade, nfvi_host_uuid=%s, nfvi_host_name=%s, "
"upgrade_inprogress=%s, recover_instances=%s." %
(nfvi_host_uuid, nfvi_host_name, upgrade_inprogress,
recover_instances))
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is not None:
host.nfvi_host_upgrade_status(upgrade_inprogress, recover_instances)
return True
def _nfvi_host_add_callback(nfvi_host_uuid, nfvi_host_name):
"""
NFVI Host add callback
"""
DLOG.debug("Host add, nfvi_host=%s." % nfvi_host_name)
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is None:
nfvi.nfvi_get_host(nfvi_host_uuid, nfvi_host_name,
_nfvi_host_query_callback())
else:
host.nfvi_host_add()
return True
def oper_state(self):
"""
Returns the current operational state of the hypervisor
"""
from nfv_vim import tables
host_table = tables.tables_get_host_table()
host = host_table.get(self.host_name, None)
if host is not None:
if host.is_failed() and not host.is_component_failure():
return nfvi.objects.v1.HYPERVISOR_OPER_STATE.DISABLED
elif host.is_offline():
return nfvi.objects.v1.HYPERVISOR_OPER_STATE.DISABLED
return self._nfvi_hypervisor.oper_state # assume one-to-one mapping
def _nfvi_host_services_query_callback(nfvi_host_name):
"""
NFVI Host Services query callback
"""
DLOG.debug("Host-Services query, host_name=%s." % nfvi_host_name)
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is None:
return False, None
if host.nfvi_host_is_enabled():
host_oper_state = nfvi.objects.v1.HOST_OPER_STATE.ENABLED
else:
host_oper_state = nfvi.objects.v1.HOST_OPER_STATE.DISABLED
return True, host_oper_state
def _create_instance_callback(success, instance_name, instance_uuid):
"""
Handle Create-Instance callback
"""
DLOG.verbose("Create instance callback, name=%s." % instance_name)
connection = _instance_create_operations.get(instance_name, None)
if connection is not None:
instance_table = tables.tables_get_instance_table()
response = rpc.APIResponseCreateInstance()
if success:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
response.uuid = instance.uuid
response.name = instance.name
response.admin_state = instance.admin_state
response.oper_state = instance.oper_state
response.avail_status = instance.avail_status
response.action = instance.action
response.host_name = instance.host_name
response.instance_type_original_name \
= instance.instance_type_original_name
response.image_uuid = instance.image_uuid
response.vcpus = instance.vcpus
response.memory_mb = instance.memory_mb
response.disk_gb = instance.disk_gb
response.ephemeral_gb = instance.ephemeral_gb
response.swap_gb = instance.swap_gb
response.auto_recovery = instance.auto_recovery
response.live_migration_timeout \
= instance.max_live_migrate_wait_in_secs
response.live_migration_max_downtime \
= instance.max_live_migration_downtime_in_ms
if instance.host_name is not None:
host_table = tables.tables_get_host_table()
host = host_table.get(instance.host_name, None)
if host is not None:
response.host_uuid = host.uuid
else:
response.result = rpc.RPC_MSG_RESULT.FAILED
else:
response.result = rpc.RPC_MSG_RESULT.FAILED
connection.send(response.serialize())
connection.close()
DLOG.info("Sent response=%s" % response)
del _instance_create_operations[instance_name]
def is_enabled(self):
"""
Returns true if this hypervisor is enabled
"""
from nfv_vim import tables
host_table = tables.tables_get_host_table()
host = host_table.get(self.host_name, None)
if host is not None:
if host.is_failed() and not host.is_component_failure():
return False
elif host.is_offline():
return False
return (nfvi.objects.v1.HYPERVISOR_OPER_STATE.ENABLED ==
self._nfvi_hypervisor.oper_state)
def _nfvi_host_state_change_callback(nfvi_host_uuid, nfvi_host_name,
nfvi_admin_state, nfvi_oper_state,
nfvi_avail_status, nfvi_data):
"""
NFVI Host state change callback
"""
DLOG.debug("Host state-change, nfvi_host_uuid=%s, nfvi_host_name=%s, "
"nfvi_host_admin_state=%s, nfvi_host_oper_state=%s, "
"nfvi_host_avail_status=%s." %
(nfvi_host_uuid, nfvi_host_name, nfvi_admin_state,
nfvi_oper_state, nfvi_avail_status))
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host_name, None)
if host is not None:
host.nfvi_host_state_change(nfvi_admin_state, nfvi_oper_state,
nfvi_avail_status, nfvi_data)
return True
def _nfvi_host_query_callback():
"""
NFVI Host query callback
"""
response = (yield)
DLOG.verbose("Query-Host callback, response=%s." % response)
if response['completed']:
nfvi_host = response['result-data']
host_table = tables.tables_get_host_table()
host = host_table.get(nfvi_host.name, None)
if host is None:
host = objects.Host(nfvi_host)
host_table[host.name] = host
host.nfvi_host_update(nfvi_host)
else:
DLOG.error("Query-Host callback, not completed, responses=%s." %
response)
def enable_host_services(self, host_names, service):
"""
Enable a host service on a list of hosts
"""
DLOG.info("Enable host services: %s service: %s" %
(host_names, service))
host_operation = Operation(OPERATION_TYPE.ENABLE_HOST_SERVICES)
if self._host_operation is not None:
DLOG.debug("Canceling previous host operation %s, before "
"continuing with host operation %s." %
(self._host_operation.operation_type,
host_operation.operation_type))
self._host_operation = None
host_table = tables.tables_get_host_table()
host_list = list()
for host_name in host_names:
host = host_table.get(host_name, None)
if host is None:
reason = "Unknown host %s given." % host_name
DLOG.info(reason)
host_operation.set_failed(reason)
return host_operation
host.host_services_locked = False
if (objects.HOST_SERVICE_STATE.ENABLED == host.host_service_state(
service)):
host_operation.add_host(host.name, OPERATION_STATE.COMPLETED)
else:
host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS)
host_list.append(host)
for host in host_list:
self._nfvi_enable_host_services(host.uuid, host.name,
host.personality, service)
if host_operation.is_inprogress():
self._host_operation = host_operation
return host_operation
def _nfvi_guest_services_query_callback(nfvi_host_uuid, nfvi_instance_uuid):
"""
NFVI Guest Services query callback
"""
DLOG.debug("Guest-Services query, nfvi_host_uuid=%s, "
"nfvi_instance_uuid=%s." % (nfvi_host_uuid, nfvi_instance_uuid))
# scope of instance
if nfvi_instance_uuid is not None:
instance_table = tables.tables_get_instance_table()
instance = instance_table.get(nfvi_instance_uuid, None)
if instance is None:
return False, None
result = dict()
result['uuid'] = instance.uuid
result['hostname'] = instance.host_name
result['services'] = instance.guest_services.get_nfvi_guest_services()
return True, result
# scope of host
host_table = tables.tables_get_host_table()
host = host_table.get_by_uuid(nfvi_host_uuid)
if host is None:
return False, None
instance_table = tables.tables_get_instance_table()
instances = list()
for instance in instance_table.on_host(host.name):
guest_services = instance.guest_services
if guest_services.are_provisioned():
result = dict()
result['uuid'] = instance.uuid
result['hostname'] = instance.host_name
result['services'] = guest_services.get_nfvi_guest_services()
instances.append(result)
results = dict()
results['instances'] = instances
return True, results
def vim_instance_api_get_instance(connection, msg):
"""
Handle Get-Instance API request
"""
DLOG.verbose("Get instance, filter_by_uuid=%s." % msg.filter_by_uuid)
instance_table = tables.tables_get_instance_table()
response = rpc.APIResponseGetInstance()
instance = instance_table.get(msg.filter_by_uuid, None)
if instance is not None:
response.uuid = instance.uuid
response.name = instance.name
response.admin_state = instance.admin_state
response.oper_state = instance.oper_state
response.avail_status = instance.avail_status
response.action = instance.action
response.host_name = instance.host_name
response.instance_type_original_name \
= instance.instance_type_original_name
response.image_uuid = instance.image_uuid
response.vcpus = instance.vcpus
response.memory_mb = instance.memory_mb
response.disk_gb = instance.disk_gb
response.ephemeral_gb = instance.ephemeral_gb
response.swap_gb = instance.swap_gb
response.auto_recovery = instance.auto_recovery
response.live_migration_timeout \
= instance.max_live_migrate_wait_in_secs
response.live_migration_max_downtime \
= instance.max_live_migration_downtime_in_ms
if instance.host_name is not None:
host_table = tables.tables_get_host_table()
host = host_table.get(instance.host_name, None)
if host is not None:
response.host_uuid = host.uuid
else:
response.result = rpc.RPC_MSG_RESULT.NOT_FOUND
connection.send(response.serialize())
DLOG.verbose("Sent response=%s" % response)
connection.close()
def _get_routers_on_agents():
"""
Get Routers hosted by an L3 Agent
Note paging is not supported by the l3-agent api.
"""
from nfv_vim import tables
global _L3Rebalance
# Agent of interest is first in the list.
# In the case of an agent going down, this will be important
agent_id, host_name = _L3Rebalance.get_current_l3agent()
host_table = tables.tables_get_host_table()
host = host_table.get(host_name, None)
if host is not None:
_L3Rebalance.update_current_l3agent('host_uuid', host.uuid)
else:
DLOG.error("Cannot find rebalance host: %s" % host_name)
_L3Rebalance.set_state(L3_REBALANCE_STATE.DONE)
nfvi.nfvi_get_agent_routers(agent_id, _get_agent_routers_callback(agent_id))
def _audit_nfvi_hosts_callback(timer_id):
"""
Audit Hosts
"""
global _main_audit_inprogress
response = (yield)
DLOG.verbose("Audit-Hosts callback, responses=%s." % response)
if response['completed']:
host_table = tables.tables_get_host_table()
deletable_host_groups = list(host_table)
for host_name in response['incomplete-hosts']:
if host_name in deletable_host_groups:
deletable_host_groups.remove(host_name)
DLOG.info("Not deleting host %s, incomplete information "
"returned." % host_name)
for nfvi_host in response['result-data']:
host = host_table.get(nfvi_host.name, None)
if host is None:
host = objects.Host(nfvi_host)
host_table[host.name] = host
else:
if not host.is_deleted():
deletable_host_groups.remove(host.name)
if 30 <= host.elapsed_time_in_state:
# Only process the audited host information if the host has
# been in the current state for at least 30 seconds. This is
# necessary because the host state information comes from
# maintenance and the maintenance states may lag the states in
# the VIM because the VIM and maintenance process some actions
# at the same time (e.g. when a host is locked).
host.nfvi_host_update(nfvi_host)
else:
DLOG.info("Ignoring audit reply for host %s" % nfvi_host.name)
for host_name in deletable_host_groups:
host = host_table[host_name]
host.nfvi_host_delete()
if host.is_deleted():
del host_table[host.name]
# Manage host groups
host_group_table = tables.tables_get_host_group_table()
deletable_host_groups = list(host_group_table)
for host_name in response['incomplete-hosts']:
host_group = next(
(x for x in host_group_table if host_name in x.member_names),
None)
if host_group is not None:
if host_group.name in deletable_host_groups:
deletable_host_groups.remove(host_group.name)
DLOG.info(
"Not deleting host group %s, incomplete information "
"returned for host %s." % (host_group.name, host_name))
for nfvi_host_group in response['host-groups']:
host_group = host_group_table.get(nfvi_host_group.name, None)
if host_group is None:
host_group = objects.HostGroup(nfvi_host_group)
host_group_table[host_group.name] = host_group
else:
deletable_host_groups.remove(host_group.name)
host_group.nfvi_host_group_update(nfvi_host_group)
for host_group_name in deletable_host_groups:
del host_group_table[host_group_name]
else:
DLOG.error("Audit-Hosts callback, not completed, responses=%s." %
response)
_main_audit_inprogress = False
timers.timers_reschedule_timer(timer_id, 2) # 2 seconds later
def _audit_nfvi_hosts_callback(timer_id):
"""
Audit Hosts
"""
global _main_audit_inprogress
response = (yield)
DLOG.verbose("Audit-Hosts callback, responses=%s." % response)
if response['completed']:
host_table = tables.tables_get_host_table()
deletable_host_groups = host_table.keys()
for host_name in response['incomplete-hosts']:
if host_name in deletable_host_groups:
deletable_host_groups.remove(host_name)
DLOG.info("Not deleting host %s, incomplete information "
"returned." % host_name)
for nfvi_host in response['result-data']:
host = host_table.get(nfvi_host.name, None)
if host is None:
host = objects.Host(nfvi_host)
host_table[host.name] = host
else:
if not host.is_deleted():
deletable_host_groups.remove(host.name)
host.nfvi_host_update(nfvi_host)
for host_name in deletable_host_groups:
host = host_table[host_name]
host.nfvi_host_delete()
if host.is_deleted():
del host_table[host.name]
# Manage host groups
host_group_table = tables.tables_get_host_group_table()
deletable_host_groups = host_group_table.keys()
for host_name in response['incomplete-hosts']:
host_group = next((x for x in host_group_table
if host_name in x.member_names), None)
if host_group is not None:
if host_group.name in deletable_host_groups:
deletable_host_groups.remove(host_group.name)
DLOG.info("Not deleting host group %s, incomplete information "
"returned for host %s." % (host_group.name,
host_name))
for nfvi_host_group in response['host-groups']:
host_group = host_group_table.get(nfvi_host_group.name, None)
if host_group is None:
host_group = objects.HostGroup(nfvi_host_group)
host_group_table[host_group.name] = host_group
else:
deletable_host_groups.remove(host_group.name)
host_group.nfvi_host_group_update(nfvi_host_group)
for host_group_name in deletable_host_groups:
del host_group_table[host_group_name]
else:
DLOG.error("Audit-Hosts callback, not completed, responses=%s."
% response)
_main_audit_inprogress = False
timers.timers_reschedule_timer(timer_id, 2) # 2 seconds later
def _system_state_query_callback():
"""
System state query callback
"""
global _alarm_data
global _minimum_hosts, _dor_stabilized, _dor_completed
global _dor_complete_percentage
global _system_state_get_inprogress, _system_state_gathered
response = (yield)
DLOG.info("System-State callback, response=%s." % response)
_system_state_get_inprogress = False
if response['completed']:
result_data = response.get('result-data', None)
if result_data is not None:
host_table = tables.tables_get_host_table()
total_hosts = 0
dor_complete_hosts = 0
for host_data in result_data:
host = host_table.get(host_data['hostname'], None)
if host is not None:
if objects.HOST_PERSONALITY.COMPUTE in host.personality:
DLOG.info("Host %s uptime is %s, host_uuid=%s." %
(host_data['hostname'], host_data['uptime'],
host_data['uuid']))
total_hosts += 1
if host_data['uptime'] >= _dor_complete_uptime:
dor_complete_hosts += 1
if 0 < total_hosts:
completion_percentage = (dor_complete_hosts * 100 /
total_hosts)
else:
completion_percentage = 0
if _dor_complete_percentage <= completion_percentage:
_dor_completed = True
DLOG.info("DOR host completion percentage is %s, threshold=%s." %
(completion_percentage, _dor_complete_percentage))
if not _dor_completed and total_hosts < _minimum_hosts:
DLOG.info("DOR total hosts %s is less than minimum hosts %s." %
(total_hosts, _minimum_hosts))
if process_uptime_in_secs() >= _dor_process_uptime:
_dor_completed = True
elif not _dor_stabilized:
_dor_stabilized = True
DLOG.info("DOR stabilized.")
if _dor_completed:
open(NFV_VIM_DOR_COMPLETE_FILE, 'w').close()
_dor_stabilized = True
if _alarm_data is not None:
alarm.clear_general_alarm(_alarm_data)
event_log.issue_general_log(
event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT)
_alarm_data = None
DLOG.info("DOR completed.")
_system_state_gathered = True
elif _alarm_data is None:
_alarm_data = \
alarm.raise_general_alarm(alarm.ALARM_TYPE.MULTI_NODE_RECOVERY_MODE)
event_log.issue_general_log(
event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_ENTER)
