def swact_hosts(self, host_names): """ Swact a list of hosts """ DLOG.info("Swact hosts: %s" % host_names) host_operation = Operation(OPERATION_TYPE.SWACT_HOSTS) if self._host_operation is not None: DLOG.debug("Canceling previous host operation %s, before " "continuing with host operation %s." % (self._host_operation.operation_type, host_operation.operation_type)) self._host_operation = None host_table = tables.tables_get_host_table() for host_name in host_names: host = host_table.get(host_name, None) if host is None: reason = "Unknown host %s given." % host_name DLOG.info(reason) host_operation.set_failed(reason) return host_operation host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS) self._nfvi_swact_host(host.uuid, host.name) if host_operation.is_inprogress(): self._host_operation = host_operation return host_operation
def _nfvi_host_get_callback(nfvi_host_uuid, nfvi_host_name): """ NFVI Host get callback """ DLOG.debug("Host get, nfvi_host_uuid=%s, nfvi_host_name=%s." % (nfvi_host_uuid, nfvi_host_name)) instances = 0 instances_failed = 0 instances_stopped = 0 host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host_name, None) if host is not None: instance_table = tables.tables_get_instance_table() for instance in instance_table.on_host(host.name): if instance.is_deleting() or instance.is_deleted(): continue if instance.is_failed(): instances_failed += 1 if instance.is_locked(): instances_stopped += 1 instances += 1 DLOG.info("Host %s has %s instances, failed=%s, stopped=%s." % (host.name, instances, instances_failed, instances_stopped)) return True, instances, instances_failed, instances_stopped
def _nfvi_host_action_callback(nfvi_host_uuid, nfvi_host_name, do_action): """ NFVI host action callback """ DLOG.debug("Host action, host_uuid=%s, host_name=%s, do_action=%s." % (nfvi_host_uuid, nfvi_host_name, do_action)) host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host_name, None) if host is not None: if nfvi.objects.v1.HOST_ACTION.UNLOCK == do_action: host.unlock() elif nfvi.objects.v1.HOST_ACTION.LOCK == do_action: host.lock() elif nfvi.objects.v1.HOST_ACTION.LOCK_FORCE == do_action: host.lock(force=True) elif nfvi.objects.v1.HOST_ACTION.DELETE == do_action: host.nfvi_host_delete() else: DLOG.info("Unknown action %s received for %s." % (do_action, nfvi_host_name)) return True
def _nfvi_disable_host_services_callback(self): """ NFVI Disable Host Services Callback """ from nfv_vim import directors response = (yield) DLOG.verbose("NFVI Disable Host Services callback response=%s." % response) if not response['completed']: DLOG.info( "Disable of host services on host %s failed, reason=%s." % (response['host_name'], response['reason'])) host_table = tables.tables_get_host_table() host = host_table.get(response['host_name'], None) if host is None: DLOG.verbose("Host %s does not exist." % response['host_name']) return if self._host_operation is None: DLOG.verbose("No host %s operation inprogress." % host.name) return if OPERATION_TYPE.DISABLE_HOST_SERVICES != \ self._host_operation.operation_type: DLOG.verbose("Unexpected host %s operation %s, ignoring." % (host.name, self._host_operation.operation_type)) return sw_mgmt_director = directors.get_sw_mgmt_director() sw_mgmt_director.disable_host_services_failed(host)
def _nfvi_lock_host_callback(self): """ NFVI Lock Host Callback """ from nfv_vim import directors response = (yield) DLOG.verbose("NFVI Lock Host callback response=%s." % response) if not response['completed']: DLOG.info("Lock of host %s failed, reason=%s." % (response['host_name'], response['reason'])) host_table = tables.tables_get_host_table() host = host_table.get(response['host_name'], None) if host is None: DLOG.verbose("Host %s does not exist." % response['host_name']) return if self._host_operation is None: DLOG.verbose("No host %s operation inprogress." % host.name) return if OPERATION_TYPE.LOCK_HOSTS != self._host_operation.operation_type: DLOG.verbose("Unexpected host %s operation %s, ignoring." % (host.name, self._host_operation.operation_type)) return sw_mgmt_director = directors.get_sw_mgmt_director() sw_mgmt_director.host_lock_failed(host)
def _nfvi_periodic_timer_event(): """ Periodic timer for hosts """ while True: timer_id = (yield) DLOG.verbose("NFVI periodic timer called, timer_id=%s." % timer_id) host_table = tables.tables_get_host_table() for host in host_table.values(): host.periodic_timer()
def _nfvi_host_upgrade_callback(nfvi_host_uuid, nfvi_host_name, upgrade_inprogress, recover_instances): """ NFVI Host upgrade callback """ DLOG.debug("Host upgrade, nfvi_host_uuid=%s, nfvi_host_name=%s, " "upgrade_inprogress=%s, recover_instances=%s." % (nfvi_host_uuid, nfvi_host_name, upgrade_inprogress, recover_instances)) host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host_name, None) if host is not None: host.nfvi_host_upgrade_status(upgrade_inprogress, recover_instances) return True
def _nfvi_host_add_callback(nfvi_host_uuid, nfvi_host_name): """ NFVI Host add callback """ DLOG.debug("Host add, nfvi_host=%s." % nfvi_host_name) host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host_name, None) if host is None: nfvi.nfvi_get_host(nfvi_host_uuid, nfvi_host_name, _nfvi_host_query_callback()) else: host.nfvi_host_add() return True
def oper_state(self): """ Returns the current operational state of the hypervisor """ from nfv_vim import tables host_table = tables.tables_get_host_table() host = host_table.get(self.host_name, None) if host is not None: if host.is_failed() and not host.is_component_failure(): return nfvi.objects.v1.HYPERVISOR_OPER_STATE.DISABLED elif host.is_offline(): return nfvi.objects.v1.HYPERVISOR_OPER_STATE.DISABLED return self._nfvi_hypervisor.oper_state # assume one-to-one mapping
def _nfvi_host_services_query_callback(nfvi_host_name): """ NFVI Host Services query callback """ DLOG.debug("Host-Services query, host_name=%s." % nfvi_host_name) host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host_name, None) if host is None: return False, None if host.nfvi_host_is_enabled(): host_oper_state = nfvi.objects.v1.HOST_OPER_STATE.ENABLED else: host_oper_state = nfvi.objects.v1.HOST_OPER_STATE.DISABLED return True, host_oper_state
def _create_instance_callback(success, instance_name, instance_uuid): """ Handle Create-Instance callback """ DLOG.verbose("Create instance callback, name=%s." % instance_name) connection = _instance_create_operations.get(instance_name, None) if connection is not None: instance_table = tables.tables_get_instance_table() response = rpc.APIResponseCreateInstance() if success: instance = instance_table.get(instance_uuid, None) if instance is not None: response.uuid = instance.uuid response.name = instance.name response.admin_state = instance.admin_state response.oper_state = instance.oper_state response.avail_status = instance.avail_status response.action = instance.action response.host_name = instance.host_name response.instance_type_original_name \ = instance.instance_type_original_name response.image_uuid = instance.image_uuid response.vcpus = instance.vcpus response.memory_mb = instance.memory_mb response.disk_gb = instance.disk_gb response.ephemeral_gb = instance.ephemeral_gb response.swap_gb = instance.swap_gb response.auto_recovery = instance.auto_recovery response.live_migration_timeout \ = instance.max_live_migrate_wait_in_secs response.live_migration_max_downtime \ = instance.max_live_migration_downtime_in_ms if instance.host_name is not None: host_table = tables.tables_get_host_table() host = host_table.get(instance.host_name, None) if host is not None: response.host_uuid = host.uuid else: response.result = rpc.RPC_MSG_RESULT.FAILED else: response.result = rpc.RPC_MSG_RESULT.FAILED connection.send(response.serialize()) connection.close() DLOG.info("Sent response=%s" % response) del _instance_create_operations[instance_name]
def is_enabled(self): """ Returns true if this hypervisor is enabled """ from nfv_vim import tables host_table = tables.tables_get_host_table() host = host_table.get(self.host_name, None) if host is not None: if host.is_failed() and not host.is_component_failure(): return False elif host.is_offline(): return False return (nfvi.objects.v1.HYPERVISOR_OPER_STATE.ENABLED == self._nfvi_hypervisor.oper_state)
def _nfvi_host_state_change_callback(nfvi_host_uuid, nfvi_host_name, nfvi_admin_state, nfvi_oper_state, nfvi_avail_status, nfvi_data): """ NFVI Host state change callback """ DLOG.debug("Host state-change, nfvi_host_uuid=%s, nfvi_host_name=%s, " "nfvi_host_admin_state=%s, nfvi_host_oper_state=%s, " "nfvi_host_avail_status=%s." % (nfvi_host_uuid, nfvi_host_name, nfvi_admin_state, nfvi_oper_state, nfvi_avail_status)) host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host_name, None) if host is not None: host.nfvi_host_state_change(nfvi_admin_state, nfvi_oper_state, nfvi_avail_status, nfvi_data) return True
def _nfvi_host_query_callback(): """ NFVI Host query callback """ response = (yield) DLOG.verbose("Query-Host callback, response=%s." % response) if response['completed']: nfvi_host = response['result-data'] host_table = tables.tables_get_host_table() host = host_table.get(nfvi_host.name, None) if host is None: host = objects.Host(nfvi_host) host_table[host.name] = host host.nfvi_host_update(nfvi_host) else: DLOG.error("Query-Host callback, not completed, responses=%s." % response)
def enable_host_services(self, host_names, service): """ Enable a host service on a list of hosts """ DLOG.info("Enable host services: %s service: %s" % (host_names, service)) host_operation = Operation(OPERATION_TYPE.ENABLE_HOST_SERVICES) if self._host_operation is not None: DLOG.debug("Canceling previous host operation %s, before " "continuing with host operation %s." % (self._host_operation.operation_type, host_operation.operation_type)) self._host_operation = None host_table = tables.tables_get_host_table() host_list = list() for host_name in host_names: host = host_table.get(host_name, None) if host is None: reason = "Unknown host %s given." % host_name DLOG.info(reason) host_operation.set_failed(reason) return host_operation host.host_services_locked = False if (objects.HOST_SERVICE_STATE.ENABLED == host.host_service_state( service)): host_operation.add_host(host.name, OPERATION_STATE.COMPLETED) else: host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS) host_list.append(host) for host in host_list: self._nfvi_enable_host_services(host.uuid, host.name, host.personality, service) if host_operation.is_inprogress(): self._host_operation = host_operation return host_operation
def _nfvi_guest_services_query_callback(nfvi_host_uuid, nfvi_instance_uuid): """ NFVI Guest Services query callback """ DLOG.debug("Guest-Services query, nfvi_host_uuid=%s, " "nfvi_instance_uuid=%s." % (nfvi_host_uuid, nfvi_instance_uuid)) # scope of instance if nfvi_instance_uuid is not None: instance_table = tables.tables_get_instance_table() instance = instance_table.get(nfvi_instance_uuid, None) if instance is None: return False, None result = dict() result['uuid'] = instance.uuid result['hostname'] = instance.host_name result['services'] = instance.guest_services.get_nfvi_guest_services() return True, result # scope of host host_table = tables.tables_get_host_table() host = host_table.get_by_uuid(nfvi_host_uuid) if host is None: return False, None instance_table = tables.tables_get_instance_table() instances = list() for instance in instance_table.on_host(host.name): guest_services = instance.guest_services if guest_services.are_provisioned(): result = dict() result['uuid'] = instance.uuid result['hostname'] = instance.host_name result['services'] = guest_services.get_nfvi_guest_services() instances.append(result) results = dict() results['instances'] = instances return True, results
def vim_instance_api_get_instance(connection, msg): """ Handle Get-Instance API request """ DLOG.verbose("Get instance, filter_by_uuid=%s." % msg.filter_by_uuid) instance_table = tables.tables_get_instance_table() response = rpc.APIResponseGetInstance() instance = instance_table.get(msg.filter_by_uuid, None) if instance is not None: response.uuid = instance.uuid response.name = instance.name response.admin_state = instance.admin_state response.oper_state = instance.oper_state response.avail_status = instance.avail_status response.action = instance.action response.host_name = instance.host_name response.instance_type_original_name \ = instance.instance_type_original_name response.image_uuid = instance.image_uuid response.vcpus = instance.vcpus response.memory_mb = instance.memory_mb response.disk_gb = instance.disk_gb response.ephemeral_gb = instance.ephemeral_gb response.swap_gb = instance.swap_gb response.auto_recovery = instance.auto_recovery response.live_migration_timeout \ = instance.max_live_migrate_wait_in_secs response.live_migration_max_downtime \ = instance.max_live_migration_downtime_in_ms if instance.host_name is not None: host_table = tables.tables_get_host_table() host = host_table.get(instance.host_name, None) if host is not None: response.host_uuid = host.uuid else: response.result = rpc.RPC_MSG_RESULT.NOT_FOUND connection.send(response.serialize()) DLOG.verbose("Sent response=%s" % response) connection.close()
def _get_routers_on_agents(): """ Get Routers hosted by an L3 Agent Note paging is not supported by the l3-agent api. """ from nfv_vim import tables global _L3Rebalance # Agent of interest is first in the list. # In the case of an agent going down, this will be important agent_id, host_name = _L3Rebalance.get_current_l3agent() host_table = tables.tables_get_host_table() host = host_table.get(host_name, None) if host is not None: _L3Rebalance.update_current_l3agent('host_uuid', host.uuid) else: DLOG.error("Cannot find rebalance host: %s" % host_name) _L3Rebalance.set_state(L3_REBALANCE_STATE.DONE) nfvi.nfvi_get_agent_routers(agent_id, _get_agent_routers_callback(agent_id))
def _audit_nfvi_hosts_callback(timer_id): """ Audit Hosts """ global _main_audit_inprogress response = (yield) DLOG.verbose("Audit-Hosts callback, responses=%s." % response) if response['completed']: host_table = tables.tables_get_host_table() deletable_host_groups = list(host_table) for host_name in response['incomplete-hosts']: if host_name in deletable_host_groups: deletable_host_groups.remove(host_name) DLOG.info("Not deleting host %s, incomplete information " "returned." % host_name) for nfvi_host in response['result-data']: host = host_table.get(nfvi_host.name, None) if host is None: host = objects.Host(nfvi_host) host_table[host.name] = host else: if not host.is_deleted(): deletable_host_groups.remove(host.name) if 30 <= host.elapsed_time_in_state: # Only process the audited host information if the host has # been in the current state for at least 30 seconds. This is # necessary because the host state information comes from # maintenance and the maintenance states may lag the states in # the VIM because the VIM and maintenance process some actions # at the same time (e.g. when a host is locked). host.nfvi_host_update(nfvi_host) else: DLOG.info("Ignoring audit reply for host %s" % nfvi_host.name) for host_name in deletable_host_groups: host = host_table[host_name] host.nfvi_host_delete() if host.is_deleted(): del host_table[host.name] # Manage host groups host_group_table = tables.tables_get_host_group_table() deletable_host_groups = list(host_group_table) for host_name in response['incomplete-hosts']: host_group = next( (x for x in host_group_table if host_name in x.member_names), None) if host_group is not None: if host_group.name in deletable_host_groups: deletable_host_groups.remove(host_group.name) DLOG.info( "Not deleting host group %s, incomplete information " "returned for host %s." % (host_group.name, host_name)) for nfvi_host_group in response['host-groups']: host_group = host_group_table.get(nfvi_host_group.name, None) if host_group is None: host_group = objects.HostGroup(nfvi_host_group) host_group_table[host_group.name] = host_group else: deletable_host_groups.remove(host_group.name) host_group.nfvi_host_group_update(nfvi_host_group) for host_group_name in deletable_host_groups: del host_group_table[host_group_name] else: DLOG.error("Audit-Hosts callback, not completed, responses=%s." % response) _main_audit_inprogress = False timers.timers_reschedule_timer(timer_id, 2) # 2 seconds later
def _audit_nfvi_hosts_callback(timer_id): """ Audit Hosts """ global _main_audit_inprogress response = (yield) DLOG.verbose("Audit-Hosts callback, responses=%s." % response) if response['completed']: host_table = tables.tables_get_host_table() deletable_host_groups = host_table.keys() for host_name in response['incomplete-hosts']: if host_name in deletable_host_groups: deletable_host_groups.remove(host_name) DLOG.info("Not deleting host %s, incomplete information " "returned." % host_name) for nfvi_host in response['result-data']: host = host_table.get(nfvi_host.name, None) if host is None: host = objects.Host(nfvi_host) host_table[host.name] = host else: if not host.is_deleted(): deletable_host_groups.remove(host.name) host.nfvi_host_update(nfvi_host) for host_name in deletable_host_groups: host = host_table[host_name] host.nfvi_host_delete() if host.is_deleted(): del host_table[host.name] # Manage host groups host_group_table = tables.tables_get_host_group_table() deletable_host_groups = host_group_table.keys() for host_name in response['incomplete-hosts']: host_group = next((x for x in host_group_table if host_name in x.member_names), None) if host_group is not None: if host_group.name in deletable_host_groups: deletable_host_groups.remove(host_group.name) DLOG.info("Not deleting host group %s, incomplete information " "returned for host %s." % (host_group.name, host_name)) for nfvi_host_group in response['host-groups']: host_group = host_group_table.get(nfvi_host_group.name, None) if host_group is None: host_group = objects.HostGroup(nfvi_host_group) host_group_table[host_group.name] = host_group else: deletable_host_groups.remove(host_group.name) host_group.nfvi_host_group_update(nfvi_host_group) for host_group_name in deletable_host_groups: del host_group_table[host_group_name] else: DLOG.error("Audit-Hosts callback, not completed, responses=%s." % response) _main_audit_inprogress = False timers.timers_reschedule_timer(timer_id, 2) # 2 seconds later
def _system_state_query_callback(): """ System state query callback """ global _alarm_data global _minimum_hosts, _dor_stabilized, _dor_completed global _dor_complete_percentage global _system_state_get_inprogress, _system_state_gathered response = (yield) DLOG.info("System-State callback, response=%s." % response) _system_state_get_inprogress = False if response['completed']: result_data = response.get('result-data', None) if result_data is not None: host_table = tables.tables_get_host_table() total_hosts = 0 dor_complete_hosts = 0 for host_data in result_data: host = host_table.get(host_data['hostname'], None) if host is not None: if objects.HOST_PERSONALITY.COMPUTE in host.personality: DLOG.info("Host %s uptime is %s, host_uuid=%s." % (host_data['hostname'], host_data['uptime'], host_data['uuid'])) total_hosts += 1 if host_data['uptime'] >= _dor_complete_uptime: dor_complete_hosts += 1 if 0 < total_hosts: completion_percentage = (dor_complete_hosts * 100 / total_hosts) else: completion_percentage = 0 if _dor_complete_percentage <= completion_percentage: _dor_completed = True DLOG.info("DOR host completion percentage is %s, threshold=%s." % (completion_percentage, _dor_complete_percentage)) if not _dor_completed and total_hosts < _minimum_hosts: DLOG.info("DOR total hosts %s is less than minimum hosts %s." % (total_hosts, _minimum_hosts)) if process_uptime_in_secs() >= _dor_process_uptime: _dor_completed = True elif not _dor_stabilized: _dor_stabilized = True DLOG.info("DOR stabilized.") if _dor_completed: open(NFV_VIM_DOR_COMPLETE_FILE, 'w').close() _dor_stabilized = True if _alarm_data is not None: alarm.clear_general_alarm(_alarm_data) event_log.issue_general_log( event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_EXIT) _alarm_data = None DLOG.info("DOR completed.") _system_state_gathered = True elif _alarm_data is None: _alarm_data = \ alarm.raise_general_alarm(alarm.ALARM_TYPE.MULTI_NODE_RECOVERY_MODE) event_log.issue_general_log( event_log.EVENT_ID.MULTI_NODE_RECOVERY_MODE_ENTER)