def inspect_vnic_rates(self, instance, duration=None): vm_moid = self._ops.get_vm_moid(instance.id) if not vm_moid: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in VMware Vsphere') % instance.id) vnic_stats = {} vnic_ids = set() for net_counter in (VC_NETWORK_RX_COUNTER, VC_NETWORK_TX_COUNTER): net_counter_id = self._ops.get_perf_counter_id(net_counter) vnic_id_to_stats_map = self._ops.query_vm_device_stats( vm_moid, net_counter_id, duration) vnic_stats[net_counter] = vnic_id_to_stats_map vnic_ids.update(vnic_id_to_stats_map.iterkeys()) # Stats provided from vSphere are in KB/s, converting it to B/s. for vnic_id in vnic_ids: rx_bytes_rate = ( vnic_stats[VC_NETWORK_RX_COUNTER].get(vnic_id, 0) * units.Ki) tx_bytes_rate = ( vnic_stats[VC_NETWORK_TX_COUNTER].get(vnic_id, 0) * units.Ki) stats = virt_inspector.InterfaceRateStats(rx_bytes_rate, tx_bytes_rate) interface = virt_inspector.Interface(name=vnic_id, mac=None, fref=None, parameters=None) yield (interface, stats)
def inspect_disk_rates(self, instance, duration=None): vm_moid = self._ops.get_vm_moid(instance.id) if not vm_moid: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in VMware Vsphere') % instance.id) disk_stats = {} disk_ids = set() disk_counters = [ VC_DISK_READ_RATE_CNTR, VC_DISK_READ_REQUESTS_RATE_CNTR, VC_DISK_WRITE_RATE_CNTR, VC_DISK_WRITE_REQUESTS_RATE_CNTR ] for disk_counter in disk_counters: disk_counter_id = self._ops.get_perf_counter_id(disk_counter) disk_id_to_stat_map = self._ops.query_vm_device_stats( vm_moid, disk_counter_id, duration) disk_stats[disk_counter] = disk_id_to_stat_map disk_ids.update(disk_id_to_stat_map.iterkeys()) for disk_id in disk_ids: def stat_val(counter_name): return disk_stats[counter_name].get(disk_id, 0) disk = virt_inspector.Disk(device=disk_id) # Stats provided from vSphere are in KB/s, converting it to B/s. disk_rate_info = virt_inspector.DiskRateStats( read_bytes_rate=stat_val(VC_DISK_READ_RATE_CNTR) * units.Ki, read_requests_rate=stat_val(VC_DISK_READ_REQUESTS_RATE_CNTR), write_bytes_rate=stat_val(VC_DISK_WRITE_RATE_CNTR) * units.Ki, write_requests_rate=stat_val(VC_DISK_WRITE_REQUESTS_RATE_CNTR)) yield (disk, disk_rate_info)
def _lookup_by_name(self, instance_name): try: return self._get_connection().lookupByName(instance_name) except Exception as ex: error_code = ex.get_error_code() if libvirt else 'unknown' msg = ("Error from libvirt while looking up %(instance_name)s: " "[Error Code %(error_code)s] %(ex)s" % locals()) raise virt_inspector.InstanceNotFoundException(msg)
def inspect_vnics(self, instance, duration): """Inspect the vNIC statistics for an instance. :param instance: the target instance :param duration: the last 'n' seconds, over which the value should be inspected. The PowerVM implementation does not make use of the duration field. :return: for each vNIC, the number of bytes & packets received and transmitted """ # Get the current and previous sample. Delta is performed between # these two. uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) # If the cur_metric is none, then the instance can not be found in the # sample and an error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) # If there isn't network information, this is because the Virtual # I/O Metrics were turned off. Have to pass through this method. if cur_metric.network is None: return # Get the network interfaces. A 'cna' is a Client VM's Network Adapter client_cnas = self._get_cnas(uuid) for metric_cna in cur_metric.network.cnas: # Get the mac, but if it isn't found, then move to the next. Might # have been removed since the last sample. mac = self.mac_for_metric_cna(metric_cna, client_cnas) if mac is None: continue # The name will be the location code. MAC is identified from # above. Others appear libvirt specific. # # PowerVM doesn't specify drops by receive vs. transmit. Since we # have the client adapter, we assume all are receive drops. # There are no error metrics available. yield virt_inspector.InterfaceStats( name=metric_cna.physical_location, mac=mac, fref=None, parameters=None, rx_bytes=metric_cna.received_bytes, rx_packets=metric_cna.received_packets, rx_drop=metric_cna.dropped_packets, rx_errors=0, tx_bytes=metric_cna.sent_bytes, tx_packets=metric_cna.sent_packets, tx_drop=0, tx_errors=0)
def _lookup_vm(self, vm_name): vms = self._conn.Msvm_ComputerSystem(ElementName=vm_name) n = len(vms) if n == 0: raise inspector.InstanceNotFoundException( _('VM %s not found on Hyper-V') % vm_name) elif n > 1: raise HyperVException(_('Duplicate VM name found: %s') % vm_name) else: return vms[0]
def _lookup_by_name(self, instance_name): vm_refs = self.session.VM.get_by_name_label(instance_name) n = len(vm_refs) if n == 0: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in XenServer') % instance_name) elif n > 1: raise XenapiException( _('Multiple VM %s found in XenServer') % instance_name) else: return vm_refs[0]
def inspect_memory_usage(self, instance, duration=None): vm_moid = self._ops.get_vm_moid(instance.id) if vm_moid is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in VMware Vsphere') % instance.id) mem_counter_id = self._ops.get_perf_counter_id( VC_AVERAGE_MEMORY_CONSUMED_CNTR) memory = self._ops.query_vm_aggregate_stats(vm_moid, mem_counter_id, duration) # Stat provided from vSphere is in KB, converting it to MB. memory = memory / units.Ki return virt_inspector.MemoryUsageStats(usage=memory)
def _lookup_by_name(self, instance_name): try: return self._get_connection().lookupByName(instance_name) except Exception as ex: if not libvirt or not isinstance(ex, libvirt.libvirtError): raise virt_inspector.InspectorException(unicode(ex)) error_code = ex.get_error_code() msg = ("Error from libvirt while looking up %(instance_name)s: " "[Error Code %(error_code)s] " "%(ex)s" % {'instance_name': instance_name, 'error_code': error_code, 'ex': ex}) raise virt_inspector.InstanceNotFoundException(msg)
def inspect_vnics(self, instance): """Inspect the vNIC statistics for an instance. :param instance: the target instance :return: for each vNIC, the number of bytes & packets received and transmitted """ # Get the current and previous sample. Delta is performed between # these two. uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) # If the cur_metric is none, then the instance can not be found in the # sample and an error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) # If there isn't network information, this is because the Virtual # I/O Metrics were turned off. Have to pass through this method. if cur_metric.network is None: return # Get the network interfaces. A 'cna' is a Client VM's Network Adapter client_cnas = self._get_cnas(uuid) for metric_cna in cur_metric.network.cnas: # Get the mac, but if it isn't found, then move to the next. Might # have been removed since the last sample. mac = self.mac_for_metric_cna(metric_cna, client_cnas) if mac is None: continue # The name will be the location code. MAC is identified from # above. Others appear libvirt specific. interface = virt_inspector.Interface( name=metric_cna.physical_location, mac=mac, fref=None, parameters=None) stats = virt_inspector.InterfaceStats( rx_bytes=metric_cna.received_bytes, rx_packets=metric_cna.received_packets, tx_bytes=metric_cna.sent_bytes, tx_packets=metric_cna.sent_packets) # Yield the stats up to the invoker yield (interface, stats)
def _get_vm_mobj_not_power_off_or_raise(self, instance): vm_mobj = self._ops.get_vm_mobj(instance.id) if vm_mobj is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in VMware vSphere') % instance.id) vm_powerState = self._ops.query_vm_property(vm_mobj, 'runtime.powerState') if vm_powerState == "poweredOff": raise virt_inspector.InstanceShutOffException( _('VM %s is poweroff in VMware vSphere') % instance.id) return vm_mobj
def inspect_disks(self, instance, duration): """Inspect the disk statistics for an instance. The response is a generator of the values. :param instance: the target instance :param duration: the last 'n' seconds, over which the value should be inspected. The PowerVM implementation does not make use of the duration field. :return disk: The Disk indicating the device for the storage device. :return stats: The DiskStats indicating the read/write data to the device. """ # Get the current and previous sample. Delta is performed between # these two. uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) # If the cur_metric is none, then the instance can not be found in the # sample and an error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) # If there isn't storage information, this is because the Virtual # I/O Metrics were turned off. Have to pass through this method. if cur_metric.storage is None: LOG.debug("Current storage metric was unavailable from the API " "instance %s." % instance.name) return # Bundle together the SCSI and virtual FC adapters adpts = cur_metric.storage.virt_adpts + cur_metric.storage.vfc_adpts # Loop through all the storage adapters for adpt in adpts: # PowerVM only shows the connection (SCSI or FC). Name after # the connection name yield virt_inspector.DiskStats(device=adpt.name, read_requests=adpt.num_reads, read_bytes=adpt.read_bytes, write_requests=adpt.num_writes, write_bytes=adpt.write_bytes, errors=0, wr_total_times=0, rd_total_times=0)
def inspect_cpu_util(self, instance, duration=None): vm_moid = self._ops.get_vm_moid(instance.id) if vm_moid is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in VMware Vsphere') % instance.id) cpu_util_counter_id = self._ops.get_perf_counter_id( VC_AVERAGE_CPU_CONSUMED_CNTR) cpu_util = self._ops.query_vm_aggregate_stats( vm_moid, cpu_util_counter_id, duration) # For this counter vSphere returns values scaled-up by 100, since the # corresponding API can't return decimals, but only longs. # For e.g. if the utilization is 12.34%, the value returned is 1234. # Hence, dividing by 100. cpu_util = cpu_util / 100 return virt_inspector.CPUUtilStats(util=cpu_util)
def _lookup_by_uuid(self, instance): instance_name = util.instance_name(instance) try: return self.connection.lookupByUUIDString(instance.id) except libvirt.libvirtError as ex: if libvirt_utils.is_disconnection_exception(ex): raise msg = _("Error from libvirt while looking up instance " "<name=%(name)s, id=%(id)s>: " "[Error Code %(error_code)s] " "%(ex)s") % { 'name': instance_name, 'id': instance.id, 'error_code': ex.get_error_code(), 'ex': ex } raise virt_inspector.InstanceNotFoundException(msg) except Exception as ex: raise virt_inspector.InspectorException(six.text_type(ex))
def _lookup_by_name(self, instance_name): try: return self._get_connection().lookupByName(instance_name) except Exception as ex: if not libvirt or not isinstance(ex, libvirt.libvirtError): raise virt_inspector.InspectorException(six.text_type(ex)) error_code = ex.get_error_code() if (error_code == libvirt.VIR_ERR_SYSTEM_ERROR and ex.get_error_domain() in (libvirt.VIR_FROM_REMOTE, libvirt.VIR_FROM_RPC)): raise msg = ("Error from libvirt while looking up %(instance_name)s: " "[Error Code %(error_code)s] " "%(ex)s" % { 'instance_name': instance_name, 'error_code': error_code, 'ex': ex }) raise virt_inspector.InstanceNotFoundException(msg)
def inspect_cpus(self, instance): """Inspect the CPU statistics for an instance. :param instance: the target instance :return: the number of CPUs and cumulative CPU time """ uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) # If the current metric is none, then the instance can not be found in # the sample set. An error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) cpu_time = (cur_metric.processor.util_cap_proc_cycles + cur_metric.processor.util_uncap_proc_cycles) return virt_inspector.CPUStats(number=cur_metric.processor.virt_procs, time=cpu_time)
def _lookup_by_uuid(self, instance): instance_name = util.instance_name(instance) try: return self._get_connection().lookupByUUIDString(instance.id) except Exception as ex: if not libvirt or not isinstance(ex, libvirt.libvirtError): raise virt_inspector.InspectorException(six.text_type(ex)) error_code = ex.get_error_code() if (error_code == libvirt.VIR_ERR_SYSTEM_ERROR and ex.get_error_domain() in (libvirt.VIR_FROM_REMOTE, libvirt.VIR_FROM_RPC)): raise msg = _("Error from libvirt while looking up instance " "<name=%(name)s, id=%(id)s>: " "[Error Code %(error_code)s] " "%(ex)s") % { 'name': instance_name, 'id': instance.id, 'error_code': error_code, 'ex': ex } raise virt_inspector.InstanceNotFoundException(msg)
def _get_inst_stat(self, meter, instance): inst_name = zvmutils.get_inst_name(instance) # zvm inspector can not get instance info in shutdown stat if zvmutils.get_inst_power_state(instance) == 0x04: msg = _("Can not get vm info in shutdown state " "for %s") % inst_name raise virt_inspector.InstanceShutOffException(msg) self._check_expiration_and_update_cache(meter) inst_stat = self.cache.get(meter, inst_name) if inst_stat is None: userid = (self.instances.get(inst_name) or zvmutils.get_userid(inst_name)) self._update_cache(meter, {inst_name: userid}) inst_stat = self.cache.get(meter, inst_name) if inst_stat is None: msg = _("Can not get vm info for %s") % inst_name raise virt_inspector.InstanceNotFoundException(msg) else: return inst_stat
def inspect_vnic_rates(self, instance, duration): """Inspect the vNIC rate statistics for an instance. :param instance: the target instance :param duration: the last 'n' seconds, over which the value should be inspected The PowerVM implementation does not make use of the duration field. :return: for each vNIC, the rate of bytes & packets received and transmitted """ # Get the current and previous sample. Delta is performed between # these two. uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) prev_date, prev_metric = self.vm_metrics.get_previous_metric(uuid) # If the current is none, then the instance can not be found in the # sample and an error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) # If there isn't network information, this is because the Virtual # I/O Metrics were turned off. Have to pass through this method. if (cur_metric.network is None or prev_metric is None or prev_metric.network is None): return # Get the network interfaces. A 'cna' is a Client VM's Network Adapter client_cnas = self._get_cnas(uuid) def find_prev_net(metric_cna): """Finds the metric vNIC from the previous sample's vNICs.""" # If no previous, return None if prev_metric is None or prev_metric.network is None: return None for prev_cna in prev_metric.network.cnas: if prev_cna.physical_location == metric_cna.physical_location: return prev_cna # Couldn't find a previous. Maybe the interface was recently # added to the instance? Return None return None # Need to determine the time delta between the samples. This is # usually 30 seconds from the API, but the metrics will be specific. date_delta_num = float((cur_date - prev_date).seconds) for metric_cna in cur_metric.network.cnas: # Get the mac, but if it isn't found, then move to the next. Might # have been removed since the last sample. mac = self.mac_for_metric_cna(metric_cna, client_cnas) if mac is None: continue # Note that here, the previous may be none. That simply indicates # that the adapter was dynamically added to the VM before the # previous collection. Not the migration scenario above. # In this case, we can default the base to 0. prev = find_prev_net(metric_cna) rx_bytes_diff = (metric_cna.received_bytes - (0 if prev is None else prev.received_bytes)) tx_bytes_diff = (metric_cna.sent_bytes - (0 if prev is None else prev.sent_bytes)) # Stats are the difference in the bytes, divided by the difference # in time between the two samples. rx_rate = float(rx_bytes_diff) / float(date_delta_num) tx_rate = float(tx_bytes_diff) / float(date_delta_num) # The name will be the location code. MAC is identified from # above. Others appear libvirt specific. yield virt_inspector.InterfaceRateStats( name=metric_cna.physical_location, mac=mac, fref=None, parameters=None, rx_bytes_rate=rx_rate, tx_bytes_rate=tx_rate)
def inspect_disk_iops(self, instance, duration): """Inspect the Disk Input/Output operations per second for an instance. The response is a generator of the values. :param instance: the target instance :param duration: the last 'n' seconds, over which the value should be inspected. The PowerVM implementation does not make use of the duration field. :return disk: The Disk indicating the device for the storage device. :return stats: The DiskIOPSStats indicating the I/O operations per second for the device. """ # Get the current and previous sample. Delta is performed between # these two. uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) prev_date, prev_metric = self.vm_metrics.get_previous_metric(uuid) # If the cur_metric is none, then the instance can not be found in the # sample and an error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) # If there isn't storage information, this may be because the Virtual # I/O Metrics were turned off. If the previous metric is unavailable, # also have to pass through this method. if (cur_metric.storage is None or prev_metric is None or prev_metric.storage is None): LOG.debug("Current storage metric was unavailable from the API " "instance %s." % instance.name) return # Need to determine the time delta between the samples. This is # usually 30 seconds from the API, but the metrics will be specific. # However, if there is no previous sample, then we have to estimate. # Therefore, we estimate 15 seconds - half of the standard 30 seconds. date_delta = ((cur_date - prev_date) if prev_date is not None else datetime.timedelta(seconds=15)) # Bundle together the SCSI and virtual FC adapters cur_adpts = (cur_metric.storage.virt_adpts + cur_metric.storage.vfc_adpts) prev_adpts = (prev_metric.storage.virt_adpts + prev_metric.storage.vfc_adpts) def find_prev(cur_adpt): for prev_adpt in prev_adpts: if prev_adpt.name == cur_adpt.name: return prev_adpt return None # Loop through all the storage adapters for cur_adpt in cur_adpts: # IOPs is the read/write counts of the current - prev divided by # second difference between the two, rounded to the integer. :-) cur_ops = cur_adpt.num_reads + cur_adpt.num_writes # The previous adapter may be None. This simply indicates that the # adapter was added between the previous sample and this one. It # does not indicate a live migrate scenario like noted above, as # the VM itself hasn't moved. prev_adpt = find_prev(cur_adpt) prev_ops = ((prev_adpt.num_reads + prev_adpt.num_writes) if prev_adpt else 0) iops = (cur_ops - prev_ops) // date_delta.seconds # PowerVM only shows the connection (SCSI or FC). Name after # the connection name yield virt_inspector.DiskIOPSStats(device=cur_adpt.name, iops_count=iops)
def inspect_instance(self, instance, duration): """Inspect the statistics for an instance. :param instance: the target instance :param duration: the last 'n' seconds, over which the value should be inspected. The PowerVM implementation does not make use of the duration field. :return: the instance statistics """ uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) # If the current metric is none, then the instance can not be found in # the sample set. An error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) cpu_time = (cur_metric.processor.util_cap_proc_cycles + cur_metric.processor.util_uncap_proc_cycles) cpu_num = cur_metric.processor.virt_procs # The duration is ignored. There is precedent for this in other # inspectors if the platform doesn't support duration. # # Given the nature of PowerVM to collect samples over coarse periods # of time, it does not lend well to duration based collection. # Therefore this works by gathering the latest utilization from the # samples and ignores the duration. # Get the current and previous sample. Delta is performed between # these two. prev_date, prev_metric = self.vm_metrics.get_previous_metric(uuid) # Get the current data. cur_util_cap = cur_metric.processor.util_cap_proc_cycles cur_util_uncap = cur_metric.processor.util_uncap_proc_cycles cur_idle = cur_metric.processor.idle_proc_cycles cur_donated = cur_metric.processor.donated_proc_cycles cur_entitled = cur_metric.processor.entitled_proc_cycles # Get the previous sample data if prev_metric is None: # If there is no previous sample, that is either a new VM or is # a live migrated system. A live migrated system will pull all # of its metrics with it. The issue with this is it could have # CPU cycles for months of run time. So we can't really determine # the CPU utilization within the last X seconds...because to THIS # host it's new (only in the cur_metric). So we error out, the # inspector will use a debug message in the log. LOG.warning( 'Unable to derive CPU Utilization for VM %s. It is ' 'either a new VM or was recently migrated. It will be ' 'collected in the next inspection cycle.', instance.name) return virt_inspector.InstanceStats(cpu_time=cpu_time, cpu_number=cpu_num) # Gather the previous metrics prev_util_cap = prev_metric.processor.util_cap_proc_cycles prev_util_uncap = prev_metric.processor.util_uncap_proc_cycles prev_idle = prev_metric.processor.idle_proc_cycles prev_donated = prev_metric.processor.donated_proc_cycles prev_entitled = prev_metric.processor.entitled_proc_cycles # Utilization can be driven by multiple factors on PowerVM. # PowerVM has 'entitled' cycles. These are cycles that, if the VM # needs them, they get them no matter what. # # In terms of how those cycles are returned from the API: # util_cap_proc_cycles - How many cycles from the guaranteed # capacity were used. # util_uncap_proc_cycles - How many cycles were used that were # taken from spare (which is either unused processors cycles # or donated cycles from other VMs). # idle_proc_cycles - How many cycles (as reported by the OS to the # hypervisor) were reported as idle. # donated_proc_cycles - Cycles that were not needed by this VM that # were given to another VM in need of cycles. # # # So the final utilization equation is: # (util cap + util uncap - idle - donated) / entitled # # It is important to note that idle and donated proc cycles are # included in the 'util_cap_proc_cycles'. That is why they are # subtracted out. # # The interesting aspect of this is that the CPU Utilization can go # dramatically above 100% if there are free processors or if the # other workloads are in a lull. util_cap = cur_util_cap - prev_util_cap util_uncap = cur_util_uncap - prev_util_uncap idle = cur_idle - prev_idle donated = cur_donated - prev_donated entitled = cur_entitled - prev_entitled # If the entitled is zero, that generally means that the VM has not # been started yet (everything else would be zero as well). So to # avoid a divide by zero error, just return 0% in that case. util = (float(util_cap + util_uncap - idle - donated) / float(entitled) if entitled else 0.0) # Utilization is reported as percents. Therefore, multiply by 100.0 # to get a readable percentage based format. return virt_inspector.InstanceStats(cpu_util=util * 100.0, cpu_time=cpu_time, cpu_number=cpu_num)