def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(memory_resident=1.0), virt_inspector.InstanceStats(memory_resident=2.0), virt_inspector.InstanceStats(), virt_inspector.InstanceShutOffException(), ) mgr = manager.AgentManager(0, self.CONF) pollster = instance_stats.MemoryResidentPollster(self.CONF) @mock.patch('ceilometer.compute.pollsters.LOG') def _verify_resident_memory_metering(expected_count, expected_resident_memory_mb, expected_warnings, mylog): samples = list(pollster.get_samples(mgr, {}, [self.instance])) self.assertEqual(expected_count, len(samples)) if expected_count > 0: self.assertEqual(set(['memory.resident']), set([s.name for s in samples])) self.assertEqual(expected_resident_memory_mb, samples[0].volume) else: self.assertEqual(expected_warnings, mylog.warning.call_count) self.assertEqual(0, mylog.exception.call_count) _verify_resident_memory_metering(1, 1.0, 0) _verify_resident_memory_metering(1, 2.0, 0) _verify_resident_memory_metering(0, 0, 1) _verify_resident_memory_metering(0, 0, 0)
def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(memory_swap_in=1.0, memory_swap_out=2.0), virt_inspector.InstanceStats(memory_swap_in=3.0, memory_swap_out=4.0), ) mgr = manager.AgentManager(0, self.CONF) def _check_memory_swap_in(expected_swap_in): pollster = instance_stats.MemorySwapInPollster(self.CONF) samples = list(pollster.get_samples(mgr, {}, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['memory.swap.in']), set([s.name for s in samples])) self.assertEqual(expected_swap_in, samples[0].volume) def _check_memory_swap_out(expected_swap_out): pollster = instance_stats.MemorySwapOutPollster(self.CONF) samples = list(pollster.get_samples(mgr, {}, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['memory.swap.out']), set([s.name for s in samples])) self.assertEqual(expected_swap_out, samples[0].volume) _check_memory_swap_in(1.0) _check_memory_swap_out(4.0)
def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(memory_bandwidth_total=1892352, memory_bandwidth_local=1802240), virt_inspector.InstanceStats(memory_bandwidth_total=1081344, memory_bandwidth_local=90112), ) mgr = manager.AgentManager(0, self.CONF) def _check_memory_bandwidth_total(expected_usage): pollster = instance_stats.MemoryBandwidthTotalPollster(self.CONF) samples = list(pollster.get_samples(mgr, {}, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['memory.bandwidth.total']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) def _check_memory_bandwidth_local(expected_usage): pollster = instance_stats.MemoryBandwidthLocalPollster(self.CONF) samples = list(pollster.get_samples(mgr, {}, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['memory.bandwidth.local']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) _check_memory_bandwidth_total(1892352) _check_memory_bandwidth_local(90112)
def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(cpu_time=1 * (10**6), cpu_number=2, vcpu_number=2), virt_inspector.InstanceStats(cpu_time=3 * (10**6), cpu_number=2, vcpu_number=2), # cpu_time resets on instance restart virt_inspector.InstanceStats(cpu_time=2 * (10**6), cpu_number=2, vcpu_number=2), ) mgr = manager.AgentManager(0, self.CONF) pollster = instance_stats.CPUPollster(self.CONF) def _verify_cpu_metering(expected_time): cache = {} samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['cpu']), set([s.name for s in samples])) self.assertEqual(expected_time, samples[0].volume) self.assertEqual(2, samples[0].resource_metadata.get('cpu_number')) self.assertEqual(2, samples[0].resource_metadata.get('vcpu_number')) # ensure elapsed time between polling cycles is non-zero time.sleep(0.001) _verify_cpu_metering(1 * (10**6)) _verify_cpu_metering(3 * (10**6)) _verify_cpu_metering(2 * (10**6))
def inspect_instance(self, instance, duration=None): domain = self._get_domain_not_shut_off_or_raise(instance) memory_used = memory_resident = None memory_stats = domain.memoryStats() # Stat provided from libvirt is in KB, converting it to MB. if 'available' in memory_stats and 'unused' in memory_stats: memory_used = (memory_stats['available'] - memory_stats['unused']) / units.Ki if 'rss' in memory_stats: memory_resident = memory_stats['rss'] / units.Ki # TODO(sileht): stats also have the disk/vnic info # we could use that instead of the old method for Queen stats = self.connection.domainListGetStats([domain], 0)[0][1] return virt_inspector.InstanceStats( cpu_number=stats.get('vcpu.current'), cpu_time=stats.get('cpu.time'), memory_usage=memory_used, memory_resident=memory_resident, cpu_cycles=stats.get("perf.cpu_cycles"), instructions=stats.get("perf.instructions"), cache_references=stats.get("perf.cache_references"), cache_misses=stats.get("perf.cache_misses"), memory_bandwidth_total=stats.get("perf.mbmt"), memory_bandwidth_local=stats.get("perf.mbml"), cpu_l3_cache_usage=stats.get("perf.cmt"), )
def inspect_instance(self, instance, duration): instance_name = util.instance_name(instance) vm_ref = self._lookup_by_name(instance_name) cpu_util = self._get_cpu_usage(vm_ref, instance_name) memory_usage = self._get_memory_usage(vm_ref) return virt_inspector.InstanceStats(cpu_util=cpu_util, memory_usage=memory_usage)
def inspect_instance(self, instance, duration=None): domain = self._get_domain_not_shut_off_or_raise(instance) memory_used = memory_resident = None memory_swap_in = memory_swap_out = None memory_stats = domain.memoryStats() # Stat provided from libvirt is in KB, converting it to MB. if 'usable' in memory_stats and 'available' in memory_stats: memory_used = (memory_stats['available'] - memory_stats['usable']) / units.Ki elif 'available' in memory_stats and 'unused' in memory_stats: memory_used = (memory_stats['available'] - memory_stats['unused']) / units.Ki if 'rss' in memory_stats: memory_resident = memory_stats['rss'] / units.Ki if 'swap_in' in memory_stats and 'swap_out' in memory_stats: memory_swap_in = memory_stats['swap_in'] / units.Ki memory_swap_out = memory_stats['swap_out'] / units.Ki # TODO(sileht): stats also have the disk/vnic info # we could use that instead of the old method for Queen stats = self.connection.domainListGetStats([domain], 0)[0][1] cpu_time = 0 current_cpus = stats.get('vcpu.current') # Iterate over the maximum number of CPUs here, and count the # actual number encountered, since the vcpu.x structure can # have holes according to # https://libvirt.org/git/?p=libvirt.git;a=blob;f=src/libvirt-domain.c # virConnectGetAllDomainStats() for vcpu in six.moves.range(stats.get('vcpu.maximum', 0)): try: cpu_time += (stats.get('vcpu.%s.time' % vcpu) + stats.get('vcpu.%s.wait' % vcpu)) current_cpus -= 1 except TypeError: # pass here, if there are too many holes, the cpu count will # not match, so don't need special error handling. pass if current_cpus: # There wasn't enough data, so fall back cpu_time = stats.get('cpu.time') return virt_inspector.InstanceStats( cpu_number=stats.get('vcpu.current'), cpu_time=cpu_time, memory_usage=memory_used, memory_resident=memory_resident, memory_swap_in=memory_swap_in, memory_swap_out=memory_swap_out, cpu_cycles=stats.get("perf.cpu_cycles"), instructions=stats.get("perf.instructions"), cache_references=stats.get("perf.cache_references"), cache_misses=stats.get("perf.cache_misses"), memory_bandwidth_total=stats.get("perf.mbmt"), memory_bandwidth_local=stats.get("perf.mbml"), cpu_l3_cache_usage=stats.get("perf.cmt"), )
def inspect_instance(self, instance, duration): instance_name = util.instance_name(instance) vm_ref = self._lookup_by_name(instance_name) cpu_util = self._get_cpu_usage(vm_ref, instance_name) memory_usage = self._get_memory_usage(vm_ref) LOG.debug("inspect_instance, cpu_util: %(cpu)s, memory_usage: %(mem)s", {'cpu': cpu_util, 'mem': memory_usage}, instance=instance) return virt_inspector.InstanceStats(cpu_util=cpu_util, memory_usage=memory_usage)
def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(cpu_util=40), virt_inspector.InstanceStats(cpu_util=60), ) mgr = manager.AgentManager(0, self.CONF) pollster = instance_stats.CPUUtilPollster(self.CONF) def _verify_cpu_util_metering(expected_util): cache = {} samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['cpu_util']), set([s.name for s in samples])) self.assertEqual(expected_util, samples[0].volume) _verify_cpu_util_metering(40) _verify_cpu_util_metering(60)
def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(cpu_l3_cache_usage=90112), virt_inspector.InstanceStats(cpu_l3_cache_usage=180224), ) mgr = manager.AgentManager(0, self.CONF) pollster = instance_stats.CPUL3CachePollster(self.CONF) def _verify_cpu_l3_cache_metering(expected_usage): cache = {} samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['cpu_l3_cache']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) _verify_cpu_l3_cache_metering(90112) _verify_cpu_l3_cache_metering(180224)
def inspect_instance(self, instance, duration): instance_name = util.instance_name(instance) (cpu_clock_used, cpu_count, uptime) = self._utils.get_cpu_metrics(instance_name) cpu_percent_used = cpu_clock_used / self._host_max_cpu_clock # Nanoseconds cpu_time = (int(uptime * cpu_percent_used) * units.k) memory_usage = self._utils.get_memory_metrics(instance_name) return virt_inspector.InstanceStats(cpu_number=cpu_count, cpu_time=cpu_time, memory_usage=memory_usage)
def test_get_samples(self): self._mock_inspect_instance( virt_inspector.InstanceStats(cpu_cycles=7259361, instructions=8815623, cache_references=74184, cache_misses=16737) ) mgr = manager.AgentManager(0, self.CONF) cache = {} def _check_perf_events_cpu_cycles(expected_usage): pollster = instance_stats.PerfCPUCyclesPollster(self.CONF) samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['perf.cpu.cycles']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) def _check_perf_events_instructions(expected_usage): pollster = instance_stats.PerfInstructionsPollster(self.CONF) samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['perf.instructions']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) def _check_perf_events_cache_references(expected_usage): pollster = instance_stats.PerfCacheReferencesPollster( self.CONF) samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['perf.cache.references']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) def _check_perf_events_cache_misses(expected_usage): pollster = instance_stats.PerfCacheMissesPollster(self.CONF) samples = list(pollster.get_samples(mgr, cache, [self.instance])) self.assertEqual(1, len(samples)) self.assertEqual(set(['perf.cache.misses']), set([s.name for s in samples])) self.assertEqual(expected_usage, samples[0].volume) _check_perf_events_cpu_cycles(7259361) _check_perf_events_instructions(8815623) _check_perf_events_cache_references(74184) _check_perf_events_cache_misses(16737)
def inspect_instance(self, instance, duration): vm_mobj = self._get_vm_mobj_not_power_off_or_raise(instance) cpu_util_counter_id = self._ops.get_perf_counter_id( VC_AVERAGE_CPU_CONSUMED_CNTR) cpu_util = self._ops.query_vm_aggregate_stats( vm_mobj, cpu_util_counter_id, duration) # For this counter vSphere returns values scaled-up by 100, since the # corresponding API can't return decimals, but only longs. # For e.g. if the utilization is 12.34%, the value returned is 1234. # Hence, dividing by 100. cpu_util = cpu_util / 100 mem_counter_id = self._ops.get_perf_counter_id( VC_AVERAGE_MEMORY_CONSUMED_CNTR) memory = self._ops.query_vm_aggregate_stats( vm_mobj, mem_counter_id, duration) # Stat provided from vSphere is in KB, converting it to MB. memory = memory / units.Ki return virt_inspector.InstanceStats( cpu_util=cpu_util, memory_usage=memory)
def inspect_instance(self, instance, duration): """Inspect the statistics for an instance. :param instance: the target instance :param duration: the last 'n' seconds, over which the value should be inspected. The PowerVM implementation does not make use of the duration field. :return: the instance statistics """ uuid = self._puuid(instance) cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid) # If the current metric is none, then the instance can not be found in # the sample set. An error should be raised. if cur_metric is None: raise virt_inspector.InstanceNotFoundException( _('VM %s not found in PowerVM Metrics Sample.') % instance.name) cpu_time = (cur_metric.processor.util_cap_proc_cycles + cur_metric.processor.util_uncap_proc_cycles) cpu_num = cur_metric.processor.virt_procs # The duration is ignored. There is precedent for this in other # inspectors if the platform doesn't support duration. # # Given the nature of PowerVM to collect samples over coarse periods # of time, it does not lend well to duration based collection. # Therefore this works by gathering the latest utilization from the # samples and ignores the duration. # Get the current and previous sample. Delta is performed between # these two. prev_date, prev_metric = self.vm_metrics.get_previous_metric(uuid) # Get the current data. cur_util_cap = cur_metric.processor.util_cap_proc_cycles cur_util_uncap = cur_metric.processor.util_uncap_proc_cycles cur_idle = cur_metric.processor.idle_proc_cycles cur_donated = cur_metric.processor.donated_proc_cycles cur_entitled = cur_metric.processor.entitled_proc_cycles # Get the previous sample data if prev_metric is None: # If there is no previous sample, that is either a new VM or is # a live migrated system. A live migrated system will pull all # of its metrics with it. The issue with this is it could have # CPU cycles for months of run time. So we can't really determine # the CPU utilization within the last X seconds...because to THIS # host it's new (only in the cur_metric). So we error out, the # inspector will use a debug message in the log. LOG.warning( 'Unable to derive CPU Utilization for VM %s. It is ' 'either a new VM or was recently migrated. It will be ' 'collected in the next inspection cycle.', instance.name) return virt_inspector.InstanceStats(cpu_time=cpu_time, cpu_number=cpu_num) # Gather the previous metrics prev_util_cap = prev_metric.processor.util_cap_proc_cycles prev_util_uncap = prev_metric.processor.util_uncap_proc_cycles prev_idle = prev_metric.processor.idle_proc_cycles prev_donated = prev_metric.processor.donated_proc_cycles prev_entitled = prev_metric.processor.entitled_proc_cycles # Utilization can be driven by multiple factors on PowerVM. # PowerVM has 'entitled' cycles. These are cycles that, if the VM # needs them, they get them no matter what. # # In terms of how those cycles are returned from the API: # util_cap_proc_cycles - How many cycles from the guaranteed # capacity were used. # util_uncap_proc_cycles - How many cycles were used that were # taken from spare (which is either unused processors cycles # or donated cycles from other VMs). # idle_proc_cycles - How many cycles (as reported by the OS to the # hypervisor) were reported as idle. # donated_proc_cycles - Cycles that were not needed by this VM that # were given to another VM in need of cycles. # # # So the final utilization equation is: # (util cap + util uncap - idle - donated) / entitled # # It is important to note that idle and donated proc cycles are # included in the 'util_cap_proc_cycles'. That is why they are # subtracted out. # # The interesting aspect of this is that the CPU Utilization can go # dramatically above 100% if there are free processors or if the # other workloads are in a lull. util_cap = cur_util_cap - prev_util_cap util_uncap = cur_util_uncap - prev_util_uncap idle = cur_idle - prev_idle donated = cur_donated - prev_donated entitled = cur_entitled - prev_entitled # If the entitled is zero, that generally means that the VM has not # been started yet (everything else would be zero as well). So to # avoid a divide by zero error, just return 0% in that case. util = (float(util_cap + util_uncap - idle - donated) / float(entitled) if entitled else 0.0) # Utilization is reported as percents. Therefore, multiply by 100.0 # to get a readable percentage based format. return virt_inspector.InstanceStats(cpu_util=util * 100.0, cpu_time=cpu_time, cpu_number=cpu_num)