def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(memory_resident=1.0),
virt_inspector.InstanceStats(memory_resident=2.0),
virt_inspector.InstanceStats(),
virt_inspector.InstanceShutOffException(),
)
mgr = manager.AgentManager(0, self.CONF)
pollster = instance_stats.MemoryResidentPollster(self.CONF)
@mock.patch('ceilometer.compute.pollsters.LOG')
def _verify_resident_memory_metering(expected_count,
expected_resident_memory_mb,
expected_warnings, mylog):
samples = list(pollster.get_samples(mgr, {}, [self.instance]))
self.assertEqual(expected_count, len(samples))
if expected_count > 0:
self.assertEqual(set(['memory.resident']),
set([s.name for s in samples]))
self.assertEqual(expected_resident_memory_mb,
samples[0].volume)
else:
self.assertEqual(expected_warnings, mylog.warning.call_count)
self.assertEqual(0, mylog.exception.call_count)
_verify_resident_memory_metering(1, 1.0, 0)
_verify_resident_memory_metering(1, 2.0, 0)
_verify_resident_memory_metering(0, 0, 1)
_verify_resident_memory_metering(0, 0, 0)
def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(memory_swap_in=1.0,
memory_swap_out=2.0),
virt_inspector.InstanceStats(memory_swap_in=3.0,
memory_swap_out=4.0),
)
mgr = manager.AgentManager(0, self.CONF)
def _check_memory_swap_in(expected_swap_in):
pollster = instance_stats.MemorySwapInPollster(self.CONF)
samples = list(pollster.get_samples(mgr, {}, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['memory.swap.in']),
set([s.name for s in samples]))
self.assertEqual(expected_swap_in, samples[0].volume)
def _check_memory_swap_out(expected_swap_out):
pollster = instance_stats.MemorySwapOutPollster(self.CONF)
samples = list(pollster.get_samples(mgr, {}, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['memory.swap.out']),
set([s.name for s in samples]))
self.assertEqual(expected_swap_out, samples[0].volume)
_check_memory_swap_in(1.0)
_check_memory_swap_out(4.0)
def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(memory_bandwidth_total=1892352,
memory_bandwidth_local=1802240),
virt_inspector.InstanceStats(memory_bandwidth_total=1081344,
memory_bandwidth_local=90112),
)
mgr = manager.AgentManager(0, self.CONF)
def _check_memory_bandwidth_total(expected_usage):
pollster = instance_stats.MemoryBandwidthTotalPollster(self.CONF)
samples = list(pollster.get_samples(mgr, {}, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['memory.bandwidth.total']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
def _check_memory_bandwidth_local(expected_usage):
pollster = instance_stats.MemoryBandwidthLocalPollster(self.CONF)
samples = list(pollster.get_samples(mgr, {}, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['memory.bandwidth.local']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
_check_memory_bandwidth_total(1892352)
_check_memory_bandwidth_local(90112)
def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(cpu_time=1 * (10**6),
cpu_number=2,
vcpu_number=2),
virt_inspector.InstanceStats(cpu_time=3 * (10**6),
cpu_number=2,
vcpu_number=2),
# cpu_time resets on instance restart
virt_inspector.InstanceStats(cpu_time=2 * (10**6),
cpu_number=2,
vcpu_number=2),
)
mgr = manager.AgentManager(0, self.CONF)
pollster = instance_stats.CPUPollster(self.CONF)
def _verify_cpu_metering(expected_time):
cache = {}
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['cpu']), set([s.name for s in samples]))
self.assertEqual(expected_time, samples[0].volume)
self.assertEqual(2, samples[0].resource_metadata.get('cpu_number'))
self.assertEqual(2,
samples[0].resource_metadata.get('vcpu_number'))
# ensure elapsed time between polling cycles is non-zero
time.sleep(0.001)
_verify_cpu_metering(1 * (10**6))
_verify_cpu_metering(3 * (10**6))
_verify_cpu_metering(2 * (10**6))
def inspect_instance(self, instance, duration=None):
domain = self._get_domain_not_shut_off_or_raise(instance)
memory_used = memory_resident = None
memory_stats = domain.memoryStats()
# Stat provided from libvirt is in KB, converting it to MB.
if 'available' in memory_stats and 'unused' in memory_stats:
memory_used = (memory_stats['available'] -
memory_stats['unused']) / units.Ki
if 'rss' in memory_stats:
memory_resident = memory_stats['rss'] / units.Ki
# TODO(sileht): stats also have the disk/vnic info
# we could use that instead of the old method for Queen
stats = self.connection.domainListGetStats([domain], 0)[0][1]
return virt_inspector.InstanceStats(
cpu_number=stats.get('vcpu.current'),
cpu_time=stats.get('cpu.time'),
memory_usage=memory_used,
memory_resident=memory_resident,
cpu_cycles=stats.get("perf.cpu_cycles"),
instructions=stats.get("perf.instructions"),
cache_references=stats.get("perf.cache_references"),
cache_misses=stats.get("perf.cache_misses"),
memory_bandwidth_total=stats.get("perf.mbmt"),
memory_bandwidth_local=stats.get("perf.mbml"),
cpu_l3_cache_usage=stats.get("perf.cmt"),
)
def inspect_instance(self, instance, duration):
instance_name = util.instance_name(instance)
vm_ref = self._lookup_by_name(instance_name)
cpu_util = self._get_cpu_usage(vm_ref, instance_name)
memory_usage = self._get_memory_usage(vm_ref)
return virt_inspector.InstanceStats(cpu_util=cpu_util,
memory_usage=memory_usage)
def inspect_instance(self, instance, duration=None):
domain = self._get_domain_not_shut_off_or_raise(instance)
memory_used = memory_resident = None
memory_swap_in = memory_swap_out = None
memory_stats = domain.memoryStats()
# Stat provided from libvirt is in KB, converting it to MB.
if 'usable' in memory_stats and 'available' in memory_stats:
memory_used = (memory_stats['available'] -
memory_stats['usable']) / units.Ki
elif 'available' in memory_stats and 'unused' in memory_stats:
memory_used = (memory_stats['available'] -
memory_stats['unused']) / units.Ki
if 'rss' in memory_stats:
memory_resident = memory_stats['rss'] / units.Ki
if 'swap_in' in memory_stats and 'swap_out' in memory_stats:
memory_swap_in = memory_stats['swap_in'] / units.Ki
memory_swap_out = memory_stats['swap_out'] / units.Ki
# TODO(sileht): stats also have the disk/vnic info
# we could use that instead of the old method for Queen
stats = self.connection.domainListGetStats([domain], 0)[0][1]
cpu_time = 0
current_cpus = stats.get('vcpu.current')
# Iterate over the maximum number of CPUs here, and count the
# actual number encountered, since the vcpu.x structure can
# have holes according to
# https://libvirt.org/git/?p=libvirt.git;a=blob;f=src/libvirt-domain.c
# virConnectGetAllDomainStats()
for vcpu in six.moves.range(stats.get('vcpu.maximum', 0)):
try:
cpu_time += (stats.get('vcpu.%s.time' % vcpu) +
stats.get('vcpu.%s.wait' % vcpu))
current_cpus -= 1
except TypeError:
# pass here, if there are too many holes, the cpu count will
# not match, so don't need special error handling.
pass
if current_cpus:
# There wasn't enough data, so fall back
cpu_time = stats.get('cpu.time')
return virt_inspector.InstanceStats(
cpu_number=stats.get('vcpu.current'),
cpu_time=cpu_time,
memory_usage=memory_used,
memory_resident=memory_resident,
memory_swap_in=memory_swap_in,
memory_swap_out=memory_swap_out,
cpu_cycles=stats.get("perf.cpu_cycles"),
instructions=stats.get("perf.instructions"),
cache_references=stats.get("perf.cache_references"),
cache_misses=stats.get("perf.cache_misses"),
memory_bandwidth_total=stats.get("perf.mbmt"),
memory_bandwidth_local=stats.get("perf.mbml"),
cpu_l3_cache_usage=stats.get("perf.cmt"),
)
def inspect_instance(self, instance, duration):
instance_name = util.instance_name(instance)
vm_ref = self._lookup_by_name(instance_name)
cpu_util = self._get_cpu_usage(vm_ref, instance_name)
memory_usage = self._get_memory_usage(vm_ref)
LOG.debug("inspect_instance, cpu_util: %(cpu)s, memory_usage: %(mem)s",
{'cpu': cpu_util, 'mem': memory_usage}, instance=instance)
return virt_inspector.InstanceStats(cpu_util=cpu_util,
memory_usage=memory_usage)
def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(cpu_util=40),
virt_inspector.InstanceStats(cpu_util=60),
)
mgr = manager.AgentManager(0, self.CONF)
pollster = instance_stats.CPUUtilPollster(self.CONF)
def _verify_cpu_util_metering(expected_util):
cache = {}
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['cpu_util']), set([s.name for s in samples]))
self.assertEqual(expected_util, samples[0].volume)
_verify_cpu_util_metering(40)
_verify_cpu_util_metering(60)
def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(cpu_l3_cache_usage=90112),
virt_inspector.InstanceStats(cpu_l3_cache_usage=180224),
)
mgr = manager.AgentManager(0, self.CONF)
pollster = instance_stats.CPUL3CachePollster(self.CONF)
def _verify_cpu_l3_cache_metering(expected_usage):
cache = {}
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['cpu_l3_cache']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
_verify_cpu_l3_cache_metering(90112)
_verify_cpu_l3_cache_metering(180224)
def inspect_instance(self, instance, duration):
instance_name = util.instance_name(instance)
(cpu_clock_used, cpu_count,
uptime) = self._utils.get_cpu_metrics(instance_name)
cpu_percent_used = cpu_clock_used / self._host_max_cpu_clock
# Nanoseconds
cpu_time = (int(uptime * cpu_percent_used) * units.k)
memory_usage = self._utils.get_memory_metrics(instance_name)
return virt_inspector.InstanceStats(cpu_number=cpu_count,
cpu_time=cpu_time,
memory_usage=memory_usage)
def test_get_samples(self):
self._mock_inspect_instance(
virt_inspector.InstanceStats(cpu_cycles=7259361,
instructions=8815623,
cache_references=74184,
cache_misses=16737)
)
mgr = manager.AgentManager(0, self.CONF)
cache = {}
def _check_perf_events_cpu_cycles(expected_usage):
pollster = instance_stats.PerfCPUCyclesPollster(self.CONF)
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['perf.cpu.cycles']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
def _check_perf_events_instructions(expected_usage):
pollster = instance_stats.PerfInstructionsPollster(self.CONF)
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['perf.instructions']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
def _check_perf_events_cache_references(expected_usage):
pollster = instance_stats.PerfCacheReferencesPollster(
self.CONF)
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['perf.cache.references']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
def _check_perf_events_cache_misses(expected_usage):
pollster = instance_stats.PerfCacheMissesPollster(self.CONF)
samples = list(pollster.get_samples(mgr, cache, [self.instance]))
self.assertEqual(1, len(samples))
self.assertEqual(set(['perf.cache.misses']),
set([s.name for s in samples]))
self.assertEqual(expected_usage, samples[0].volume)
_check_perf_events_cpu_cycles(7259361)
_check_perf_events_instructions(8815623)
_check_perf_events_cache_references(74184)
_check_perf_events_cache_misses(16737)
def inspect_instance(self, instance, duration):
vm_mobj = self._get_vm_mobj_not_power_off_or_raise(instance)
cpu_util_counter_id = self._ops.get_perf_counter_id(
VC_AVERAGE_CPU_CONSUMED_CNTR)
cpu_util = self._ops.query_vm_aggregate_stats(
vm_mobj, cpu_util_counter_id, duration)
# For this counter vSphere returns values scaled-up by 100, since the
# corresponding API can't return decimals, but only longs.
# For e.g. if the utilization is 12.34%, the value returned is 1234.
# Hence, dividing by 100.
cpu_util = cpu_util / 100
mem_counter_id = self._ops.get_perf_counter_id(
VC_AVERAGE_MEMORY_CONSUMED_CNTR)
memory = self._ops.query_vm_aggregate_stats(
vm_mobj, mem_counter_id, duration)
# Stat provided from vSphere is in KB, converting it to MB.
memory = memory / units.Ki
return virt_inspector.InstanceStats(
cpu_util=cpu_util,
memory_usage=memory)
def inspect_instance(self, instance, duration):
"""Inspect the statistics for an instance.
:param instance: the target instance
:param duration: the last 'n' seconds, over which the value should be
inspected.
The PowerVM implementation does not make use of the duration
field.
:return: the instance statistics
"""
uuid = self._puuid(instance)
cur_date, cur_metric = self.vm_metrics.get_latest_metric(uuid)
# If the current metric is none, then the instance can not be found in
# the sample set. An error should be raised.
if cur_metric is None:
raise virt_inspector.InstanceNotFoundException(
_('VM %s not found in PowerVM Metrics Sample.') %
instance.name)
cpu_time = (cur_metric.processor.util_cap_proc_cycles +
cur_metric.processor.util_uncap_proc_cycles)
cpu_num = cur_metric.processor.virt_procs
# The duration is ignored. There is precedent for this in other
# inspectors if the platform doesn't support duration.
#
# Given the nature of PowerVM to collect samples over coarse periods
# of time, it does not lend well to duration based collection.
# Therefore this works by gathering the latest utilization from the
# samples and ignores the duration.
# Get the current and previous sample. Delta is performed between
# these two.
prev_date, prev_metric = self.vm_metrics.get_previous_metric(uuid)
# Get the current data.
cur_util_cap = cur_metric.processor.util_cap_proc_cycles
cur_util_uncap = cur_metric.processor.util_uncap_proc_cycles
cur_idle = cur_metric.processor.idle_proc_cycles
cur_donated = cur_metric.processor.donated_proc_cycles
cur_entitled = cur_metric.processor.entitled_proc_cycles
# Get the previous sample data
if prev_metric is None:
# If there is no previous sample, that is either a new VM or is
# a live migrated system. A live migrated system will pull all
# of its metrics with it. The issue with this is it could have
# CPU cycles for months of run time. So we can't really determine
# the CPU utilization within the last X seconds...because to THIS
# host it's new (only in the cur_metric). So we error out, the
# inspector will use a debug message in the log.
LOG.warning(
'Unable to derive CPU Utilization for VM %s. It is '
'either a new VM or was recently migrated. It will be '
'collected in the next inspection cycle.', instance.name)
return virt_inspector.InstanceStats(cpu_time=cpu_time,
cpu_number=cpu_num)
# Gather the previous metrics
prev_util_cap = prev_metric.processor.util_cap_proc_cycles
prev_util_uncap = prev_metric.processor.util_uncap_proc_cycles
prev_idle = prev_metric.processor.idle_proc_cycles
prev_donated = prev_metric.processor.donated_proc_cycles
prev_entitled = prev_metric.processor.entitled_proc_cycles
# Utilization can be driven by multiple factors on PowerVM.
# PowerVM has 'entitled' cycles. These are cycles that, if the VM
# needs them, they get them no matter what.
#
# In terms of how those cycles are returned from the API:
# util_cap_proc_cycles - How many cycles from the guaranteed
# capacity were used.
# util_uncap_proc_cycles - How many cycles were used that were
# taken from spare (which is either unused processors cycles
# or donated cycles from other VMs).
# idle_proc_cycles - How many cycles (as reported by the OS to the
# hypervisor) were reported as idle.
# donated_proc_cycles - Cycles that were not needed by this VM that
# were given to another VM in need of cycles.
#
#
# So the final utilization equation is:
# (util cap + util uncap - idle - donated) / entitled
#
# It is important to note that idle and donated proc cycles are
# included in the 'util_cap_proc_cycles'. That is why they are
# subtracted out.
#
# The interesting aspect of this is that the CPU Utilization can go
# dramatically above 100% if there are free processors or if the
# other workloads are in a lull.
util_cap = cur_util_cap - prev_util_cap
util_uncap = cur_util_uncap - prev_util_uncap
idle = cur_idle - prev_idle
donated = cur_donated - prev_donated
entitled = cur_entitled - prev_entitled
# If the entitled is zero, that generally means that the VM has not
# been started yet (everything else would be zero as well). So to
# avoid a divide by zero error, just return 0% in that case.
util = (float(util_cap + util_uncap - idle - donated) /
float(entitled) if entitled else 0.0)
# Utilization is reported as percents. Therefore, multiply by 100.0
# to get a readable percentage based format.
return virt_inspector.InstanceStats(cpu_util=util * 100.0,
cpu_time=cpu_time,
cpu_number=cpu_num)
