def _get_cgroup_memory(treadmill_core_memory_limit):
"""get cgroup memory parameter for treadmill core/apps group
"""
total_physical_mem = sysinfo.mem_info().total * 1024
total_treadmill_mem = cgutils.get_memory_limit('treadmill')
if total_treadmill_mem > total_physical_mem:
_LOGGER.warning(
'memory limit for treadmill group > physical, use physical %s',
utils.bytes_to_readable(total_treadmill_mem, 'B'),
)
total_treadmill_mem = total_physical_mem
# calculate memory allocation
if treadmill_core_memory_limit is not None:
core_memory = utils.size_to_bytes(treadmill_core_memory_limit)
treadmill_memory = total_treadmill_mem
apps_memory = treadmill_memory - core_memory
else:
core_memory = apps_memory = None
_LOGGER.info(
'Configuring memory limits: '
'phycial total: %s, '
'treadmill total: %s, '
'treadmill core: %s, '
'treadmill apps: %s', utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(total_treadmill_mem, 'B'),
utils.bytes_to_readable(core_memory, 'B'),
utils.bytes_to_readable(apps_memory, 'B'))
return (core_memory, apps_memory)
def cginit(mem, mem_core, cpu, cpu_cores):
"""Initialize core and system cgroups."""
if cpu_cores > 0:
tm_cpu_shares = sysinfo.bogomips_linux(range(0, cpu_cores))
tm_core_cpu_shares = int(tm_cpu_shares * 0.01)
tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares
total_cores = sysinfo.cpu_count()
system_cores = range(cpu_cores, total_cores)
system_cpu_shares = sysinfo.bogomips_linux(system_cores)
_LOGGER.info(
'Configuring CPU limits: '
'treadmill cores: %d, '
'treadmill: %d, '
'treadmill core: %d, '
'treadmill apps: %d', cpu_cores, tm_cpu_shares,
tm_core_cpu_shares, tm_apps_cpu_shares)
else:
total_cores = sysinfo.cpu_count()
total_cpu_shares = sysinfo.bogomips_linux(range(0, total_cores))
tm_cpu_shares = int(total_cpu_shares * utils.cpu_units(cpu) /
100.0)
system_cpu_shares = int(total_cpu_shares - tm_cpu_shares)
tm_core_cpu_shares = int(tm_cpu_shares * 0.01)
tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares
_LOGGER.info(
'Configuring CPU limits: '
'total: %d, '
'treadmill: %d, '
'system: %d, '
'treadmill core: %d, '
'treadmill apps: %d', total_cpu_shares, tm_cpu_shares,
system_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares)
tm_mem = utils.size_to_bytes(mem)
tm_core_mem = utils.size_to_bytes(mem_core)
total_physical_mem = sysinfo.mem_info().total * 1024
if tm_mem <= 0:
# For readability, instead of + tm_mem (negative).
real_tm_mem = total_physical_mem - abs(tm_mem)
else:
real_tm_mem = tm_mem
_LOGGER.info(
'Configuring memory limits: '
'total: %s, '
'treadmill total: %s, '
'treadmill core: %s',
utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(real_tm_mem, 'B'),
utils.bytes_to_readable(tm_core_mem, 'B'))
cgutils.create_treadmill_cgroups(system_cpu_shares, tm_cpu_shares,
tm_core_cpu_shares,
tm_apps_cpu_shares, cpu_cores,
real_tm_mem, tm_core_mem)
def root():
"""Root handler."""
mem = sysinfo.mem_info()
return flask.jsonify({
'hostname': sysinfo.hostname(),
'cpu': sysinfo.cpu_count(),
'memory': mem.total,
})
def app_metrics(cgrp, blkio_major_minor=None):
"""Returns app metrics or empty dict if app not found."""
result = {
'memusage': 0,
'softmem': 0,
'hardmem': 0,
'cpuusage': 0,
'cpuusage_ratio': 0,
'blk_read_iops': 0,
'blk_write_iops': 0,
'blk_read_bps': 0,
'blk_write_bps': 0,
}
meminfo = sysinfo.mem_info()
meminfo_total_bytes = meminfo.total * 1024
try:
memusage, softmem, hardmem = read_memory_stats(cgrp)
if softmem > meminfo_total_bytes:
softmem = meminfo_total_bytes
if hardmem > meminfo_total_bytes:
hardmem = meminfo_total_bytes
result.update({
'memusage': memusage,
'softmem': softmem,
'hardmem': hardmem,
})
cpuusage, _, cpuusage_ratio = read_cpu_stats(cgrp)
result.update({
'cpuusage': cpuusage,
'cpuusage_ratio': cpuusage_ratio,
})
blkusage = read_blkio_stats(cgrp, blkio_major_minor)
result.update({
'blk_read_iops': blkusage['read_iops'],
'blk_write_iops': blkusage['write_iops'],
'blk_read_bps': blkusage['read_bps'],
'blk_write_bps': blkusage['write_bps'],
})
except IOError as err:
if err.errno != errno.ENOENT:
raise err
except OSError as err:
if err.errno != errno.ENOENT:
raise err
return result
def test_mem_info(self):
"""Mock test for mem info."""
proc_meminfo = """
MemTotal: 7992596 kB
MemFree: 3572940 kB
Buffers: 202564 kB
Cached: 2371108 kB
SwapCached: 0 kB
Active: 2959388 kB
Inactive: 868476 kB
HighTotal: 0 kB
HighFree: 0 kB
LowTotal: 7992596 kB
LowFree: 3572940 kB
SwapTotal: 4064436 kB
SwapFree: 4064436 kB
Dirty: 240 kB
Writeback: 0 kB
AnonPages: 1254148 kB
Mapped: 104244 kB
Slab: 500152 kB
PageTables: 17180 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
CommitLimit: 11257772 kB
Committed_AS: 2268028 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 335508 kB
VmallocChunk: 34359375019 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
Hugepagesize: 2048 kB
"""
open_mock = mock.mock_open(read_data=proc_meminfo.strip())
with mock.patch('builtins.open', open_mock, create=True):
meminfo = sysinfo.mem_info()
self.assertEqual(7992596, meminfo.total)
def test_mem_info(self):
"""Mock test for mem info."""
meminfo = sysinfo.mem_info()
self.assertEqual(7992596, meminfo.total)
def app_metrics(rrd_last, raw_metrics, sys_major_minor):
""" get rrd readable metrics"""
result = {
'memusage': 0,
'softmem': 0,
'hardmem': 0,
'cputotal': 0,
'cpuusage': 0,
'cpuusage_ratio': 0,
'blk_read_iops': 0,
'blk_write_iops': 0,
'blk_read_bps': 0,
'blk_write_bps': 0,
'fs_used_bytes': 0,
}
memusage = raw_metrics['memory.usage_in_bytes']
softmem = raw_metrics['memory.soft_limit_in_bytes']
hardmem = raw_metrics['memory.limit_in_bytes']
meminfo = sysinfo.mem_info()
meminfo_total_bytes = meminfo.total * 1024
if softmem > meminfo_total_bytes:
softmem = meminfo_total_bytes
if hardmem > meminfo_total_bytes:
hardmem = meminfo_total_bytes
result.update({
'memusage': memusage,
'softmem': softmem,
'hardmem': hardmem,
})
cpuusage = raw_metrics['cpuacct.usage']
timestamp = raw_metrics['timestamp']
# cpuacct usage from cgroup is in nanosecond
if 'cputotal' in rrd_last and rrd_last['cputotal'] != 0:
usage_delta_in_sec = float(cpuusage - rrd_last['cputotal']) / 10e9
time_delta = timestamp - rrd_last['timestamp']
else:
# just give a rough estimate in the beginning
usage_delta_in_sec = float(cpuusage) / 10e9
time_delta = timestamp
cpu_share = raw_metrics['cpu.shares']
(cpuusage_percent, _, cpuusage_ratio) = get_cpu_metrics(
usage_delta_in_sec,
time_delta,
cpu_share,
)
result.update({
'cputotal': cpuusage,
'cpuusage': cpuusage_percent,
'cpuusage_ratio': cpuusage_ratio,
})
# TODO: so far a container only has one blk device with major minor
blk_iops = raw_metrics['blkio.throttle.io_serviced'].get(
sys_major_minor, {
'Read': 0,
'Write': 0
})
result.update({
'blk_read_iops': blk_iops['Read'],
'blk_write_iops': blk_iops['Write']
})
blk_bps = raw_metrics['blkio.throttle.io_service_bytes'].get(
sys_major_minor, {
'Read': 0,
'Write': 0
})
result.update({
'blk_read_bps': blk_bps['Read'],
'blk_write_bps': blk_bps['Write']
})
result['fs_used_bytes'] = raw_metrics.get('fs.used_bytes', 0)
# finally append timestamp
result['timestamp'] = raw_metrics['timestamp']
return result
def _cgroup_init(treadmill_core_cpu_shares,
treadmill_core_cpuset_cpus,
treadmill_apps_cpuset_cpus,
treadmill_core_memory_limit):
# calculate CPU shares allocation
total_cores = sysinfo.cpu_count()
total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores))
core_cpu_shares = int(
total_cpu_shares *
utils.cpu_units(treadmill_core_cpu_shares) / 100.0
)
apps_cpu_shares = total_cpu_shares - core_cpu_shares
_LOGGER.info(
'Configuring CPU shares: '
'total: %d, '
'treadmill core: %d, '
'treadmill apps: %d',
total_cpu_shares,
core_cpu_shares,
apps_cpu_shares
)
# calculate memory allocation
core_memory = utils.size_to_bytes(treadmill_core_memory_limit)
total_physical_mem = sysinfo.mem_info().total * 1024
# XXX: tm_mem = utils.size_to_bytes(treadmill_mem)
# XXX: if tm_mem <= 0:
# XXX: real_tm_mem = total_physical_mem - abs(tm_mem)
# XXX: else:
# XXX: real_tm_mem = tm_mem
treadmill_memory = total_physical_mem
apps_memory = treadmill_memory - core_memory
_LOGGER.info(
'Configuring memory limits: '
'total: %s, '
'treadmill core: %s, '
'treadmill apps: %s',
utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(core_memory, 'B'),
utils.bytes_to_readable(apps_memory, 'B')
)
# calculate cpuset cores allocation
core_cpuset_cpus = _parse_cpuset_cpus(
treadmill_core_cpuset_cpus
)
apps_cpuset_cpus = _parse_cpuset_cpus(
treadmill_apps_cpuset_cpus
)
_LOGGER.info(
'Configuring cpuset cores: '
'treadmill core cpuset: %s, '
'treadmill app cpuset: %s',
core_cpuset_cpus,
apps_cpuset_cpus
)
cgutils.create_treadmill_cgroups(
core_cpu_shares,
apps_cpu_shares,
core_cpuset_cpus,
apps_cpuset_cpus,
core_memory,
apps_memory
)
def _cgroup_init(treadmill_core_cpu_shares,
treadmill_core_cpuset_cpus,
treadmill_apps_cpuset_cpus,
treadmill_core_memory_limit):
# TODO: Refactor this whole function:
# * It should be mostly folded into cgutils
# * There is no need to read sysinfo for CPUs, use cpuset cgroup.
# * There is no need to read bogomips for relative weight core/apps.
# * There is no need to read physical memory here, use cgroups.
# calculate CPU shares allocation
total_cores = sysinfo.cpu_count()
total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores))
if treadmill_core_cpu_shares is not None:
core_cpu_shares = int(
total_cpu_shares *
utils.cpu_units(treadmill_core_cpu_shares) / 100.0
)
apps_cpu_shares = total_cpu_shares - core_cpu_shares
else:
core_cpu_shares = apps_cpu_shares = None
_LOGGER.info(
'Configuring CPU shares: '
'total: %d, '
'treadmill core: %r, '
'treadmill apps: %r',
total_cpu_shares,
core_cpu_shares,
apps_cpu_shares
)
# calculate memory allocation
total_physical_mem = sysinfo.mem_info().total * 1024
if treadmill_core_memory_limit is not None:
core_memory = utils.size_to_bytes(treadmill_core_memory_limit)
# XXX: tm_mem = utils.size_to_bytes(treadmill_mem)
# XXX: if tm_mem <= 0:
# XXX: real_tm_mem = total_physical_mem - abs(tm_mem)
# XXX: else:
# XXX: real_tm_mem = tm_mem
treadmill_memory = total_physical_mem
apps_memory = treadmill_memory - core_memory
else:
core_memory = apps_memory = None
_LOGGER.info(
'Configuring memory limits: '
'total: %s, '
'treadmill core: %r, '
'treadmill apps: %r',
utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(core_memory, 'B'),
utils.bytes_to_readable(apps_memory, 'B')
)
# calculate cpuset cores allocation
if treadmill_core_cpuset_cpus is not None:
core_cpuset_cpus = _parse_cpuset_cpus(
treadmill_core_cpuset_cpus
)
# TODO: Calculate from apps as treadmill - core
apps_cpuset_cpus = _parse_cpuset_cpus(
treadmill_apps_cpuset_cpus
)
else:
core_cpuset_cpus = apps_cpuset_cpus = None
_LOGGER.info(
'Configuring cpuset cores: '
'treadmill cpuset: %r, '
'treadmill core cpuset: %r, '
'treadmill app cpuset: %r',
'TBD',
core_cpuset_cpus,
apps_cpuset_cpus
)
cgutils.create_treadmill_cgroups(
core_cpu_shares,
apps_cpu_shares,
core_cpuset_cpus,
apps_cpuset_cpus,
core_memory,
apps_memory
)
def app_metrics(cgrp, rrd_last, sys_major_minor, block_dev):
""" get rrd readable metrics"""
result = {
'memusage': 0,
'softmem': 0,
'hardmem': 0,
'cputotal': 0,
'cpuusage': 0,
'cpuusage_ratio': 0,
'blk_read_iops': 0,
'blk_write_iops': 0,
'blk_read_bps': 0,
'blk_write_bps': 0,
'fs_used_bytes': 0,
}
_LOGGER.debug('Getting metrics from cgroup %s, sys_maj_min %s', cgrp,
sys_major_minor)
raw_metrics = metrics.app_metrics(cgrp, block_dev)
memusage = raw_metrics['memory.usage_in_bytes']
softmem = raw_metrics['memory.soft_limit_in_bytes']
hardmem = raw_metrics['memory.limit_in_bytes']
meminfo = sysinfo.mem_info()
meminfo_total_bytes = meminfo.total * 1024
if softmem > meminfo_total_bytes:
softmem = meminfo_total_bytes
if hardmem > meminfo_total_bytes:
hardmem = meminfo_total_bytes
result.update({
'memusage': memusage,
'softmem': softmem,
'hardmem': hardmem,
})
cpuusage = raw_metrics['cpuacct.usage']
timestamp = raw_metrics['timestamp']
# cpuacct usage from cgroup is in nanosecond
if 'cpu_total' in rrd_last and rrd_last['cpu_total'] != 0:
usage_delta_in_sec = float(cpuusage - rrd_last['cpu_total']) / 10e9
time_delta = timestamp - rrd_last['timestamp']
else:
# just give a rough estimate in the beginning
usage_delta_in_sec = float(cpuusage) / 10e9
stat = cgutils.stat('cpuacct', cgrp, 'cpuacct.usage')
time_delta = timestamp - stat.st_mtime
(cpuusage_percent, _,
cpuusage_ratio) = get_cpu_metrics(cgrp, usage_delta_in_sec, time_delta)
result.update({
'cputotal': cpuusage,
'cpuusage': cpuusage_percent,
'cpuusage_ratio': cpuusage_ratio,
})
# TODO: so far a container only has one blk device with major minor
blk_iops = raw_metrics['blkio.throttle.io_serviced'].get(
sys_major_minor, {
'Read': 0,
'Write': 0
})
result.update({
'blk_read_iops': blk_iops['Read'],
'blk_write_iops': blk_iops['Write']
})
blk_bps = raw_metrics['blkio.throttle.io_service_bytes'].get(
sys_major_minor, {
'Read': 0,
'Write': 0
})
result.update({
'blk_read_bps': blk_bps['Read'],
'blk_write_bps': blk_bps['Write']
})
result['fs_used_bytes'] = raw_metrics.get('fs.used_byes', 0)
return result
