def _get_cgroup_memory(treadmill_core_memory_limit): """get cgroup memory parameter for treadmill core/apps group """ total_physical_mem = sysinfo.mem_info().total * 1024 total_treadmill_mem = cgutils.get_memory_limit('treadmill') if total_treadmill_mem > total_physical_mem: _LOGGER.warning( 'memory limit for treadmill group > physical, use physical %s', utils.bytes_to_readable(total_treadmill_mem, 'B'), ) total_treadmill_mem = total_physical_mem # calculate memory allocation if treadmill_core_memory_limit is not None: core_memory = utils.size_to_bytes(treadmill_core_memory_limit) treadmill_memory = total_treadmill_mem apps_memory = treadmill_memory - core_memory else: core_memory = apps_memory = None _LOGGER.info( 'Configuring memory limits: ' 'phycial total: %s, ' 'treadmill total: %s, ' 'treadmill core: %s, ' 'treadmill apps: %s', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(total_treadmill_mem, 'B'), utils.bytes_to_readable(core_memory, 'B'), utils.bytes_to_readable(apps_memory, 'B')) return (core_memory, apps_memory)
def cginit(mem, mem_core, cpu, cpu_cores): """Initialize core and system cgroups.""" if cpu_cores > 0: tm_cpu_shares = sysinfo.bogomips_linux(range(0, cpu_cores)) tm_core_cpu_shares = int(tm_cpu_shares * 0.01) tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares total_cores = sysinfo.cpu_count() system_cores = range(cpu_cores, total_cores) system_cpu_shares = sysinfo.bogomips_linux(system_cores) _LOGGER.info( 'Configuring CPU limits: ' 'treadmill cores: %d, ' 'treadmill: %d, ' 'treadmill core: %d, ' 'treadmill apps: %d', cpu_cores, tm_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares) else: total_cores = sysinfo.cpu_count() total_cpu_shares = sysinfo.bogomips_linux(range(0, total_cores)) tm_cpu_shares = int(total_cpu_shares * utils.cpu_units(cpu) / 100.0) system_cpu_shares = int(total_cpu_shares - tm_cpu_shares) tm_core_cpu_shares = int(tm_cpu_shares * 0.01) tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares _LOGGER.info( 'Configuring CPU limits: ' 'total: %d, ' 'treadmill: %d, ' 'system: %d, ' 'treadmill core: %d, ' 'treadmill apps: %d', total_cpu_shares, tm_cpu_shares, system_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares) tm_mem = utils.size_to_bytes(mem) tm_core_mem = utils.size_to_bytes(mem_core) total_physical_mem = sysinfo.mem_info().total * 1024 if tm_mem <= 0: # For readability, instead of + tm_mem (negative). real_tm_mem = total_physical_mem - abs(tm_mem) else: real_tm_mem = tm_mem _LOGGER.info( 'Configuring memory limits: ' 'total: %s, ' 'treadmill total: %s, ' 'treadmill core: %s', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(real_tm_mem, 'B'), utils.bytes_to_readable(tm_core_mem, 'B')) cgutils.create_treadmill_cgroups(system_cpu_shares, tm_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares, cpu_cores, real_tm_mem, tm_core_mem)
def root(): """Root handler.""" mem = sysinfo.mem_info() return flask.jsonify({ 'hostname': sysinfo.hostname(), 'cpu': sysinfo.cpu_count(), 'memory': mem.total, })
def app_metrics(cgrp, blkio_major_minor=None): """Returns app metrics or empty dict if app not found.""" result = { 'memusage': 0, 'softmem': 0, 'hardmem': 0, 'cpuusage': 0, 'cpuusage_ratio': 0, 'blk_read_iops': 0, 'blk_write_iops': 0, 'blk_read_bps': 0, 'blk_write_bps': 0, } meminfo = sysinfo.mem_info() meminfo_total_bytes = meminfo.total * 1024 try: memusage, softmem, hardmem = read_memory_stats(cgrp) if softmem > meminfo_total_bytes: softmem = meminfo_total_bytes if hardmem > meminfo_total_bytes: hardmem = meminfo_total_bytes result.update({ 'memusage': memusage, 'softmem': softmem, 'hardmem': hardmem, }) cpuusage, _, cpuusage_ratio = read_cpu_stats(cgrp) result.update({ 'cpuusage': cpuusage, 'cpuusage_ratio': cpuusage_ratio, }) blkusage = read_blkio_stats(cgrp, blkio_major_minor) result.update({ 'blk_read_iops': blkusage['read_iops'], 'blk_write_iops': blkusage['write_iops'], 'blk_read_bps': blkusage['read_bps'], 'blk_write_bps': blkusage['write_bps'], }) except IOError as err: if err.errno != errno.ENOENT: raise err except OSError as err: if err.errno != errno.ENOENT: raise err return result
def test_mem_info(self): """Mock test for mem info.""" proc_meminfo = """ MemTotal: 7992596 kB MemFree: 3572940 kB Buffers: 202564 kB Cached: 2371108 kB SwapCached: 0 kB Active: 2959388 kB Inactive: 868476 kB HighTotal: 0 kB HighFree: 0 kB LowTotal: 7992596 kB LowFree: 3572940 kB SwapTotal: 4064436 kB SwapFree: 4064436 kB Dirty: 240 kB Writeback: 0 kB AnonPages: 1254148 kB Mapped: 104244 kB Slab: 500152 kB PageTables: 17180 kB NFS_Unstable: 0 kB Bounce: 0 kB CommitLimit: 11257772 kB Committed_AS: 2268028 kB VmallocTotal: 34359738367 kB VmallocUsed: 335508 kB VmallocChunk: 34359375019 kB HugePages_Total: 0 HugePages_Free: 0 HugePages_Rsvd: 0 Hugepagesize: 2048 kB """ open_mock = mock.mock_open(read_data=proc_meminfo.strip()) with mock.patch('builtins.open', open_mock, create=True): meminfo = sysinfo.mem_info() self.assertEqual(7992596, meminfo.total)
def test_mem_info(self): """Mock test for mem info.""" meminfo = sysinfo.mem_info() self.assertEqual(7992596, meminfo.total)
def app_metrics(rrd_last, raw_metrics, sys_major_minor): """ get rrd readable metrics""" result = { 'memusage': 0, 'softmem': 0, 'hardmem': 0, 'cputotal': 0, 'cpuusage': 0, 'cpuusage_ratio': 0, 'blk_read_iops': 0, 'blk_write_iops': 0, 'blk_read_bps': 0, 'blk_write_bps': 0, 'fs_used_bytes': 0, } memusage = raw_metrics['memory.usage_in_bytes'] softmem = raw_metrics['memory.soft_limit_in_bytes'] hardmem = raw_metrics['memory.limit_in_bytes'] meminfo = sysinfo.mem_info() meminfo_total_bytes = meminfo.total * 1024 if softmem > meminfo_total_bytes: softmem = meminfo_total_bytes if hardmem > meminfo_total_bytes: hardmem = meminfo_total_bytes result.update({ 'memusage': memusage, 'softmem': softmem, 'hardmem': hardmem, }) cpuusage = raw_metrics['cpuacct.usage'] timestamp = raw_metrics['timestamp'] # cpuacct usage from cgroup is in nanosecond if 'cputotal' in rrd_last and rrd_last['cputotal'] != 0: usage_delta_in_sec = float(cpuusage - rrd_last['cputotal']) / 10e9 time_delta = timestamp - rrd_last['timestamp'] else: # just give a rough estimate in the beginning usage_delta_in_sec = float(cpuusage) / 10e9 time_delta = timestamp cpu_share = raw_metrics['cpu.shares'] (cpuusage_percent, _, cpuusage_ratio) = get_cpu_metrics( usage_delta_in_sec, time_delta, cpu_share, ) result.update({ 'cputotal': cpuusage, 'cpuusage': cpuusage_percent, 'cpuusage_ratio': cpuusage_ratio, }) # TODO: so far a container only has one blk device with major minor blk_iops = raw_metrics['blkio.throttle.io_serviced'].get( sys_major_minor, { 'Read': 0, 'Write': 0 }) result.update({ 'blk_read_iops': blk_iops['Read'], 'blk_write_iops': blk_iops['Write'] }) blk_bps = raw_metrics['blkio.throttle.io_service_bytes'].get( sys_major_minor, { 'Read': 0, 'Write': 0 }) result.update({ 'blk_read_bps': blk_bps['Read'], 'blk_write_bps': blk_bps['Write'] }) result['fs_used_bytes'] = raw_metrics.get('fs.used_bytes', 0) # finally append timestamp result['timestamp'] = raw_metrics['timestamp'] return result
def _cgroup_init(treadmill_core_cpu_shares, treadmill_core_cpuset_cpus, treadmill_apps_cpuset_cpus, treadmill_core_memory_limit): # calculate CPU shares allocation total_cores = sysinfo.cpu_count() total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores)) core_cpu_shares = int( total_cpu_shares * utils.cpu_units(treadmill_core_cpu_shares) / 100.0 ) apps_cpu_shares = total_cpu_shares - core_cpu_shares _LOGGER.info( 'Configuring CPU shares: ' 'total: %d, ' 'treadmill core: %d, ' 'treadmill apps: %d', total_cpu_shares, core_cpu_shares, apps_cpu_shares ) # calculate memory allocation core_memory = utils.size_to_bytes(treadmill_core_memory_limit) total_physical_mem = sysinfo.mem_info().total * 1024 # XXX: tm_mem = utils.size_to_bytes(treadmill_mem) # XXX: if tm_mem <= 0: # XXX: real_tm_mem = total_physical_mem - abs(tm_mem) # XXX: else: # XXX: real_tm_mem = tm_mem treadmill_memory = total_physical_mem apps_memory = treadmill_memory - core_memory _LOGGER.info( 'Configuring memory limits: ' 'total: %s, ' 'treadmill core: %s, ' 'treadmill apps: %s', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(core_memory, 'B'), utils.bytes_to_readable(apps_memory, 'B') ) # calculate cpuset cores allocation core_cpuset_cpus = _parse_cpuset_cpus( treadmill_core_cpuset_cpus ) apps_cpuset_cpus = _parse_cpuset_cpus( treadmill_apps_cpuset_cpus ) _LOGGER.info( 'Configuring cpuset cores: ' 'treadmill core cpuset: %s, ' 'treadmill app cpuset: %s', core_cpuset_cpus, apps_cpuset_cpus ) cgutils.create_treadmill_cgroups( core_cpu_shares, apps_cpu_shares, core_cpuset_cpus, apps_cpuset_cpus, core_memory, apps_memory )
def _cgroup_init(treadmill_core_cpu_shares, treadmill_core_cpuset_cpus, treadmill_apps_cpuset_cpus, treadmill_core_memory_limit): # TODO: Refactor this whole function: # * It should be mostly folded into cgutils # * There is no need to read sysinfo for CPUs, use cpuset cgroup. # * There is no need to read bogomips for relative weight core/apps. # * There is no need to read physical memory here, use cgroups. # calculate CPU shares allocation total_cores = sysinfo.cpu_count() total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores)) if treadmill_core_cpu_shares is not None: core_cpu_shares = int( total_cpu_shares * utils.cpu_units(treadmill_core_cpu_shares) / 100.0 ) apps_cpu_shares = total_cpu_shares - core_cpu_shares else: core_cpu_shares = apps_cpu_shares = None _LOGGER.info( 'Configuring CPU shares: ' 'total: %d, ' 'treadmill core: %r, ' 'treadmill apps: %r', total_cpu_shares, core_cpu_shares, apps_cpu_shares ) # calculate memory allocation total_physical_mem = sysinfo.mem_info().total * 1024 if treadmill_core_memory_limit is not None: core_memory = utils.size_to_bytes(treadmill_core_memory_limit) # XXX: tm_mem = utils.size_to_bytes(treadmill_mem) # XXX: if tm_mem <= 0: # XXX: real_tm_mem = total_physical_mem - abs(tm_mem) # XXX: else: # XXX: real_tm_mem = tm_mem treadmill_memory = total_physical_mem apps_memory = treadmill_memory - core_memory else: core_memory = apps_memory = None _LOGGER.info( 'Configuring memory limits: ' 'total: %s, ' 'treadmill core: %r, ' 'treadmill apps: %r', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(core_memory, 'B'), utils.bytes_to_readable(apps_memory, 'B') ) # calculate cpuset cores allocation if treadmill_core_cpuset_cpus is not None: core_cpuset_cpus = _parse_cpuset_cpus( treadmill_core_cpuset_cpus ) # TODO: Calculate from apps as treadmill - core apps_cpuset_cpus = _parse_cpuset_cpus( treadmill_apps_cpuset_cpus ) else: core_cpuset_cpus = apps_cpuset_cpus = None _LOGGER.info( 'Configuring cpuset cores: ' 'treadmill cpuset: %r, ' 'treadmill core cpuset: %r, ' 'treadmill app cpuset: %r', 'TBD', core_cpuset_cpus, apps_cpuset_cpus ) cgutils.create_treadmill_cgroups( core_cpu_shares, apps_cpu_shares, core_cpuset_cpus, apps_cpuset_cpus, core_memory, apps_memory )
def app_metrics(cgrp, rrd_last, sys_major_minor, block_dev): """ get rrd readable metrics""" result = { 'memusage': 0, 'softmem': 0, 'hardmem': 0, 'cputotal': 0, 'cpuusage': 0, 'cpuusage_ratio': 0, 'blk_read_iops': 0, 'blk_write_iops': 0, 'blk_read_bps': 0, 'blk_write_bps': 0, 'fs_used_bytes': 0, } _LOGGER.debug('Getting metrics from cgroup %s, sys_maj_min %s', cgrp, sys_major_minor) raw_metrics = metrics.app_metrics(cgrp, block_dev) memusage = raw_metrics['memory.usage_in_bytes'] softmem = raw_metrics['memory.soft_limit_in_bytes'] hardmem = raw_metrics['memory.limit_in_bytes'] meminfo = sysinfo.mem_info() meminfo_total_bytes = meminfo.total * 1024 if softmem > meminfo_total_bytes: softmem = meminfo_total_bytes if hardmem > meminfo_total_bytes: hardmem = meminfo_total_bytes result.update({ 'memusage': memusage, 'softmem': softmem, 'hardmem': hardmem, }) cpuusage = raw_metrics['cpuacct.usage'] timestamp = raw_metrics['timestamp'] # cpuacct usage from cgroup is in nanosecond if 'cpu_total' in rrd_last and rrd_last['cpu_total'] != 0: usage_delta_in_sec = float(cpuusage - rrd_last['cpu_total']) / 10e9 time_delta = timestamp - rrd_last['timestamp'] else: # just give a rough estimate in the beginning usage_delta_in_sec = float(cpuusage) / 10e9 stat = cgutils.stat('cpuacct', cgrp, 'cpuacct.usage') time_delta = timestamp - stat.st_mtime (cpuusage_percent, _, cpuusage_ratio) = get_cpu_metrics(cgrp, usage_delta_in_sec, time_delta) result.update({ 'cputotal': cpuusage, 'cpuusage': cpuusage_percent, 'cpuusage_ratio': cpuusage_ratio, }) # TODO: so far a container only has one blk device with major minor blk_iops = raw_metrics['blkio.throttle.io_serviced'].get( sys_major_minor, { 'Read': 0, 'Write': 0 }) result.update({ 'blk_read_iops': blk_iops['Read'], 'blk_write_iops': blk_iops['Write'] }) blk_bps = raw_metrics['blkio.throttle.io_service_bytes'].get( sys_major_minor, { 'Read': 0, 'Write': 0 }) result.update({ 'blk_read_bps': blk_bps['Read'], 'blk_write_bps': blk_bps['Write'] }) result['fs_used_bytes'] = raw_metrics.get('fs.used_byes', 0) return result