def _configure_core_cgroups(service_name):
"""Configure service specific cgroups."""
group = os.path.join('treadmill/core', service_name)
# create group directory
for subsystem in ['memory', 'cpu', 'cpuacct', 'blkio']:
_LOGGER.info('creating and joining: %s/%s', subsystem, group)
cgutils.create(subsystem, group)
cgroups.join(subsystem, group)
# set memory usage limits
memlimits = [
'memory.limit_in_bytes', 'memory.memsw.limit_in_bytes',
'memory.soft_limit_in_bytes'
]
for limit in memlimits:
parent_limit = cgroups.get_value('memory', 'treadmill/core', limit)
_LOGGER.info('setting %s: %s', limit, parent_limit)
cgroups.set_value('memory', group, limit, parent_limit)
# set cpu share limits
cpulimits = ['cpu.shares']
for limit in cpulimits:
parent_limit = cgroups.get_value('cpu', 'treadmill/core', limit)
_LOGGER.info('setting %s: %s', limit, parent_limit)
cgroups.set_value('cpu', group, limit, parent_limit)
def _transfer_processes(subsystem, fromgroup, togroup):
"""Do the transfer of processes"""
pids = cgroups.get_value(subsystem, fromgroup, 'tasks')
for pid in pids.strip().split('\n'):
try:
cgroups.set_value(subsystem, togroup, 'tasks', pid)
except IOError as ioe: # pylint: disable=W0702
if ioe.errno not in [errno.EINVAL, errno.ESRCH]:
raise
def on_create_request(self, rsrc_id, rsrc_data):
instance_id = rsrc_id
memory_limit = rsrc_data['memory']
cpu_limit = rsrc_data['cpu']
cgrp = os.path.join('treadmill', 'apps', instance_id)
with lc.LogContext(_LOGGER, rsrc_id,
adapter_cls=lc.ContainerAdapter) as log:
log.info('Creating cgroups: %s:%s', self.SUBSYSTEMS, cgrp)
for subsystem in self.SUBSYSTEMS:
cgutils.create(subsystem, cgrp)
# blkio settings
#
cgroups.set_value('blkio', cgrp, 'blkio.weight', 100)
# memory settings
#
self._register_oom_handler(cgrp, instance_id)
cgroups.set_value('memory', cgrp, 'memory.soft_limit_in_bytes',
memory_limit)
# TODO: set hardlimit to app.memory and comment the
# reset_memory block until proper solution for
# cgroup race condition is implemented.
cgutils.set_memory_hardlimit(cgrp, memory_limit)
# expunged = cgutils.reset_memory_limit_in_bytes()
# for expunged_uniq_name in expunged:
# exp_app_dir = os.path.join(tm_env.apps_dir,
# expunged_uniq_name)
# with open(os.path.join(exp_app_dir,
# 'services', 'finished'), 'w') as f:
# f.write('oom')
# exp_cgrp = os.path.join('treadmill', 'apps',
# expunged_uniq_name)
# cgutils.kill_apps_in_cgroup('memory', exp_cgrp,
# delete_cgrp=False)
# cpu settings
#
# Calculate the value of cpu shares for the app.
#
# [treadmill/apps/cpu.shares] = <total bogomips allocated to TM>
#
# [treadmill/apps/<app>/cpu.shares] = app.cpu * BMIPS_PER_CPU
#
app_cpu_pcnt = utils.cpu_units(cpu_limit) / 100.
app_bogomips = app_cpu_pcnt * sysinfo.BMIPS_PER_CPU
app_cpu_shares = int(app_bogomips)
log.info('created in cpu:%s with %s shares', cgrp, app_cpu_shares)
cgroups.set_cpu_shares(cgrp, app_cpu_shares)
return {subsystem: cgrp for subsystem in self.SUBSYSTEMS}
def _configure_service_cgroups(cgroup, root_cgroup):
"""Configure service specific cgroups."""
from treadmill import cgroups
from treadmill import cgutils
if cgroup == '.':
cgroup = os.path.basename(os.path.realpath('.'))
if os.path.isabs(cgroup):
group = os.path.join(root_cgroup, cgroup.lstrip('/'))
else:
group = os.path.join(
cgutils.core_group_name(root_cgroup), cgroup
)
parent = os.path.dirname(group)
# create group directory
for subsystem in ['memory', 'cpu', 'cpuacct', 'cpuset', 'blkio']:
_LOGGER.info('creating : %s/%s', subsystem, group)
cgutils.create(subsystem, group)
# set memory usage limits
memlimits = ['memory.limit_in_bytes',
'memory.memsw.limit_in_bytes',
'memory.soft_limit_in_bytes']
for limit in memlimits:
parent_limit = cgroups.get_value('memory', parent, limit)
_LOGGER.info('setting %s: %s', limit, parent_limit)
cgroups.set_value('memory', group, limit, parent_limit)
# set cpu share limits
cpulimits = ['cpu.shares']
for limit in cpulimits:
parent_limit = cgroups.get_value('cpu', parent, limit)
_LOGGER.info('setting %s: %s', limit, parent_limit)
cgroups.set_value('cpu', group, limit, parent_limit)
# set cpuset
cgroups.inherit_value('cpuset', group, 'cpuset.cpus')
cgroups.inherit_value('cpuset', group, 'cpuset.mems')
# join cgroup
for subsystem in ['memory', 'cpu', 'cpuacct', 'cpuset', 'blkio']:
_LOGGER.info('joining: %s/%s', subsystem, group)
cgroups.join(subsystem, group)
def cgexec(into, subcommand):
"""execs command into given cgroup(s).
"""
cgrps = [cgrp.split(':') for cgrp in into]
for (subsystem, path) in cgrps:
pathplus = path.split('=')
if len(pathplus) == 2:
group = os.path.dirname(pathplus[0])
pseudofile = os.path.basename(pathplus[0])
value = pathplus[1]
cgroups.set_value(subsystem, group, pseudofile, value)
else:
cgutils.create(subsystem, path)
cgroups.join(subsystem, path)
if subcommand:
execargs = list(subcommand)
utils.sane_execvp(execargs[0], execargs)
def get_memory_oom_eventfd(cgrp):
"""Create, register and return a eventfd for a cgroup OOM notification.
Args:
cgrp ``str``: path to a cgroup root directory.
Returns:
``int``: eventfd(2) filedescriptor.
"""
# 1/ create an eventfd
efd = eventfd.eventfd(0, eventfd.EFD_CLOEXEC)
# 2/ open memory.oom_control
oom_control_file = cgroups.makepath('memory', cgrp, 'memory.oom_control')
with io.open(oom_control_file) as oom_control:
# 3/ write '<eventfd_fd> <oom_control_fd> to cgroup.event_control
cgroups.set_value(
'memory', cgrp, 'cgroup.event_control',
'{eventfd_fd} {oom_control_fd}'.format(
eventfd_fd=efd,
oom_control_fd=oom_control.fileno(),
))
return efd
def create_treadmill_cgroups(treadmill_core_cpu_shares,
treadmill_apps_cpu_shares,
treadmill_core_cpuset_cpus,
treadmill_apps_cpuset_cpus,
treadmill_core_memory,
treadmill_apps_memory):
"""This is the core cgroup setup. Should be applied to a cleaned env.
"""
create('cpu', 'treadmill/core')
create('cpu', 'treadmill/apps')
cgroups.set_value('cpu', 'treadmill/core',
'cpu.shares', treadmill_core_cpu_shares)
cgroups.set_value('cpu', 'treadmill/apps',
'cpu.shares', treadmill_apps_cpu_shares)
create('cpuacct', 'treadmill/core')
create('cpuacct', 'treadmill/apps')
# CPU sets
create('cpuset', 'treadmill/core')
create('cpuset', 'treadmill/apps')
cgroups.inherit_value('cpuset', 'treadmill/core', 'cpuset.mems')
cgroups.inherit_value('cpuset', 'treadmill/apps', 'cpuset.mems')
# cgroup combines duplicate cores automatically
cgroups.set_value(
'cpuset', 'treadmill/core', 'cpuset.cpus',
treadmill_core_cpuset_cpus
)
cgroups.set_value(
'cpuset', 'treadmill/apps', 'cpuset.cpus',
treadmill_apps_cpuset_cpus
)
# Memory
create('memory', 'treadmill/core')
create('memory', 'treadmill/apps')
set_memory_hardlimit('treadmill/core', treadmill_core_memory)
cgroups.set_value('memory', 'treadmill/core',
'memory.soft_limit_in_bytes', treadmill_core_memory)
set_memory_hardlimit('treadmill/apps', treadmill_apps_memory)
cgroups.set_value('memory', 'treadmill/apps',
'memory.soft_limit_in_bytes', treadmill_apps_memory)
def _raise_memory_hardlimit(cgrp, limit):
"""Raise the cgroup memory hardlimit."""
cgroups.set_value('memory', cgrp,
'memory.memsw.limit_in_bytes', limit)
cgroups.set_value('memory', cgrp,
'memory.limit_in_bytes', limit)
def set_cpu_shares(cgrp, shares):
"""set cpu shares"""
return cgroups.set_value('cpu', cgrp, 'cpu.shares', shares)
def _lower_memory_hardlimit(cgrp, limit):
"""Lower the cgroup memory hardlimit."""
cgroups.set_value('memory', cgrp,
'memory.limit_in_bytes', limit)
cgroups.set_value('memory', cgrp,
'memory.memsw.limit_in_bytes', limit)
def memory_force_empty(cgrp, value=1):
""" set memory force_empty value """
cgroups.set_value('memory', cgrp, 'memory.force_empty',
value)
def memory_charge_immigrate(cgrp, value=1):
""" set memory move_charge_at_immigrate value """
cgroups.set_value('memory', cgrp, 'memory.move_charge_at_immigrate',
value)
def on_create_request(self, rsrc_id, rsrc_data):
app_unique_name = rsrc_id
size = rsrc_data['size']
read_bps = self._default_read_bps
write_bps = self._default_write_bps
read_iops = self._default_read_iops
write_iops = self._default_write_iops
with lc.LogContext(_LOGGER, rsrc_id,
adapter_cls=lc.ContainerAdapter) as log:
log.info('Processing request')
size_in_bytes = utils.size_to_bytes(size)
uniqueid = _uniqueid(app_unique_name)
# Create the logical volume
existing_volume = uniqueid in self._volumes
if not existing_volume:
needed = math.ceil(size_in_bytes /
self._vg_status['extent_size'])
if needed > self._vg_status['extent_free']:
# If we do not have enough space, delay the creation until
# another volume is deleted.
log.info(
'Delaying request %r until %d extents are free.'
' Current volumes: %r', rsrc_id, needed, self._volumes)
self._pending.append(rsrc_id)
return None
lvm.lvcreate(
volume=uniqueid,
group=self._vg_name,
size_in_bytes=size_in_bytes,
)
# We just created a volume, refresh cached status from LVM
self._vg_status = localdiskutils.refresh_vg_status(
self._vg_name)
lv_info = lvm.lvdisplay(volume=uniqueid, group=self._vg_name)
# Configure block device using cgroups (this is idempotent)
# FIXME(boysson): The unique id <-> cgroup relation should be
# captured in the cgroup module.
cgrp = os.path.join('treadmill', 'apps', app_unique_name)
cgutils.create('blkio', cgrp)
major, minor = lv_info['dev_major'], lv_info['dev_minor']
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.write_bps_device',
'{major}:{minor} {bps}'.format(
major=major,
minor=minor,
bps=utils.size_to_bytes(write_bps),
))
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.read_bps_device',
'{major}:{minor} {bps}'.format(
major=major,
minor=minor,
bps=utils.size_to_bytes(read_bps),
))
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.write_iops_device',
'{major}:{minor} {iops}'.format(major=major,
minor=minor,
iops=write_iops))
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.read_iops_device',
'{major}:{minor} {iops}'.format(major=major,
minor=minor,
iops=read_iops))
volume_data = {
k: lv_info[k]
for k in
['name', 'block_dev', 'dev_major', 'dev_minor', 'extent_size']
}
# Record existence of the volume.
self._volumes[lv_info['name']] = volume_data
return volume_data
def create_treadmill_cgroups(core_cpu_shares, apps_cpu_shares,
core_cpuset_cpus, apps_cpuset_cpus, core_memory,
apps_memory, cgroup_prefix):
"""This is the core cgroup setup. Should be applied to a cleaned env.
"""
# generate core/apps group name: treadmill, treadmill.slice etc
core_group = core_group_name(cgroup_prefix)
apps_group = apps_group_name(cgroup_prefix)
# CPU and CPU Accounting (typically joined).
create('cpu', core_group)
create('cpu', apps_group)
create('cpuacct', core_group)
create('cpuacct', apps_group)
if core_cpu_shares is not None:
cgroups.set_value('cpu', core_group, 'cpu.shares', core_cpu_shares)
cgroups.set_value('cpu', apps_group, 'cpu.shares', apps_cpu_shares)
# CPU sets
create('cpuset', core_group)
create('cpuset', apps_group)
cgroups.inherit_value('cpuset', core_group, 'cpuset.mems')
cgroups.inherit_value('cpuset', apps_group, 'cpuset.mems')
# cgroup combines duplicate cores automatically
if core_cpuset_cpus is not None:
cgroups.set_value('cpuset', core_group, 'cpuset.cpus',
core_cpuset_cpus)
cgroups.set_value('cpuset', apps_group, 'cpuset.cpus',
apps_cpuset_cpus)
else:
cgroups.inherit_value('cpuset', core_group, 'cpuset.cpus')
cgroups.inherit_value('cpuset', apps_group, 'cpuset.cpus')
# Memory
create('memory', core_group)
create('memory', apps_group)
if core_memory is not None:
set_memory_hardlimit(core_group, core_memory)
cgroups.set_value('memory', core_group, 'memory.soft_limit_in_bytes',
core_memory)
set_memory_hardlimit(apps_group, apps_memory)
cgroups.set_value('memory', apps_group, 'memory.soft_limit_in_bytes',
apps_memory)
def create_treadmill_cgroups(core_cpu_shares, apps_cpu_shares,
core_cpuset_cpus, apps_cpuset_cpus, core_memory,
apps_memory):
"""This is the core cgroup setup. Should be applied to a cleaned env.
"""
# CPU and CPU Accounting (typically joined).
create('cpu', 'treadmill/core')
create('cpu', 'treadmill/apps')
create('cpuacct', 'treadmill/core')
create('cpuacct', 'treadmill/apps')
if core_cpu_shares is not None:
cgroups.set_value('cpu', 'treadmill/core', 'cpu.shares',
core_cpu_shares)
cgroups.set_value('cpu', 'treadmill/apps', 'cpu.shares',
apps_cpu_shares)
# CPU sets
create('cpuset', 'treadmill/core')
create('cpuset', 'treadmill/apps')
cgroups.inherit_value('cpuset', 'treadmill/core', 'cpuset.mems')
cgroups.inherit_value('cpuset', 'treadmill/apps', 'cpuset.mems')
# cgroup combines duplicate cores automatically
if core_cpuset_cpus is not None:
cgroups.set_value('cpuset', 'treadmill/core', 'cpuset.cpus',
core_cpuset_cpus)
cgroups.set_value('cpuset', 'treadmill/apps', 'cpuset.cpus',
apps_cpuset_cpus)
# Memory
create('memory', 'treadmill/core')
create('memory', 'treadmill/apps')
if core_memory is not None:
set_memory_hardlimit('treadmill/core', core_memory)
cgroups.set_value('memory', 'treadmill/core',
'memory.soft_limit_in_bytes', core_memory)
set_memory_hardlimit('treadmill/apps', apps_memory)
cgroups.set_value('memory', 'treadmill/apps',
'memory.soft_limit_in_bytes', apps_memory)
def create_treadmill_cgroups(system_cpu_shares, treadmill_cpu_shares,
treadmill_core_cpu_shares,
treadmill_apps_cpu_shares, treadmill_cpu_cores,
treadmill_mem, treadmill_core_mem):
"""This is the core cgroup setup. Should be applied to a cleaned env."""
create('cpu', 'system')
create('cpu', 'treadmill')
create('cpu', 'treadmill/core')
create('cpu', 'treadmill/apps')
cgroups.set_value('cpu', 'treadmill', 'cpu.shares', treadmill_cpu_shares)
cgroups.set_value('cpu', 'system', 'cpu.shares', system_cpu_shares)
cgroups.set_value('cpu', 'treadmill/core', 'cpu.shares',
treadmill_core_cpu_shares)
cgroups.set_value('cpu', 'treadmill/apps', 'cpu.shares',
treadmill_apps_cpu_shares)
create('cpuacct', 'system')
create('cpuacct', 'treadmill')
create('cpuacct', 'treadmill/core')
create('cpuacct', 'treadmill/apps')
create('cpuset', 'system')
create('cpuset', 'treadmill')
mems = cgroups.get_value('cpuset', '', 'cpuset.mems')
cgroups.set_value('cpuset', 'system', 'cpuset.mems', mems)
cgroups.set_value('cpuset', 'treadmill', 'cpuset.mems', mems)
cores_max = sysinfo.cpu_count() - 1
if treadmill_cpu_cores > cores_max:
raise exc.TreadmillError('Not enough cpu cores.')
if treadmill_cpu_cores > 0:
cgroups.set_value('cpuset', 'treadmill', 'cpuset.cpus',
'%d-%d' % (0, treadmill_cpu_cores - 1))
cgroups.set_value('cpuset', 'system', 'cpuset.cpus',
'%d-%d' % (treadmill_cpu_cores, cores_max))
else:
cgroups.set_value('cpuset', 'treadmill', 'cpuset.cpus',
'%d-%d' % (0, cores_max))
cgroups.set_value('cpuset', 'system', 'cpuset.cpus',
'%d-%d' % (0, cores_max))
create('memory', 'system')
create('memory', 'treadmill')
if cgroups.get_value('memory', 'treadmill', 'memory.use_hierarchy') != 1:
cgroups.set_value('memory', 'treadmill', 'memory.use_hierarchy', '1')
set_memory_hardlimit('treadmill', treadmill_mem)
oom_value = 'oom_kill_disable 0\nunder_oom 0\n'
if cgroups.get_data('memory', 'treadmill',
'memory.oom_control') != oom_value:
cgroups.set_value('memory', 'treadmill', 'memory.oom_control', '0')
create('memory', 'treadmill/core')
create('memory', 'treadmill/apps')
set_memory_hardlimit('treadmill/core', treadmill_core_mem)
cgroups.set_value('memory', 'treadmill/core', 'memory.soft_limit_in_bytes',
treadmill_core_mem)
# It is possible to use qualifiers in the input, for calculation of the
# difference, get memory limits in bytes as defined in cgroup.
total_mem_bytes = cgroups.get_value('memory', 'treadmill',
'memory.limit_in_bytes')
core_mem_bytes = cgroups.get_value('memory', 'treadmill/core',
'memory.limit_in_bytes')
apps_mem_bytes = (total_mem_bytes - core_mem_bytes)
set_memory_hardlimit('treadmill/apps', apps_mem_bytes)