def _check_capacity(cell, allocation, rsrc):
"""Check that there is enough free space for the allocation.
"""
try:
old = _admin_cell_alloc().get([cell, allocation])
if old['partition'] != rsrc['partition']:
old = {'cpu': '0%', 'memory': '0G', 'disk': '0G'}
except ldap_exceptions.LDAPNoSuchObjectResult:
old = {'cpu': '0%', 'memory': '0G', 'disk': '0G'}
free = _partition_free(rsrc['partition'], cell)
if (free['cpu'] + utils.cpu_units(old['cpu']) < utils.cpu_units(
rsrc['cpu'])):
raise exc.InvalidInputError(__name__,
'Not enough cpu capacity in partition.')
if (free['memory'] + utils.size_to_bytes(old['memory']) <
utils.size_to_bytes(rsrc['memory'])):
raise exc.InvalidInputError(
__name__, 'Not enough memory capacity in partition.')
if (free['disk'] + utils.size_to_bytes(old['disk']) < utils.size_to_bytes(
rsrc['disk'])):
raise exc.InvalidInputError(__name__,
'Not enough disk capacity in partition.')
def _calc_free_traits(limits, allocs, old_id):
"""Calculate free capacity in partition by trait.
Note: allocation with old_id is not counted.
"""
free = {}
for limit in limits:
free[limit['trait']] = {
'cpu': utils.cpu_units(limit['cpu']),
'disk': utils.size_to_bytes(limit['disk']),
'memory': utils.size_to_bytes(limit['memory']),
}
for alloc in allocs:
# skip allocation with old_id
if alloc['_id'] == old_id:
continue
for trait in alloc['traits']:
if trait in free:
free[trait]['cpu'] -= utils.cpu_units(alloc['cpu'])
free[trait]['disk'] -= utils.size_to_bytes(alloc['cpu'])
free[trait]['memory'] -= utils.size_to_bytes(alloc['cpu'])
return free
def cginit(mem, mem_core, cpu, cpu_cores):
"""Initialize core and system cgroups."""
if cpu_cores > 0:
tm_cpu_shares = sysinfo.bogomips_linux(range(0, cpu_cores))
tm_core_cpu_shares = int(tm_cpu_shares * 0.01)
tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares
total_cores = sysinfo.cpu_count()
system_cores = range(cpu_cores, total_cores)
system_cpu_shares = sysinfo.bogomips_linux(system_cores)
_LOGGER.info(
'Configuring CPU limits: '
'treadmill cores: %d, '
'treadmill: %d, '
'treadmill core: %d, '
'treadmill apps: %d', cpu_cores, tm_cpu_shares,
tm_core_cpu_shares, tm_apps_cpu_shares)
else:
total_cores = sysinfo.cpu_count()
total_cpu_shares = sysinfo.bogomips_linux(range(0, total_cores))
tm_cpu_shares = int(total_cpu_shares * utils.cpu_units(cpu) /
100.0)
system_cpu_shares = int(total_cpu_shares - tm_cpu_shares)
tm_core_cpu_shares = int(tm_cpu_shares * 0.01)
tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares
_LOGGER.info(
'Configuring CPU limits: '
'total: %d, '
'treadmill: %d, '
'system: %d, '
'treadmill core: %d, '
'treadmill apps: %d', total_cpu_shares, tm_cpu_shares,
system_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares)
tm_mem = utils.size_to_bytes(mem)
tm_core_mem = utils.size_to_bytes(mem_core)
total_physical_mem = sysinfo.mem_info().total * 1024
if tm_mem <= 0:
# For readability, instead of + tm_mem (negative).
real_tm_mem = total_physical_mem - abs(tm_mem)
else:
real_tm_mem = tm_mem
_LOGGER.info(
'Configuring memory limits: '
'total: %s, '
'treadmill total: %s, '
'treadmill core: %s',
utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(real_tm_mem, 'B'),
utils.bytes_to_readable(tm_core_mem, 'B'))
cgutils.create_treadmill_cgroups(system_cpu_shares, tm_cpu_shares,
tm_core_cpu_shares,
tm_apps_cpu_shares, cpu_cores,
real_tm_mem, tm_core_mem)
def _check_limit(limit, request, extra_info=''):
"""Check capacity limit for reqested allocation.
"""
if utils.cpu_units(request['cpu']) > limit['cpu']:
raise exc.InvalidInputError(
__name__, 'Not enough cpu capacity in partition.' + extra_info)
if utils.size_to_bytes(request['disk']) > limit['disk']:
raise exc.InvalidInputError(
__name__, 'Not enough disk capacity in partition.' + extra_info)
if utils.size_to_bytes(request['memory']) > limit['memory']:
raise exc.InvalidInputError(
__name__, 'Not enough memory capacity in partition.' + extra_info)
def convert_disk_size_to_int(ctx, param, value):
"""Convert friendly cli option to int
"""
del ctx
del param
size_in_bytes = utils.size_to_bytes(value)
return size_in_bytes // 1024 // 1024 // 1024
def check_available_volume():
"""Check if we have enough space for benchmark.
"""
vg_status = localdiskutils.refresh_vg_status(vg_name)
available_volume = vg_status['extent_size'] * \
vg_status['extent_free']
return available_volume > utils.size_to_bytes(total_volume)
def _get_cgroup_memory(treadmill_core_memory_limit):
"""get cgroup memory parameter for treadmill core/apps group
"""
total_physical_mem = sysinfo.mem_info().total * 1024
total_treadmill_mem = cgutils.get_memory_limit('treadmill')
if total_treadmill_mem > total_physical_mem:
_LOGGER.warning(
'memory limit for treadmill group > physical, use physical %s',
utils.bytes_to_readable(total_treadmill_mem, 'B'),
)
total_treadmill_mem = total_physical_mem
# calculate memory allocation
if treadmill_core_memory_limit is not None:
core_memory = utils.size_to_bytes(treadmill_core_memory_limit)
treadmill_memory = total_treadmill_mem
apps_memory = treadmill_memory - core_memory
else:
core_memory = apps_memory = None
_LOGGER.info(
'Configuring memory limits: '
'phycial total: %s, '
'treadmill total: %s, '
'treadmill core: %s, '
'treadmill apps: %s', utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(total_treadmill_mem, 'B'),
utils.bytes_to_readable(core_memory, 'B'),
utils.bytes_to_readable(apps_memory, 'B'))
return (core_memory, apps_memory)
def setup_benchmark_env():
"""Prepare environment for benchmark.
"""
lvm.lvcreate(volume=benchmark_lv,
group=vg_name,
size_in_bytes=utils.size_to_bytes(total_volume))
fs_linux.blk_fs_create(device)
fs_linux.mount_filesystem(device, base_path, fs_type='ext4')
def test_size_to_bytes(self):
"""Test conversion of units to bytes."""
self.assertEquals(10, utils.size_to_bytes(10))
self.assertEquals(-10, utils.size_to_bytes(-10))
self.assertEquals(10, utils.size_to_bytes('10'))
self.assertEquals(-10, utils.size_to_bytes('-10'))
self.assertEquals(10 * 1024, utils.size_to_bytes('10K'))
self.assertEquals(-10 * 1024, utils.size_to_bytes('-10K'))
self.assertEquals(-10 * 1024 * 1024, utils.size_to_bytes('-10M'))
def _partition_free(partition, cell):
"""Calculate free capacity for given partition."""
try:
part_obj = _admin_partition().get([partition, cell])
except ldap3.LDAPNoSuchObjectResult:
# pretend partition has zero capacity
part_obj = {'cpu': '0%', 'memory': '0G', 'disk': '0G'}
allocs = _admin_cell_alloc().list({'cell': cell, 'partition': partition})
cpu = utils.cpu_units(part_obj['cpu'])
memory = utils.size_to_bytes(part_obj['memory'])
disk = utils.size_to_bytes(part_obj['disk'])
for alloc in allocs:
cpu -= utils.cpu_units(alloc['cpu'])
memory -= utils.size_to_bytes(alloc['memory'])
disk -= utils.size_to_bytes(alloc['disk'])
return {'cpu': cpu, 'memory': memory, 'disk': disk}
def _calc_free(limit, allocs, old_id):
"""Calculate free capacity in partition.
Note: allocation with old_id is not counted.
"""
free = {
'cpu': utils.cpu_units(limit['cpu']),
'disk': utils.size_to_bytes(limit['disk']),
'memory': utils.size_to_bytes(limit['memory']),
}
for alloc in allocs:
# skip allocation with old_id
if alloc['_id'] == old_id:
continue
free['cpu'] -= utils.cpu_units(alloc['cpu'])
free['disk'] -= utils.size_to_bytes(alloc['disk'])
free['memory'] -= utils.size_to_bytes(alloc['memory'])
return free
def refresh_vg_status(group):
"""Query LVM for the current volume group status.
"""
vg_info = lvm.vgdisplay(group=group)
status = {
'name':
vg_info['name'],
'extent_size':
utils.size_to_bytes('{kb}k'.format(kb=vg_info['extent_size'])),
'extent_free':
vg_info['extent_free'],
'extent_nb':
vg_info['extent_nb'],
}
return status
def create_image(img_name, img_location, img_size):
"""Create a sparse file of the appropriate size.
"""
fs.mkdir_safe(img_location)
filename = os.path.join(img_location, img_name)
retries = 10
while retries > 0:
retries -= 1
try:
stats = os.stat(filename)
os.unlink(filename)
_LOGGER.info('Disk image found and unlinked: %r; stat: %r',
filename, stats)
except OSError as err:
if err.errno == errno.ENOENT:
pass
else:
raise
available_size = sysinfo.disk_usage(img_location)
img_size_bytes = utils.size_to_bytes(img_size)
if img_size_bytes <= 0:
real_img_size = available_size.free - abs(img_size_bytes)
else:
real_img_size = img_size_bytes
if (real_img_size < TREADMILL_MIN_VG_SIZE or available_size.free <
real_img_size + TREADMILL_MIN_RESERVE_SIZE):
raise exc.NodeSetupError('Not enough free disk space')
if fs.create_excl(filename, real_img_size):
break
if retries == 0:
raise exc.NodeSetupError('Something is messing with '
'disk image creation')
def _cgroup_init(treadmill_core_cpu_shares,
treadmill_core_cpuset_cpus,
treadmill_apps_cpuset_cpus,
treadmill_core_memory_limit):
# TODO: Refactor this whole function:
# * It should be mostly folded into cgutils
# * There is no need to read sysinfo for CPUs, use cpuset cgroup.
# * There is no need to read bogomips for relative weight core/apps.
# * There is no need to read physical memory here, use cgroups.
# calculate CPU shares allocation
total_cores = sysinfo.cpu_count()
total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores))
if treadmill_core_cpu_shares is not None:
core_cpu_shares = int(
total_cpu_shares *
utils.cpu_units(treadmill_core_cpu_shares) / 100.0
)
apps_cpu_shares = total_cpu_shares - core_cpu_shares
else:
core_cpu_shares = apps_cpu_shares = None
_LOGGER.info(
'Configuring CPU shares: '
'total: %d, '
'treadmill core: %r, '
'treadmill apps: %r',
total_cpu_shares,
core_cpu_shares,
apps_cpu_shares
)
# calculate memory allocation
total_physical_mem = sysinfo.mem_info().total * 1024
if treadmill_core_memory_limit is not None:
core_memory = utils.size_to_bytes(treadmill_core_memory_limit)
# XXX: tm_mem = utils.size_to_bytes(treadmill_mem)
# XXX: if tm_mem <= 0:
# XXX: real_tm_mem = total_physical_mem - abs(tm_mem)
# XXX: else:
# XXX: real_tm_mem = tm_mem
treadmill_memory = total_physical_mem
apps_memory = treadmill_memory - core_memory
else:
core_memory = apps_memory = None
_LOGGER.info(
'Configuring memory limits: '
'total: %s, '
'treadmill core: %r, '
'treadmill apps: %r',
utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(core_memory, 'B'),
utils.bytes_to_readable(apps_memory, 'B')
)
# calculate cpuset cores allocation
if treadmill_core_cpuset_cpus is not None:
core_cpuset_cpus = _parse_cpuset_cpus(
treadmill_core_cpuset_cpus
)
# TODO: Calculate from apps as treadmill - core
apps_cpuset_cpus = _parse_cpuset_cpus(
treadmill_apps_cpuset_cpus
)
else:
core_cpuset_cpus = apps_cpuset_cpus = None
_LOGGER.info(
'Configuring cpuset cores: '
'treadmill cpuset: %r, '
'treadmill core cpuset: %r, '
'treadmill app cpuset: %r',
'TBD',
core_cpuset_cpus,
apps_cpuset_cpus
)
cgutils.create_treadmill_cgroups(
core_cpu_shares,
apps_cpu_shares,
core_cpuset_cpus,
apps_cpuset_cpus,
core_memory,
apps_memory
)
from treadmill import lvm
from treadmill import subproc
from treadmill import sysinfo
from treadmill import utils
_LOGGER = logging.getLogger(__name__)
#: Name of the Treadmill LVM volume group
TREADMILL_VG = 'treadmill'
#: Name of the Treadmill loopback image file
TREADMILL_IMG = 'treadmill.img'
#: Minimum size for the Treadmill volume group. If we can't use this much, the
#: server node start will fail
TREADMILL_MIN_VG_SIZE = utils.size_to_bytes('100M')
#: Minimum free disk space to leave for the OS
TREADMILL_MIN_RESERVE_SIZE = utils.size_to_bytes('100M')
#: Number of loop devices to initialize
TREADMILL_LOOPDEV_NB = 8
def refresh_vg_status(group):
"""Query LVM for the current volume group status.
"""
vg_info = lvm.vgdisplay(group=group)
status = {
'name':
vg_info['name'],
import errno
import hashlib
import io
import logging
import os
import tempfile
import kazoo.exceptions
import six
from treadmill import fs
from treadmill import utils
from treadmill import zkutils
from treadmill import zknamespace as z
_ZK_DATA_SIZE_LIMIT = utils.size_to_bytes('1M')
_LOGGER = logging.getLogger(__name__)
class ZkCachedEntry(
collections.namedtuple('_ZkCachedEntry',
['fname', 'chksum', 'ctime'])):
"""Entry in the local data cache.
"""
pass
class ZkDataEntry(
collections.namedtuple('_ZkDataEntry', ['zname', 'chksum', 'seq'])):
"""Entry in the Zookeeper store.
def on_create_request(self, rsrc_id, rsrc_data):
app_unique_name = rsrc_id
size = rsrc_data['size']
read_bps = self._default_read_bps
write_bps = self._default_write_bps
read_iops = self._default_read_iops
write_iops = self._default_write_iops
with lc.LogContext(_LOGGER, rsrc_id,
adapter_cls=lc.ContainerAdapter) as log:
log.info('Processing request')
size_in_bytes = utils.size_to_bytes(size)
uniqueid = _uniqueid(app_unique_name)
# Create the logical volume
existing_volume = uniqueid in self._volumes
if not existing_volume:
needed = math.ceil(size_in_bytes /
self._vg_status['extent_size'])
if needed > self._vg_status['extent_free']:
# If we do not have enough space, delay the creation until
# another volume is deleted.
log.info(
'Delaying request %r until %d extents are free.'
' Current volumes: %r', rsrc_id, needed, self._volumes)
self._pending.append(rsrc_id)
return None
lvm.lvcreate(
volume=uniqueid,
group=self._vg_name,
size_in_bytes=size_in_bytes,
)
# We just created a volume, refresh cached status from LVM
self._vg_status = localdiskutils.refresh_vg_status(
self._vg_name)
lv_info = lvm.lvdisplay(volume=uniqueid, group=self._vg_name)
# Configure block device using cgroups (this is idempotent)
# FIXME(boysson): The unique id <-> cgroup relation should be
# captured in the cgroup module.
cgrp = os.path.join('treadmill', 'apps', app_unique_name)
cgutils.create('blkio', cgrp)
major, minor = lv_info['dev_major'], lv_info['dev_minor']
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.write_bps_device',
'{major}:{minor} {bps}'.format(
major=major,
minor=minor,
bps=utils.size_to_bytes(write_bps),
))
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.read_bps_device',
'{major}:{minor} {bps}'.format(
major=major,
minor=minor,
bps=utils.size_to_bytes(read_bps),
))
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.write_iops_device',
'{major}:{minor} {iops}'.format(major=major,
minor=minor,
iops=write_iops))
cgroups.set_value(
'blkio', cgrp, 'blkio.throttle.read_iops_device',
'{major}:{minor} {iops}'.format(major=major,
minor=minor,
iops=read_iops))
volume_data = {
k: lv_info[k]
for k in
['name', 'block_dev', 'dev_major', 'dev_minor', 'extent_size']
}
# Record existence of the volume.
self._volumes[lv_info['name']] = volume_data
return volume_data
import six
from treadmill import appcfg
from treadmill import exc
from treadmill import fs
from treadmill import utils
from treadmill import plugin_manager
from treadmill.appcfg import abort as app_abort
from treadmill.appcfg import manifest as app_manifest
STATE_JSON = 'state.json'
_LOGGER = logging.getLogger(__name__)
_ARCHIVE_LIMIT = utils.size_to_bytes('1G')
_RUNTIME_NAMESPACE = 'treadmill.runtime'
if os.name == 'posix':
# Disable C0413: should be placed at the top of the module.
from treadmill import iptables # pylint: disable=c0413
PORT_SPAN = iptables.PORT_SPAN
PROD_PORT_LOW = iptables.PROD_PORT_LOW
PROD_PORT_HIGH = iptables.PROD_PORT_HIGH
NONPROD_PORT_LOW = iptables.NONPROD_PORT_LOW
NONPROD_PORT_HIGH = iptables.NONPROD_PORT_HIGH
else:
PORT_SPAN = 8192
PROD_PORT_LOW = 32768
PROD_PORT_HIGH = PROD_PORT_LOW + PORT_SPAN - 1
NONPROD_PORT_LOW = PROD_PORT_LOW + PORT_SPAN
def _cgroup_init(treadmill_core_cpu_shares,
treadmill_core_cpuset_cpus,
treadmill_apps_cpuset_cpus,
treadmill_core_memory_limit):
# calculate CPU shares allocation
total_cores = sysinfo.cpu_count()
total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores))
core_cpu_shares = int(
total_cpu_shares *
utils.cpu_units(treadmill_core_cpu_shares) / 100.0
)
apps_cpu_shares = total_cpu_shares - core_cpu_shares
_LOGGER.info(
'Configuring CPU shares: '
'total: %d, '
'treadmill core: %d, '
'treadmill apps: %d',
total_cpu_shares,
core_cpu_shares,
apps_cpu_shares
)
# calculate memory allocation
core_memory = utils.size_to_bytes(treadmill_core_memory_limit)
total_physical_mem = sysinfo.mem_info().total * 1024
# XXX: tm_mem = utils.size_to_bytes(treadmill_mem)
# XXX: if tm_mem <= 0:
# XXX: real_tm_mem = total_physical_mem - abs(tm_mem)
# XXX: else:
# XXX: real_tm_mem = tm_mem
treadmill_memory = total_physical_mem
apps_memory = treadmill_memory - core_memory
_LOGGER.info(
'Configuring memory limits: '
'total: %s, '
'treadmill core: %s, '
'treadmill apps: %s',
utils.bytes_to_readable(total_physical_mem, 'B'),
utils.bytes_to_readable(core_memory, 'B'),
utils.bytes_to_readable(apps_memory, 'B')
)
# calculate cpuset cores allocation
core_cpuset_cpus = _parse_cpuset_cpus(
treadmill_core_cpuset_cpus
)
apps_cpuset_cpus = _parse_cpuset_cpus(
treadmill_apps_cpuset_cpus
)
_LOGGER.info(
'Configuring cpuset cores: '
'treadmill core cpuset: %s, '
'treadmill app cpuset: %s',
core_cpuset_cpus,
apps_cpuset_cpus
)
cgutils.create_treadmill_cgroups(
core_cpu_shares,
apps_cpu_shares,
core_cpuset_cpus,
apps_cpuset_cpus,
core_memory,
apps_memory
)