def _check_capacity(cell, allocation, rsrc): """Check that there is enough free space for the allocation. """ try: old = _admin_cell_alloc().get([cell, allocation]) if old['partition'] != rsrc['partition']: old = {'cpu': '0%', 'memory': '0G', 'disk': '0G'} except ldap_exceptions.LDAPNoSuchObjectResult: old = {'cpu': '0%', 'memory': '0G', 'disk': '0G'} free = _partition_free(rsrc['partition'], cell) if (free['cpu'] + utils.cpu_units(old['cpu']) < utils.cpu_units( rsrc['cpu'])): raise exc.InvalidInputError(__name__, 'Not enough cpu capacity in partition.') if (free['memory'] + utils.size_to_bytes(old['memory']) < utils.size_to_bytes(rsrc['memory'])): raise exc.InvalidInputError( __name__, 'Not enough memory capacity in partition.') if (free['disk'] + utils.size_to_bytes(old['disk']) < utils.size_to_bytes( rsrc['disk'])): raise exc.InvalidInputError(__name__, 'Not enough disk capacity in partition.')
def _calc_free_traits(limits, allocs, old_id): """Calculate free capacity in partition by trait. Note: allocation with old_id is not counted. """ free = {} for limit in limits: free[limit['trait']] = { 'cpu': utils.cpu_units(limit['cpu']), 'disk': utils.size_to_bytes(limit['disk']), 'memory': utils.size_to_bytes(limit['memory']), } for alloc in allocs: # skip allocation with old_id if alloc['_id'] == old_id: continue for trait in alloc['traits']: if trait in free: free[trait]['cpu'] -= utils.cpu_units(alloc['cpu']) free[trait]['disk'] -= utils.size_to_bytes(alloc['cpu']) free[trait]['memory'] -= utils.size_to_bytes(alloc['cpu']) return free
def cginit(mem, mem_core, cpu, cpu_cores): """Initialize core and system cgroups.""" if cpu_cores > 0: tm_cpu_shares = sysinfo.bogomips_linux(range(0, cpu_cores)) tm_core_cpu_shares = int(tm_cpu_shares * 0.01) tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares total_cores = sysinfo.cpu_count() system_cores = range(cpu_cores, total_cores) system_cpu_shares = sysinfo.bogomips_linux(system_cores) _LOGGER.info( 'Configuring CPU limits: ' 'treadmill cores: %d, ' 'treadmill: %d, ' 'treadmill core: %d, ' 'treadmill apps: %d', cpu_cores, tm_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares) else: total_cores = sysinfo.cpu_count() total_cpu_shares = sysinfo.bogomips_linux(range(0, total_cores)) tm_cpu_shares = int(total_cpu_shares * utils.cpu_units(cpu) / 100.0) system_cpu_shares = int(total_cpu_shares - tm_cpu_shares) tm_core_cpu_shares = int(tm_cpu_shares * 0.01) tm_apps_cpu_shares = tm_cpu_shares - tm_core_cpu_shares _LOGGER.info( 'Configuring CPU limits: ' 'total: %d, ' 'treadmill: %d, ' 'system: %d, ' 'treadmill core: %d, ' 'treadmill apps: %d', total_cpu_shares, tm_cpu_shares, system_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares) tm_mem = utils.size_to_bytes(mem) tm_core_mem = utils.size_to_bytes(mem_core) total_physical_mem = sysinfo.mem_info().total * 1024 if tm_mem <= 0: # For readability, instead of + tm_mem (negative). real_tm_mem = total_physical_mem - abs(tm_mem) else: real_tm_mem = tm_mem _LOGGER.info( 'Configuring memory limits: ' 'total: %s, ' 'treadmill total: %s, ' 'treadmill core: %s', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(real_tm_mem, 'B'), utils.bytes_to_readable(tm_core_mem, 'B')) cgutils.create_treadmill_cgroups(system_cpu_shares, tm_cpu_shares, tm_core_cpu_shares, tm_apps_cpu_shares, cpu_cores, real_tm_mem, tm_core_mem)
def _check_limit(limit, request, extra_info=''): """Check capacity limit for reqested allocation. """ if utils.cpu_units(request['cpu']) > limit['cpu']: raise exc.InvalidInputError( __name__, 'Not enough cpu capacity in partition.' + extra_info) if utils.size_to_bytes(request['disk']) > limit['disk']: raise exc.InvalidInputError( __name__, 'Not enough disk capacity in partition.' + extra_info) if utils.size_to_bytes(request['memory']) > limit['memory']: raise exc.InvalidInputError( __name__, 'Not enough memory capacity in partition.' + extra_info)
def convert_disk_size_to_int(ctx, param, value): """Convert friendly cli option to int """ del ctx del param size_in_bytes = utils.size_to_bytes(value) return size_in_bytes // 1024 // 1024 // 1024
def check_available_volume(): """Check if we have enough space for benchmark. """ vg_status = localdiskutils.refresh_vg_status(vg_name) available_volume = vg_status['extent_size'] * \ vg_status['extent_free'] return available_volume > utils.size_to_bytes(total_volume)
def _get_cgroup_memory(treadmill_core_memory_limit): """get cgroup memory parameter for treadmill core/apps group """ total_physical_mem = sysinfo.mem_info().total * 1024 total_treadmill_mem = cgutils.get_memory_limit('treadmill') if total_treadmill_mem > total_physical_mem: _LOGGER.warning( 'memory limit for treadmill group > physical, use physical %s', utils.bytes_to_readable(total_treadmill_mem, 'B'), ) total_treadmill_mem = total_physical_mem # calculate memory allocation if treadmill_core_memory_limit is not None: core_memory = utils.size_to_bytes(treadmill_core_memory_limit) treadmill_memory = total_treadmill_mem apps_memory = treadmill_memory - core_memory else: core_memory = apps_memory = None _LOGGER.info( 'Configuring memory limits: ' 'phycial total: %s, ' 'treadmill total: %s, ' 'treadmill core: %s, ' 'treadmill apps: %s', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(total_treadmill_mem, 'B'), utils.bytes_to_readable(core_memory, 'B'), utils.bytes_to_readable(apps_memory, 'B')) return (core_memory, apps_memory)
def setup_benchmark_env(): """Prepare environment for benchmark. """ lvm.lvcreate(volume=benchmark_lv, group=vg_name, size_in_bytes=utils.size_to_bytes(total_volume)) fs_linux.blk_fs_create(device) fs_linux.mount_filesystem(device, base_path, fs_type='ext4')
def test_size_to_bytes(self): """Test conversion of units to bytes.""" self.assertEquals(10, utils.size_to_bytes(10)) self.assertEquals(-10, utils.size_to_bytes(-10)) self.assertEquals(10, utils.size_to_bytes('10')) self.assertEquals(-10, utils.size_to_bytes('-10')) self.assertEquals(10 * 1024, utils.size_to_bytes('10K')) self.assertEquals(-10 * 1024, utils.size_to_bytes('-10K')) self.assertEquals(-10 * 1024 * 1024, utils.size_to_bytes('-10M'))
def _partition_free(partition, cell): """Calculate free capacity for given partition.""" try: part_obj = _admin_partition().get([partition, cell]) except ldap3.LDAPNoSuchObjectResult: # pretend partition has zero capacity part_obj = {'cpu': '0%', 'memory': '0G', 'disk': '0G'} allocs = _admin_cell_alloc().list({'cell': cell, 'partition': partition}) cpu = utils.cpu_units(part_obj['cpu']) memory = utils.size_to_bytes(part_obj['memory']) disk = utils.size_to_bytes(part_obj['disk']) for alloc in allocs: cpu -= utils.cpu_units(alloc['cpu']) memory -= utils.size_to_bytes(alloc['memory']) disk -= utils.size_to_bytes(alloc['disk']) return {'cpu': cpu, 'memory': memory, 'disk': disk}
def _calc_free(limit, allocs, old_id): """Calculate free capacity in partition. Note: allocation with old_id is not counted. """ free = { 'cpu': utils.cpu_units(limit['cpu']), 'disk': utils.size_to_bytes(limit['disk']), 'memory': utils.size_to_bytes(limit['memory']), } for alloc in allocs: # skip allocation with old_id if alloc['_id'] == old_id: continue free['cpu'] -= utils.cpu_units(alloc['cpu']) free['disk'] -= utils.size_to_bytes(alloc['disk']) free['memory'] -= utils.size_to_bytes(alloc['memory']) return free
def refresh_vg_status(group): """Query LVM for the current volume group status. """ vg_info = lvm.vgdisplay(group=group) status = { 'name': vg_info['name'], 'extent_size': utils.size_to_bytes('{kb}k'.format(kb=vg_info['extent_size'])), 'extent_free': vg_info['extent_free'], 'extent_nb': vg_info['extent_nb'], } return status
def create_image(img_name, img_location, img_size): """Create a sparse file of the appropriate size. """ fs.mkdir_safe(img_location) filename = os.path.join(img_location, img_name) retries = 10 while retries > 0: retries -= 1 try: stats = os.stat(filename) os.unlink(filename) _LOGGER.info('Disk image found and unlinked: %r; stat: %r', filename, stats) except OSError as err: if err.errno == errno.ENOENT: pass else: raise available_size = sysinfo.disk_usage(img_location) img_size_bytes = utils.size_to_bytes(img_size) if img_size_bytes <= 0: real_img_size = available_size.free - abs(img_size_bytes) else: real_img_size = img_size_bytes if (real_img_size < TREADMILL_MIN_VG_SIZE or available_size.free < real_img_size + TREADMILL_MIN_RESERVE_SIZE): raise exc.NodeSetupError('Not enough free disk space') if fs.create_excl(filename, real_img_size): break if retries == 0: raise exc.NodeSetupError('Something is messing with ' 'disk image creation')
def _cgroup_init(treadmill_core_cpu_shares, treadmill_core_cpuset_cpus, treadmill_apps_cpuset_cpus, treadmill_core_memory_limit): # TODO: Refactor this whole function: # * It should be mostly folded into cgutils # * There is no need to read sysinfo for CPUs, use cpuset cgroup. # * There is no need to read bogomips for relative weight core/apps. # * There is no need to read physical memory here, use cgroups. # calculate CPU shares allocation total_cores = sysinfo.cpu_count() total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores)) if treadmill_core_cpu_shares is not None: core_cpu_shares = int( total_cpu_shares * utils.cpu_units(treadmill_core_cpu_shares) / 100.0 ) apps_cpu_shares = total_cpu_shares - core_cpu_shares else: core_cpu_shares = apps_cpu_shares = None _LOGGER.info( 'Configuring CPU shares: ' 'total: %d, ' 'treadmill core: %r, ' 'treadmill apps: %r', total_cpu_shares, core_cpu_shares, apps_cpu_shares ) # calculate memory allocation total_physical_mem = sysinfo.mem_info().total * 1024 if treadmill_core_memory_limit is not None: core_memory = utils.size_to_bytes(treadmill_core_memory_limit) # XXX: tm_mem = utils.size_to_bytes(treadmill_mem) # XXX: if tm_mem <= 0: # XXX: real_tm_mem = total_physical_mem - abs(tm_mem) # XXX: else: # XXX: real_tm_mem = tm_mem treadmill_memory = total_physical_mem apps_memory = treadmill_memory - core_memory else: core_memory = apps_memory = None _LOGGER.info( 'Configuring memory limits: ' 'total: %s, ' 'treadmill core: %r, ' 'treadmill apps: %r', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(core_memory, 'B'), utils.bytes_to_readable(apps_memory, 'B') ) # calculate cpuset cores allocation if treadmill_core_cpuset_cpus is not None: core_cpuset_cpus = _parse_cpuset_cpus( treadmill_core_cpuset_cpus ) # TODO: Calculate from apps as treadmill - core apps_cpuset_cpus = _parse_cpuset_cpus( treadmill_apps_cpuset_cpus ) else: core_cpuset_cpus = apps_cpuset_cpus = None _LOGGER.info( 'Configuring cpuset cores: ' 'treadmill cpuset: %r, ' 'treadmill core cpuset: %r, ' 'treadmill app cpuset: %r', 'TBD', core_cpuset_cpus, apps_cpuset_cpus ) cgutils.create_treadmill_cgroups( core_cpu_shares, apps_cpu_shares, core_cpuset_cpus, apps_cpuset_cpus, core_memory, apps_memory )
from treadmill import lvm from treadmill import subproc from treadmill import sysinfo from treadmill import utils _LOGGER = logging.getLogger(__name__) #: Name of the Treadmill LVM volume group TREADMILL_VG = 'treadmill' #: Name of the Treadmill loopback image file TREADMILL_IMG = 'treadmill.img' #: Minimum size for the Treadmill volume group. If we can't use this much, the #: server node start will fail TREADMILL_MIN_VG_SIZE = utils.size_to_bytes('100M') #: Minimum free disk space to leave for the OS TREADMILL_MIN_RESERVE_SIZE = utils.size_to_bytes('100M') #: Number of loop devices to initialize TREADMILL_LOOPDEV_NB = 8 def refresh_vg_status(group): """Query LVM for the current volume group status. """ vg_info = lvm.vgdisplay(group=group) status = { 'name': vg_info['name'],
import errno import hashlib import io import logging import os import tempfile import kazoo.exceptions import six from treadmill import fs from treadmill import utils from treadmill import zkutils from treadmill import zknamespace as z _ZK_DATA_SIZE_LIMIT = utils.size_to_bytes('1M') _LOGGER = logging.getLogger(__name__) class ZkCachedEntry( collections.namedtuple('_ZkCachedEntry', ['fname', 'chksum', 'ctime'])): """Entry in the local data cache. """ pass class ZkDataEntry( collections.namedtuple('_ZkDataEntry', ['zname', 'chksum', 'seq'])): """Entry in the Zookeeper store.
def on_create_request(self, rsrc_id, rsrc_data): app_unique_name = rsrc_id size = rsrc_data['size'] read_bps = self._default_read_bps write_bps = self._default_write_bps read_iops = self._default_read_iops write_iops = self._default_write_iops with lc.LogContext(_LOGGER, rsrc_id, adapter_cls=lc.ContainerAdapter) as log: log.info('Processing request') size_in_bytes = utils.size_to_bytes(size) uniqueid = _uniqueid(app_unique_name) # Create the logical volume existing_volume = uniqueid in self._volumes if not existing_volume: needed = math.ceil(size_in_bytes / self._vg_status['extent_size']) if needed > self._vg_status['extent_free']: # If we do not have enough space, delay the creation until # another volume is deleted. log.info( 'Delaying request %r until %d extents are free.' ' Current volumes: %r', rsrc_id, needed, self._volumes) self._pending.append(rsrc_id) return None lvm.lvcreate( volume=uniqueid, group=self._vg_name, size_in_bytes=size_in_bytes, ) # We just created a volume, refresh cached status from LVM self._vg_status = localdiskutils.refresh_vg_status( self._vg_name) lv_info = lvm.lvdisplay(volume=uniqueid, group=self._vg_name) # Configure block device using cgroups (this is idempotent) # FIXME(boysson): The unique id <-> cgroup relation should be # captured in the cgroup module. cgrp = os.path.join('treadmill', 'apps', app_unique_name) cgutils.create('blkio', cgrp) major, minor = lv_info['dev_major'], lv_info['dev_minor'] cgroups.set_value( 'blkio', cgrp, 'blkio.throttle.write_bps_device', '{major}:{minor} {bps}'.format( major=major, minor=minor, bps=utils.size_to_bytes(write_bps), )) cgroups.set_value( 'blkio', cgrp, 'blkio.throttle.read_bps_device', '{major}:{minor} {bps}'.format( major=major, minor=minor, bps=utils.size_to_bytes(read_bps), )) cgroups.set_value( 'blkio', cgrp, 'blkio.throttle.write_iops_device', '{major}:{minor} {iops}'.format(major=major, minor=minor, iops=write_iops)) cgroups.set_value( 'blkio', cgrp, 'blkio.throttle.read_iops_device', '{major}:{minor} {iops}'.format(major=major, minor=minor, iops=read_iops)) volume_data = { k: lv_info[k] for k in ['name', 'block_dev', 'dev_major', 'dev_minor', 'extent_size'] } # Record existence of the volume. self._volumes[lv_info['name']] = volume_data return volume_data
import six from treadmill import appcfg from treadmill import exc from treadmill import fs from treadmill import utils from treadmill import plugin_manager from treadmill.appcfg import abort as app_abort from treadmill.appcfg import manifest as app_manifest STATE_JSON = 'state.json' _LOGGER = logging.getLogger(__name__) _ARCHIVE_LIMIT = utils.size_to_bytes('1G') _RUNTIME_NAMESPACE = 'treadmill.runtime' if os.name == 'posix': # Disable C0413: should be placed at the top of the module. from treadmill import iptables # pylint: disable=c0413 PORT_SPAN = iptables.PORT_SPAN PROD_PORT_LOW = iptables.PROD_PORT_LOW PROD_PORT_HIGH = iptables.PROD_PORT_HIGH NONPROD_PORT_LOW = iptables.NONPROD_PORT_LOW NONPROD_PORT_HIGH = iptables.NONPROD_PORT_HIGH else: PORT_SPAN = 8192 PROD_PORT_LOW = 32768 PROD_PORT_HIGH = PROD_PORT_LOW + PORT_SPAN - 1 NONPROD_PORT_LOW = PROD_PORT_LOW + PORT_SPAN
def _cgroup_init(treadmill_core_cpu_shares, treadmill_core_cpuset_cpus, treadmill_apps_cpuset_cpus, treadmill_core_memory_limit): # calculate CPU shares allocation total_cores = sysinfo.cpu_count() total_cpu_shares = sysinfo.bogomips(six.moves.range(total_cores)) core_cpu_shares = int( total_cpu_shares * utils.cpu_units(treadmill_core_cpu_shares) / 100.0 ) apps_cpu_shares = total_cpu_shares - core_cpu_shares _LOGGER.info( 'Configuring CPU shares: ' 'total: %d, ' 'treadmill core: %d, ' 'treadmill apps: %d', total_cpu_shares, core_cpu_shares, apps_cpu_shares ) # calculate memory allocation core_memory = utils.size_to_bytes(treadmill_core_memory_limit) total_physical_mem = sysinfo.mem_info().total * 1024 # XXX: tm_mem = utils.size_to_bytes(treadmill_mem) # XXX: if tm_mem <= 0: # XXX: real_tm_mem = total_physical_mem - abs(tm_mem) # XXX: else: # XXX: real_tm_mem = tm_mem treadmill_memory = total_physical_mem apps_memory = treadmill_memory - core_memory _LOGGER.info( 'Configuring memory limits: ' 'total: %s, ' 'treadmill core: %s, ' 'treadmill apps: %s', utils.bytes_to_readable(total_physical_mem, 'B'), utils.bytes_to_readable(core_memory, 'B'), utils.bytes_to_readable(apps_memory, 'B') ) # calculate cpuset cores allocation core_cpuset_cpus = _parse_cpuset_cpus( treadmill_core_cpuset_cpus ) apps_cpuset_cpus = _parse_cpuset_cpus( treadmill_apps_cpuset_cpus ) _LOGGER.info( 'Configuring cpuset cores: ' 'treadmill core cpuset: %s, ' 'treadmill app cpuset: %s', core_cpuset_cpus, apps_cpuset_cpus ) cgutils.create_treadmill_cgroups( core_cpu_shares, apps_cpu_shares, core_cpuset_cpus, apps_cpuset_cpus, core_memory, apps_memory )