def __init__(self):
self.logger = logging.getLogger("vcmmd.ldmgr.policy")
self.host = Host() # Singleton object with host related data
self.controllers = set()
self.low_memory_callbacks = set()
self.__ve_data = {} # Dictionary of all managed VEs to their policy data
self.__ve_data_lock = Lock()
self.counts = {}
def __init__(self):
self.logger = logging.getLogger('vcmmd.ldmgr')
self._registered_ves = {} # str -> VE
self._registered_ves_lock = threading.Lock()
self._host = Host()
cfg = VCMMDConfig()
thn = cfg.get_num('LoadManager.ThreadsNum', 5)
self._req_queue = RQueue(maxsize=25)
self._workers = [threading.Thread(target=self._worker_thread_fn) for _ in range(thn)]
[w.start() for w in self._workers]
# Load a policy
policy_name = cfg.get_str('LoadManager.Policy', self.FALLBACK_POLICY)
policy_name = self._load_alias(policy_name)
self._load_policy(policy_name)
class Policy(object):
"""Load manager policy interface.
"""
__metaclass__ = ABCMeta
MEM_PRES_PATH = "/sys/fs/cgroup/memory/memory.pressure_level"
EVENT_CONTR_PATH = "/sys/fs/cgroup/memory/cgroup.event_control"
PRESSURE_LEVEL = "medium"
def __init__(self):
self.logger = logging.getLogger("vcmmd.ldmgr.policy")
self.host = Host() # Singleton object with host related data
self.controllers = set()
self.low_memory_callbacks = set()
self.__ve_data = {} # Dictionary of all managed VEs to their policy data
self.__ve_data_lock = Lock()
self.counts = {}
def get_name(self):
return self.__class__.__name__
def get_policy_data(self, t):
with self.__ve_data_lock:
return list(self.__ve_data.get(t, {}).itervalues())
def rm_policy_data(self, t, ve):
with self.__ve_data_lock:
self.__ve_data.get(t, {}).pop(ve, None)
def set_policy_data(self, ve, data):
with self.__ve_data_lock:
t = type(data)
if t not in self.__ve_data:
self.__ve_data[t] = {}
self.__ve_data[t][ve] = data
def ve_activated(self, ve):
"""Called right after a VE gets activated.
"""
ve.set_mem(ve.config.limit, ve.mem_min)
def ve_deactivated(self, ve):
"""Called right after a VE gets deactivated.
"""
pass
def ve_registered(self, ve):
"""Called right after a VE gets activated.
"""
if ve.VE_TYPE == VE_TYPE_CT:
ve.set_mem(ve.config.limit, ve.mem_min)
def ve_unregistered(self, ve):
"""Called right after a VE gets deactivated.
"""
pass
def ve_config_updated(self, ve):
"""Called right after a VE's configuration update.
"""
ve.set_mem(ve.config.limit, ve.mem_min)
def sched_req(self):
ret = []
for ctrl in self.controllers:
ret.append(ctrl())
if self.low_memory_callbacks:
ret.append(Request(self.low_memory_watchdog, blocker=True))
return ret
def report(self, j=False):
return print_dict(self.counts, j)
def shutdown(self):
self.__watchdog__run = False
def low_memory_watchdog(self):
self.counts["low_mem_events"] = 0
self.__watchdog__run = True
efd = eventfd(0, 0)
mp = open(self.MEM_PRES_PATH)
with open(self.EVENT_CONTR_PATH, "w") as cgc:
cgc.write("%d %d %s" % (efd, mp.fileno(), self.PRESSURE_LEVEL))
p = epoll()
p.register(efd, EPOLLIN)
self.host.log_info('"Low memory" watchdog started(pressure level=%r).' % self.PRESSURE_LEVEL)
err = "shutdown event"
while self.__watchdog__run:
try:
events = p.poll(2)
for fd, event in events:
if event & EPOLLIN:
# In an eventfd, there are always 8 bytes
ret = os.read(efd, 64 / 8)
num = struct.unpack("Q", ret)
break
else:
continue
except (ValueError, OSError, IOError) as err:
break
self.host.log_debug('"Low memory" notification received: %d' % num)
for callback in self.low_memory_callbacks:
callback()
self.counts["low_mem_events"] += 1
p.close()
os.close(efd)
self.host.log_info('"Low memory" watchdog stopped(msg="%s").' % err)
class LoadManager(object):
FALLBACK_POLICY = 'NoOpPolicy'
class ShutdownException(Exception):
"""Raise when shutdown request is received
"""
pass
def __init__(self):
self.logger = logging.getLogger('vcmmd.ldmgr')
self._registered_ves = {} # str -> VE
self._registered_ves_lock = threading.Lock()
self._host = Host()
cfg = VCMMDConfig()
thn = cfg.get_num('LoadManager.ThreadsNum', 5)
self._req_queue = RQueue(maxsize=25)
self._workers = [threading.Thread(target=self._worker_thread_fn) for _ in range(thn)]
[w.start() for w in self._workers]
# Load a policy
policy_name = cfg.get_str('LoadManager.Policy', self.FALLBACK_POLICY)
policy_name = self._load_alias(policy_name)
self._load_policy(policy_name)
def _load_alias(self, policy_name):
try:
alias = importlib.import_module('vcmmd.ldmgr.policies.alias')
except ImportError as err:
return policy_name
return alias.alias.get(policy_name, policy_name)
def _load_policy(self, policy_name):
try:
policy_module = importlib.import_module(
'vcmmd.ldmgr.policies.' + policy_name)
except ImportError as err:
self.logger.error("Failed to load policy '%s': %s",
policy_name, err)
# fallback on default policy
policy_name = self.FALLBACK_POLICY
policy_module = importlib.import_module(
'vcmmd.ldmgr.policies.' + policy_name)
self._policy = getattr(policy_module, policy_name)()
self.logger.info("Loaded policy '%s'", policy_name)
reqs = self._policy.sched_req()
for req in reqs:
self._queue_request(req)
def _queue_request(self, req):
try:
self._req_queue.put(req, req.is_blocker())
except QueueFull:
self.logger.error('Too many requests, ignore(%r)', len(self._workers))
def _worker_thread_fn(self):
while True:
req = self._req_queue.get()
try:
new_req = req()
except LoadManager.ShutdownException:
return
if new_req:
self._queue_request(new_req)
def _request(sync=True):
def wrap(fn):
def wrapped(*args, **kwargs):
self = args[0]
req = Request(fn, args, kwargs)
try:
self._req_queue.put_nowait(req)
except Queue.Full:
raise VCMMDError(VCMMD_ERROR_TOO_MANY_REQUESTS)
if sync:
return req.wait()
return wrapped
return wrap
@_request()
def _do_shutdown(self):
def shutdown():
raise LoadManager.ShutdownException
for w in self._workers:
self._queue_request(Request(shutdown, blocker=True))
def shutdown(self):
self._policy.shutdown()
self._do_shutdown()
[w.join() for w in self._workers]
def _check_guarantees(self, delta):
mem_min = sum(ve.mem_min for ve in self._registered_ves.itervalues())
mem_min += delta
if mem_min > self._host.ve_mem:
raise VCMMDError(VCMMD_ERROR_UNABLE_APPLY_VE_GUARANTEE)
@_request()
def register_ve(self, ve_name, ve_type, ve_config):
with self._registered_ves_lock:
if ve_name in self._registered_ves:
raise VCMMDError(VCMMD_ERROR_VE_NAME_ALREADY_IN_USE)
ve_config.complete(DefaultVEConfig)
ve = VE(ve_type, ve_name, ve_config)
self._check_guarantees(ve.mem_min)
ve.effective_limit = min(ve.config.limit, self._host.ve_mem)
self._registered_ves[ve_name] = ve
self._policy.ve_registered(ve)
self.logger.info('Registered %s (%s)', ve, ve.config)
@_request()
def activate_ve(self, ve_name):
with self._registered_ves_lock:
ve = self._registered_ves.get(ve_name)
if ve is None:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
ve.activate()
self._policy.ve_activated(ve)
# We need to set memory.low for machine.slice to infinity, otherwise
# memory.low in sub-cgroups won't have any effect. We can't do it on
# start, because machine.slice might not exist at that time (it is
# created on demand, when the first VM starts).
#
# This is safe, because there is nothing running inside machine.slice
# but VMs, each of which should have its memory.low configured
# properly.
# TODO need only once
self._host._set_slice_mem('machine', -1, verbose=False)
@_request()
def update_ve_config(self, ve_name, ve_config):
with self._registered_ves_lock:
ve = self._registered_ves.get(ve_name)
if ve is None:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
if not ve.active:
raise VCMMDError(VCMMD_ERROR_VE_NOT_ACTIVE)
ve_config.complete(ve.config)
self._check_guarantees(ve_config.mem_min - ve.config.mem_min)
ve.set_config(ve_config)
ve.effective_limit = min(ve.config.limit, self._host.ve_mem)
self._policy.ve_config_updated(ve)
@_request()
def deactivate_ve(self, ve_name):
with self._registered_ves_lock:
ve = self._registered_ves.get(ve_name)
if ve is None:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
ve.deactivate()
self._policy.ve_deactivated(ve)
@_request()
def unregister_ve(self, ve_name):
with self._registered_ves_lock:
ve = self._registered_ves.get(ve_name)
if ve is None:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
del self._registered_ves[ve.name]
self._policy.ve_unregistered(ve)
if ve.active:
self._policy.ve_deactivated(ve)
self.logger.info('Unregistered %s', ve)
def is_ve_active(self, ve_name):
try:
with self._registered_ves_lock:
return self._registered_ves[ve_name].active
except KeyError:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
def get_ve_config(self, ve_name):
with self._registered_ves_lock:
try:
return self._registered_ves[ve_name].config.as_array()
except KeyError:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
def get_all_registered_ves(self):
result = []
with self._registered_ves_lock:
for ve in self._registered_ves.itervalues():
result.append((ve.name, ve.VE_TYPE, ve.active,
ve.config.as_array()))
return result
def get_current_policy(self):
cfg = VCMMDConfig().get_str('LoadManager.Policy')
cur = self._policy.get_name()
if self._load_alias(cfg) == cur:
return cfg
else:
return cur
def get_stats(self, ve_name):
with self._registered_ves_lock:
ve = self._registered_ves.get(ve_name)
if ve is None:
raise VCMMDError(VCMMD_ERROR_VE_NOT_REGISTERED)
res = ve.stats.report().iteritems()
return res
def get_quotas(self):
with self._registered_ves_lock:
return [(ve.name, ve.target, ve.protection)
for ve in self._registered_ves.itervalues()
if ve.active and ve.target is not None]
def get_config(self, j):
return VCMMDConfig().report(j)
def get_policy_counts(self, j):
return self._policy.report(j)