def __init__(self,
owner: MemoryConsumerPythonBackend,
shared_terminable: Optional[Terminable] = None):
LoggedThread.__init__(self)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.owner = owner
self.sleep_duration = owner.sleep_after_write
# set a worker own random number generator
self.random = random.Random(threading.get_ident())
def __init__(self,
config,
libvirt_iface,
shared_terminable: Terminable = None):
LoggedThread.__init__(self, daemon=True)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.config = config
self.libvirt_iface = libvirt_iface
self.guest_monitors = {}
self.guest_monitors_lock = threading.RLock()
def ping_background(ip, repeat=1, logger=None):
"""
Ping an IP and log the results in a background daemon thread
:param ip: The IP address
:param repeat: Ping repeats
:param logger: The logger to log to
:return: The thread object
"""
t = LoggedThread(name="ping %s" % ip,
target=ping_log,
args=(ip, repeat, logger),
daemon=True)
t.start()
return t
def __init__(self,
config,
libvirt_iface=None,
shared_terminable: Terminable = None):
# thread's name is important to Plotter, must be: Host-Monitor
self.interval = config.get('host-monitor', 'interval')
self.libvirt_iface = libvirt_iface
LoggedThread.__init__(self, daemon=True)
Monitor.__init__(self,
config,
monitor_source='host',
shared_terminable=shared_terminable)
self._set_ready()
def __init__(self, config: DictConfig, guest_id, libvirt_iface, shared_terminable: Terminable = None):
self.config = config
self.libvirt_iface = libvirt_iface
self.id = guest_id
self.logger = logging.getLogger(f"GuestMonitor-{guest_id}")
self.dom = self.libvirt_iface.getDomainFromID(guest_id)
self.interval = self.config.get('guest-monitor', 'interval')
self.info = self.get_guest_info()
if self.info is None:
raise ValueError(f"GuestMonitor-id:{guest_id} - failed to get information")
self.vm_name = self.info['name']
LoggedThread.__init__(self, name=self.vm_name, daemon=True)
Monitor.__init__(self, config, self.vm_name, self.vm_name, shared_terminable=shared_terminable)
self.start()
def start_background_thread(self,
target,
name_prefix=None,
args=(),
kwargs=None):
if not self.should_run:
return
name = f"{name_prefix}-{self.vm_name}"
self.logger.info("Starting background thread: %s", name)
try:
t = LoggedThread(target=target,
name=name,
args=args,
kwargs=kwargs,
daemon=True)
self.threads[name] = t
t.start()
except Exception as e:
self.logger.exception("Failed to initiated thread '%s': %s", name,
e)
def _update_memory(self, target):
""" :return: wait_time, target_memory """
if target is None:
return self.wait_timeout, None
grace_period = target.get('grace-period', None)
memory_alloc = target.get('alloc', {}).get('memory', None)
if memory_alloc is None:
self.logger.warning(
'Notification did not include memory allocation')
return self.wait_timeout, None
# First notification should indicate stable condition
if not is_valid_mem(self.memory_diff) and grace_period is None:
self.memory_diff = max(0, memory_alloc - self.available_memory)
LoggedThread(target=self.update_resource_diff,
name=f"{self.__log_name__}-resource-diff",
daemon=True,
verbose=False).start()
if not is_valid_mem(self.memory_diff):
return self.wait_timeout, None
memory_alloc -= self.memory_diff
if is_memory_close(memory_alloc, self.available_memory):
# memory_alloc = available ; continue
return self.wait_timeout, None
elif memory_alloc > self.available_memory and grace_period is not None:
# memory_alloc > available ; still have grace time
return grace_period, None
elif memory_alloc > self.available_memory:
# memory_alloc > available ; not more grace time
self.available_memory, stable_time = self.poll_stats(
memory_alloc,
self.available_memory,
timeout=self.STATS_POLL_TIMEOUT)
is_ready = is_memory_close(memory_alloc, self.available_memory)
if is_ready or stable_time + 1e-2 > self.STATS_POLL_TIMEOUT:
# It is a good time to update the memory diff
self.memory_diff = max(
0, memory_alloc + self.memory_diff - self.available_memory)
return self.wait_timeout, memory_alloc
else:
return 0.1, memory_alloc
elif grace_period is not None and grace_period > self.decrease_mem_time:
# memory_alloc < available ; still have grace time
return grace_period - self.decrease_mem_time, None
else:
# memory_alloc < available ; not more grace time
return self.wait_timeout, memory_alloc
def __init__(self,
config: DictConfig,
libvirt_iface,
host_monitor: Monitor,
guest_manager: GuestManager,
shared_terminable: Terminable = None):
LoggedThread.__init__(self, daemon=True)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.policy_lock = threading.RLock()
self.host_monitor = host_monitor
self.guest_manager = guest_manager
self.config = config
self.interval: float = max(self.config.get('policy', 'interval'), 1.)
self.grace_period: float = min(
self.config.get('policy', 'grace-period'), self.interval)
self.inquiry_timeout: float = min(
self.config.get('policy', 'inquiry-timeout'), self.interval)
self.resources = [
r.strip() for r in self.config.get('policy', 'resources')
]
self.policy = ClassImporter(allocators).get_class(
config.get('policy', 'allocator'))(self.resources)
self.controllers = []
self.properties = {
'libvirt_iface': libvirt_iface,
'host_monitor': host_monitor,
'guest_manager': guest_manager,
'config': config
}
self.policy_data_loggers = {}
self.client_executor = None
self.get_controllers()
def __init__(self,
*args,
shared_deferred_start: Optional['DeferredStartThread'] = None,
shared_terminable: Optional['Terminable'] = None,
minimal_sleep: Union[float, int, None] = None,
start_timeout=None,
**kwargs):
LoggedThread.__init__(self, *args, **kwargs)
Terminable.__init__(self,
shared_terminable=shared_terminable,
minimal_sleep=minimal_sleep)
self._start_time = None
if shared_deferred_start is None:
self._shared_deferred_object: Optional[DeferredStartThread] = None
self._start_event = threading.Event()
self._start_timeout = None
else:
self.share_start(shared_deferred_start)
if start_timeout is not None:
self._start_timeout = start_timeout
def __init__(self,
guest_server_port,
wait_timeout,
decrease_mem_time=1,
change_mem_func=None,
application_rss_func=None,
name=None,
spare_mem=0,
shared_terminable: Terminable = None):
LoggedThread.__init__(self, name=name, daemon=True)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.stats_collector = MemoryStatistics(meminfo=True, vmstat=False)
self.guest_server_port = guest_server_port
self.wait_timeout = max(1, wait_timeout)
self.decrease_mem_time = max(0, decrease_mem_time)
self.spare_mem = max(0, spare_mem)
self.guest_server_client: Optional[GuestClient] = None
self.memory_diff = InvalidMemory
self.available_memory = InvalidMemory
if change_mem_func is not None:
self.change_mem_func = change_mem_func
if application_rss_func is not None:
self.application_rss = application_rss_func
def logged_run(self) -> None:
wait_time = self.wait_timeout
while self.should_run:
target = self.request_target(wait_time)
if not self.should_run:
return
memory_stats = self.get_stats()
if not self.should_run:
return
app_rss = self.application_rss()
if not self.should_run:
return
self.available_memory, cache_and_buff, used = map(
memory_stats.get, ('available', 'cache_and_buff', 'used'))
wait_time, target_memory = self._update_memory(target)
self.logger.debug(
"[state] next=%.2f | target=%s | available=%.2f | used=%.2f | unused=%.2f | rss=%.2f | "
"cache=%.2f | diff=%.2f", wait_time, target_memory,
self.available_memory, used, self.available_memory - used,
app_rss, cache_and_buff, self.memory_diff)
if not self.should_run:
return
if target_memory is not None and target_memory > 0:
# take the minimum from the host's hint and current allocation,
# so we won't take more memory than available when memory is
# growing, and we will free memory when shrinkage is about to
# happen.
target_memory = min(target_memory, self.available_memory)
else:
target_memory = self.available_memory
try:
ret_target = self.change_mem_func(target_memory, used,
cache_and_buff, app_rss)
except Exception as ex:
self.logger.exception(
"Failed to update the application memory: %s", ex)
ret_target = None
if ret_target is not None:
LoggedThread(target=self.update_application_target,
name=f"{self.__log_name__}-app-target",
args=(ret_target, ),
daemon=True,
verbose=False).start()
def __init__(self, stream, name=None):
LoggedThread.__init__(self, name=name, daemon=True)
self.stream = stream
self.q = ThreadSafeDict()
def __init__(self, stream, name=None, log_level=logging.INFO):
LoggedThread.__init__(self, name=name, daemon=True)
self.stream = stream
self.log_level = log_level
self.line_count = 0
self.err_output = []
def run(self) -> None:
self.logger.info("Started deferred. Waiting for start event...")
self._start_event.wait(self._start_timeout)
if self.should_run:
LoggedThread.run(self)