def wait_for_server(self, interval=5, timeout=3, max_retries=24, shared_terminable: Terminable = None):
start_wait = time.time()
msg = EchoMessage()
self.logger.debug("Waiting for guest server.")
for c in range(max_retries):
if shared_terminable is not None and not shared_terminable.should_run:
break
start_attempt = time.time()
try:
self.send_receive_message(msg, timeout)
except socket.error:
pass
else:
self.logger.info("Guest server is ready...")
return
if c < max_retries-1:
sleep_time = interval - (time.time() - start_attempt)
if shared_terminable is not None:
shared_terminable.terminable_sleep(sleep_time)
else:
time.sleep(max(0., sleep_time))
if shared_terminable is None or shared_terminable.should_run:
raise Exception(f"Guest server is not ready after {max_retries} attempts "
f"({time.time() - start_wait} sec.).")
def __init__(self,
config: Union[DictConfig, dict, None] = None,
shared_terminable: Terminable = None):
Terminable.__init__(self, shared_terminable=shared_terminable)
self.logger = logging.getLogger(self.__class__.__name__)
self.config = DictConfig(DEFAULT_CONFIG, config)
# Set up a shared libvirt connection
libvirt_uri = self.config.get('main', 'libvirt-hypervisor-uri')
self.libvirt_iface = LibvirtInterface(libvirt_uri)
self.guest_manager = GuestManager(self.config,
libvirt_iface=self.libvirt_iface,
shared_terminable=shared_terminable)
self.host_monitor = HostMonitor(self.config,
libvirt_iface=self.libvirt_iface,
shared_terminable=shared_terminable)
self.host_policy = HostPolicy(self.config,
self.libvirt_iface,
self.host_monitor,
self.guest_manager,
shared_terminable=shared_terminable)
self.threads = {
self.guest_manager, self.host_policy, self.host_monitor
}
def __init__(self, is_remote=False, name=None):
LoggedObject.__init__(self, name)
Terminable.__init__(self)
self.is_remote = is_remote
self.guest_machine: Optional[GuestMachine] = None
self._running_process = set()
self._alloc_cpus = None
def terminate(self):
Terminable.terminate(self)
for p in list(self._running_process):
try:
p.kill()
except OSError as e:
self.logger.error('Failed to terminate benchmark: %s', e)
self._running_process.remove(p)
def __init__(self,
vm_name,
libvirt_uri,
username,
password=None,
master_image=None,
title=None,
swappiness=60,
use_image_cow=True,
base_mem=2**10,
max_mem=2**12,
base_vcpus=1,
max_vcpus=1,
cpu_pin=None,
shared_terminable: Terminable = None,
**_unknown):
"""
vm_name: the virtual machine name/alias
libvirt_uri: a libvirt URI
username: ssh login username
password: ssh login password
master_image: the filename of the master image to use for this machine (should reside on IMAGES_FOLDER)
title: a title for the machine for documentation purpose
cpu_pin: a list of cpus to pin this machine to
swappiness: the swappiness parameter for the guest OS in the VM
use_image_cow: use copy-on-write mechanism - instead of full copy of the master image
max_vcpus: set the maximum cpus that is allocated to this machine (will change permanently)
_unknown: other properties
"""
LoggedObject.__init__(self, vm_name)
Terminable.__init__(self, shared_terminable=shared_terminable)
if master_image is None:
master_image = "generic-master.qcow2"
self.vm_name = vm_name
self.libvirt_uri = libvirt_uri
self.username = username
self.password = password
self.master_image = master_image
self.title = title
self.swappiness = swappiness if 0 <= swappiness <= 100 else 60
self.use_image_cow = use_image_cow
self.base_mem = base_mem
self.max_mem = max_mem
self.base_vcpus = base_vcpus
self.max_vcpus = max_vcpus
self.cpu_pin = cpu_pin
self.driver = LibvirtDriver(self.libvirt_uri)
self.dom_driver = self.driver.get_domain(self.vm_name)
self.mac = self.dom_driver.mac()
self.ip = self.dom_driver.ip()
self.logger.info("IP: %s", str(self.ip))
def __init__(self,
owner: MemoryConsumerPythonBackend,
shared_terminable: Optional[Terminable] = None):
LoggedThread.__init__(self)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.owner = owner
self.sleep_duration = owner.sleep_after_write
# set a worker own random number generator
self.random = random.Random(threading.get_ident())
def __init__(self,
config,
libvirt_iface,
shared_terminable: Terminable = None):
LoggedThread.__init__(self, daemon=True)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.config = config
self.libvirt_iface = libvirt_iface
self.guest_monitors = {}
self.guest_monitors_lock = threading.RLock()
def __init__(self,
port=None,
wait_timeout=60,
decrease_mem_time=0,
dynamic=True,
stdout_log_level=logging.INFO,
stderr_log_level=logging.ERROR):
LoggedObject.__init__(self)
Terminable.__init__(self)
self.guest_server_port = port
self.wait_timeout = wait_timeout
self.decrease_mem_time = decrease_mem_time
self.dynamic = dynamic
self.stdout_log_level = get_verbosity_level(stdout_log_level)
self.stderr_log_level = get_verbosity_level(stderr_log_level)
self._app_proc: Optional[subprocess.Popen] = None
self._resource_control_thread: Optional[DynamicResourceControl] = None
def __init__(self,
config: DictConfig,
monitor_name=None,
monitor_source=None,
shared_terminable: Terminable = None):
Terminable.__init__(self, shared_terminable=shared_terminable)
LoggedObject.__init__(self, monitor_name)
# Guard the data with a semaphore to ensure consistency.
self.data_lock = threading.Lock()
self.statistics = deque()
self.variables = {}
self.monitor_name = monitor_name
self.monitor_source = monitor_source
self.data_logger = DataLogger('monitor', monitor_source)
self.ready = threading.Event()
self.config = config
self.properties = self._properties()
self.collectors = self._generate_collectors()
self.hist_len = self.config.get('monitor', 'sample-history-length')
def __init__(self,
config: DictConfig,
libvirt_iface,
host_monitor: Monitor,
guest_manager: GuestManager,
shared_terminable: Terminable = None):
LoggedThread.__init__(self, daemon=True)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.policy_lock = threading.RLock()
self.host_monitor = host_monitor
self.guest_manager = guest_manager
self.config = config
self.interval: float = max(self.config.get('policy', 'interval'), 1.)
self.grace_period: float = min(
self.config.get('policy', 'grace-period'), self.interval)
self.inquiry_timeout: float = min(
self.config.get('policy', 'inquiry-timeout'), self.interval)
self.resources = [
r.strip() for r in self.config.get('policy', 'resources')
]
self.policy = ClassImporter(allocators).get_class(
config.get('policy', 'allocator'))(self.resources)
self.controllers = []
self.properties = {
'libvirt_iface': libvirt_iface,
'host_monitor': host_monitor,
'guest_manager': guest_manager,
'config': config
}
self.policy_data_loggers = {}
self.client_executor = None
self.get_controllers()
def __init__(self,
guest_server_port,
wait_timeout,
decrease_mem_time=1,
change_mem_func=None,
application_rss_func=None,
name=None,
spare_mem=0,
shared_terminable: Terminable = None):
LoggedThread.__init__(self, name=name, daemon=True)
Terminable.__init__(self, shared_terminable=shared_terminable)
self.stats_collector = MemoryStatistics(meminfo=True, vmstat=False)
self.guest_server_port = guest_server_port
self.wait_timeout = max(1, wait_timeout)
self.decrease_mem_time = max(0, decrease_mem_time)
self.spare_mem = max(0, spare_mem)
self.guest_server_client: Optional[GuestClient] = None
self.memory_diff = InvalidMemory
self.available_memory = InvalidMemory
if change_mem_func is not None:
self.change_mem_func = change_mem_func
if application_rss_func is not None:
self.application_rss = application_rss_func
def terminate(self):
Terminable.terminate(self)
if self.guest_server_client is not None:
self.guest_server_client.close()
def terminate(self):
Terminable.terminate(self)
self.terminate_application()
if self._resource_control_thread is not None:
self._resource_control_thread.terminate()
def __init__(self, interval, timeout=60):
LoggedObject.__init__(self)
Terminable.__init__(self)
self.interval = interval
self.timeout = timeout
def __init__(self,
vms_desc: dict,
duration: int,
output_path: str,
host_mom_config: DictConfigType = None,
exp_config: DictConfigType = None,
extra_info: Optional[dict] = None):
"""
Parameters:
-----------
vms_desc : descriptor of the virtual machines
duration : the experiment duration
output_path : the experiment output folder path
host_mom_config : override configurations in default MOM config
exp_config: override configurations in default experiment config
extra_info : extra information to write to the info file
"""
LoggedObject.__init__(self)
Terminable.__init__(self)
# Only this experiment will catch sigint and will terminate the experiment
self.terminate_on_signal(signal.SIGINT)
self.duration = duration
self.output_path = output_path
self.extra_info = extra_info
cpu_settings.disable_hyper_threading()
self.host_machine = HostMachine()
self.host_machine.set_owner_cpu_count('system', 2)
self.exp_config = DictConfig(DEFAULT_EXP_CONFIG, exp_config)
self.mom_executor = MomThreadExecutor(host_mom_config,
shared_terminable=self)
self.mom = self.mom_executor.mom
self.host_config = self.mom.config
self.threads = []
# Start the connection to libvirt
libvirt_uri = self.host_config.get('main', 'libvirt-hypervisor-uri')
self.conn = libvirt.open(libvirt_uri)
self.host_name = self.conn.getHostname()
self.max_vcpus = self.conn.getMaxVcpus(self.conn.getType())
# Get the machines. This will update the available CPU after pinning the CPUs to the VMs
self.number_of_guests = 0
self.vms = self.get_all_vms_instances(vms_desc)
# save test information file
try:
self.write_info(self.output_path)
except Exception as e:
self.logger.error("Could not write info file: %s", e)
self.logger.debug("Python version: %s", sys.version)
self.logger.debug('Host name: %s', self.host_name)
self.logger.debug('Max qemu vCPUs: %i', self.max_vcpus)
self.logger.debug('Allocated CPUs: %s',
pprint.pformat(self.host_machine.cpu_owners))
self.logger.info("Experiment initiated: %s", output_path)
def keyboard_interrupt_handler(self, signalnum, frame):
self.logger.warning(
"Terminating experiment due to keyboard interrupt.")
Terminable.keyboard_interrupt_handler(self, signalnum, frame)