def __init__(self, is_remote=False, name=None):
LoggedObject.__init__(self, name)
Terminable.__init__(self)
self.is_remote = is_remote
self.guest_machine: Optional[GuestMachine] = None
self._running_process = set()
self._alloc_cpus = None
def __init__(self, libvirt_driver, domain, domain_name=None):
self.libvirt_driver = libvirt_driver
self.domain = domain
if domain_name is None:
domain_name = domain.name()
self.domain_name = domain_name
LoggedObject.__init__(self, domain_name)
def __init__(self, properties=None):
if properties is None:
properties = {}
self.properties: dict = properties
self.owner_name: str = self.properties.get('name', None)
self.config: DictConfig = self.properties.get('config', None)
LoggedObject.__init__(self, name=self.owner_name)
def __init__(self, output_path, export_path="exports", log_name=None):
LoggedObject.__init__(self, log_name)
if not output_path:
raise Exception("No data output path")
if not os.path.isdir(output_path):
raise Exception(f"Data output path '{output_path}' not found")
file_path = os.path.join(output_path, DEFAULT_DATA_FILE_NAME)
if not os.path.isfile(file_path):
raise Exception(f"Data file '{file_path}' not found")
self.__output_path = output_path
self.__file_name = DEFAULT_DATA_FILE_NAME
self.__data_file_path = file_path
self.__export_path = os.path.join(self.__output_path, export_path)
self.__name, _ext = os.path.splitext(self.__file_name)
self.__sql_file_path = os.path.join(self.__output_path, f'{self.__name}.sqlite3')
self.__column_names = None
reload = self.__check_reload__()
if reload:
self.logger.info("Generating DB from data log.")
self.clear_db()
self.analyze()
self.generate_cols()
self.create_exports_path()
def __init__(self, ip, name=None, port=8888, default_timeout=60, base_name="GuestClient"):
LoggedObject.__init__(self, name)
self.lock = threading.RLock()
self.ip = ip
self.port = port
self.default_timeout = default_timeout
self.client = PickleTcpClient(ip, port, timeout=default_timeout, base_name=base_name)
def __init__(self, config):
LoggedObject.__init__(self)
self.config = config
self.init_time = None
self.response_scripts = self.config.get('policy', 'response-scripts', {})
self.resource_diff = {}
self._notify_event = Event()
self._notify_event.clear()
def __init__(self,
vm_name,
libvirt_uri,
username,
password=None,
master_image=None,
title=None,
swappiness=60,
use_image_cow=True,
base_mem=2**10,
max_mem=2**12,
base_vcpus=1,
max_vcpus=1,
cpu_pin=None,
shared_terminable: Terminable = None,
**_unknown):
"""
vm_name: the virtual machine name/alias
libvirt_uri: a libvirt URI
username: ssh login username
password: ssh login password
master_image: the filename of the master image to use for this machine (should reside on IMAGES_FOLDER)
title: a title for the machine for documentation purpose
cpu_pin: a list of cpus to pin this machine to
swappiness: the swappiness parameter for the guest OS in the VM
use_image_cow: use copy-on-write mechanism - instead of full copy of the master image
max_vcpus: set the maximum cpus that is allocated to this machine (will change permanently)
_unknown: other properties
"""
LoggedObject.__init__(self, vm_name)
Terminable.__init__(self, shared_terminable=shared_terminable)
if master_image is None:
master_image = "generic-master.qcow2"
self.vm_name = vm_name
self.libvirt_uri = libvirt_uri
self.username = username
self.password = password
self.master_image = master_image
self.title = title
self.swappiness = swappiness if 0 <= swappiness <= 100 else 60
self.use_image_cow = use_image_cow
self.base_mem = base_mem
self.max_mem = max_mem
self.base_vcpus = base_vcpus
self.max_vcpus = max_vcpus
self.cpu_pin = cpu_pin
self.driver = LibvirtDriver(self.libvirt_uri)
self.dom_driver = self.driver.get_domain(self.vm_name)
self.mac = self.dom_driver.mac()
self.ip = self.dom_driver.ip()
self.logger.info("IP: %s", str(self.ip))
def __init__(self, max_memory, sleep_after_write, backend='cpp'):
LoggedObject.__init__(self)
self.max_memory = max_memory
self.sleep_after_write = sleep_after_write
self.backend = backend.lower()
self._proc: Optional[subprocess.Popen] = None
self._qthread = None
self._command_lock = None
def __init__(self, max_memory, sleep_after_write):
LoggedObject.__init__(self)
self.max_memory = max_memory
self.sleep_after_write = sleep_after_write
self.measure_start = None
self.hits = AtomicCounter()
self.throughput = AtomicCounter()
self.max_rand = 0
self.mem_arr: List[np.array] = []
self.workers: List[MemoryConsumerWorker] = []
self.change_load_lock = None
def __init__(self,
config: Union[DictConfig, dict, None] = None,
guest_name=None):
LoggedObject.__init__(self, guest_name)
self.config = DictConfig(DEFAULT_CONFIG, config)
self.monitor = Monitor(
self.config, monitor_name=guest_name
) # for collecting data, not running as a thread
self.policy = ClassImporter(guest_policy).get_class(
self.config.get('policy', 'policy'))(self.config)
self.server = GuestServer(self.config,
self.monitor,
self.policy,
guest_name=guest_name)
def __init__(self, cmd_args, encoding='utf-8', output_file=None, name=None, verbose=True):
LoggedObject.__init__(self, name)
self.cmd_args = cmd_args
self.encoding = encoding
self.output_file = output_file
self.verbose = verbose
self.out = None
self.err = None
self.output_stream: Union[TextIO, int] = subprocess.PIPE
if self.output_file:
self.output_stream = open(self.output_file, "a")
if self.verbose:
self.log_start()
self.proc = subprocess.Popen(self.cmd_args, encoding=self.encoding, stdin=subprocess.PIPE,
stdout=self.output_stream, stderr=self.output_stream)
def __init__(self,
config: DictConfig,
monitor_name=None,
monitor_source=None,
shared_terminable: Terminable = None):
Terminable.__init__(self, shared_terminable=shared_terminable)
LoggedObject.__init__(self, monitor_name)
# Guard the data with a semaphore to ensure consistency.
self.data_lock = threading.Lock()
self.statistics = deque()
self.variables = {}
self.monitor_name = monitor_name
self.monitor_source = monitor_source
self.data_logger = DataLogger('monitor', monitor_source)
self.ready = threading.Event()
self.config = config
self.properties = self._properties()
self.collectors = self._generate_collectors()
self.hist_len = self.config.get('monitor', 'sample-history-length')
def __init__(self,
port=None,
wait_timeout=60,
decrease_mem_time=0,
dynamic=True,
stdout_log_level=logging.INFO,
stderr_log_level=logging.ERROR):
LoggedObject.__init__(self)
Terminable.__init__(self)
self.guest_server_port = port
self.wait_timeout = wait_timeout
self.decrease_mem_time = decrease_mem_time
self.dynamic = dynamic
self.stdout_log_level = get_verbosity_level(stdout_log_level)
self.stderr_log_level = get_verbosity_level(stderr_log_level)
self._app_proc: Optional[subprocess.Popen] = None
self._resource_control_thread: Optional[DynamicResourceControl] = None
def __init__(self):
""" Constructor is called only once (singleton) """
# Allows reentrant by the same thread
LoggedObject.__init__(self)
self._lock = threading.RLock()
self._cpu_data = fetch_cpu_data()
self._cpu_hierarchy = self.subset("Node", "Socket", "Core", "CPU")
self._machine_core_subset = {0: self.subset("Core")}
self._node_core_subset = self.subset("Node", "Core")
self._socket_core_subset = self.subset("Socket", "Core")
self._core_core_subset = self.subset("Core", "Core")
self._core_cpu_subset = self.subset("Core", "CPU")
self._machine_availability = {0: len(self._cpu_data)}
self._node_availability = self.subset_count("Node")
self._socket_availability = self.subset_count("Socket")
self._core_availability = self.subset_count("Core")
self._cpu_availability = [True] * len(self._cpu_data)
def __init__(self,
vms_desc: dict,
duration: int,
output_path: str,
host_mom_config: DictConfigType = None,
exp_config: DictConfigType = None,
extra_info: Optional[dict] = None):
"""
Parameters:
-----------
vms_desc : descriptor of the virtual machines
duration : the experiment duration
output_path : the experiment output folder path
host_mom_config : override configurations in default MOM config
exp_config: override configurations in default experiment config
extra_info : extra information to write to the info file
"""
LoggedObject.__init__(self)
Terminable.__init__(self)
# Only this experiment will catch sigint and will terminate the experiment
self.terminate_on_signal(signal.SIGINT)
self.duration = duration
self.output_path = output_path
self.extra_info = extra_info
cpu_settings.disable_hyper_threading()
self.host_machine = HostMachine()
self.host_machine.set_owner_cpu_count('system', 2)
self.exp_config = DictConfig(DEFAULT_EXP_CONFIG, exp_config)
self.mom_executor = MomThreadExecutor(host_mom_config,
shared_terminable=self)
self.mom = self.mom_executor.mom
self.host_config = self.mom.config
self.threads = []
# Start the connection to libvirt
libvirt_uri = self.host_config.get('main', 'libvirt-hypervisor-uri')
self.conn = libvirt.open(libvirt_uri)
self.host_name = self.conn.getHostname()
self.max_vcpus = self.conn.getMaxVcpus(self.conn.getType())
# Get the machines. This will update the available CPU after pinning the CPUs to the VMs
self.number_of_guests = 0
self.vms = self.get_all_vms_instances(vms_desc)
# save test information file
try:
self.write_info(self.output_path)
except Exception as e:
self.logger.error("Could not write info file: %s", e)
self.logger.debug("Python version: %s", sys.version)
self.logger.debug('Host name: %s', self.host_name)
self.logger.debug('Max qemu vCPUs: %i', self.max_vcpus)
self.logger.debug('Allocated CPUs: %s',
pprint.pformat(self.host_machine.cpu_owners))
self.logger.info("Experiment initiated: %s", output_path)
def __init__(self, resource, properties):
LoggedObject.__init__(self)
self.resource = resource
self.libvirt = properties['libvirt_iface']
def __init__(self, *args, **kwargs):
LoggedObject.__init__(self)
self.content = dict(*args, **kwargs)
def __init__(self, uri):
LoggedObject.__init__(self)
self.conn = None
self.uri = uri
libvirt.registerErrorHandler(self.__libvirt_error_handler, None)
self.__connect()
def __init__(self):
LoggedObject.__init__(self)
self._cpu_data = cpu_settings.CpuData()
self._cpu_owners = {}
def __init__(self, module):
self.module_name = module.__name__
LoggedObject.__init__(self, self.module_name)
self._class_key = None
def __init__(self, resources):
LoggedObject.__init__(self)
self.resources = set(resources)
def __init__(self, interval, timeout=60):
LoggedObject.__init__(self)
Terminable.__init__(self)
self.interval = interval
self.timeout = timeout