def test_get_nonexistant_item_raises_keyerror(self):
datastore = Datastore()
item = sentinel.no_value
try:
item = datastore.get("NONEXISTANT")
except KeyError:
return
self.fail("Item retrieved for nonexistant key: %s" % item)
def test_get_nonexistant_item_raises_keyerror(self):
datastore = Datastore()
item = sentinel.no_value
try:
item = datastore.get("NONEXISTANT")
except KeyError:
return
self.fail("Item retrieved for nonexistant key: %s" % item)
def test_get_existing_item_with_default_returns_item(self):
# Ensures the item is returned if there is a default provided,
# and the item exists
datastore = Datastore()
expected_value = sentinel.test_value
self.mock_copy.deepcopy.side_effect = lambda x: x
with patch.dict(datastore._datastore, test_item=expected_value):
result = datastore.get("test_item", default=sentinel.default_value)
self.assertEqual(result, expected_value)
def test_get_returns_correct_item(self):
# Ensure that calling the get method returns the right value for a
# particular key, and that the value is loaded using pickle
datastore = Datastore()
with patch.dict(datastore._datastore,
test_item=sentinel.test_item_value):
result = datastore.get("test_item")
self.assertEqual(result, sentinel.test_item_value)
def test_get_returns_correct_item(self):
# Ensure that calling the get method returns the right value for a
# particular key, and that the value is loaded using pickle
datastore = Datastore()
with patch.dict(datastore._datastore,
test_item=sentinel.test_item_value):
result = datastore.get("test_item")
self.assertEqual(result, sentinel.test_item_value)
def test_get_existing_item_with_default_returns_item(self):
# Ensures the item is returned if there is a default provided,
# and the item exists
datastore = Datastore()
expected_value = sentinel.test_value
self.mock_copy.deepcopy.side_effect = lambda x: x
with patch.dict(datastore._datastore, test_item=expected_value):
result = datastore.get("test_item", default=sentinel.default_value)
self.assertEqual(result, expected_value)
def test_mapping_has_no_hostname_when_unavailible(self, virt):
config = self.create_config('test', None, type='libvirt', server='abc://server/test')
datastore = Datastore()
virt.return_value.getCapabilities.return_value = LIBVIRT_CAPABILITIES_NO_HOSTNAME_XML
virt.return_value.getType.return_value = "LIBVIRT_TYPE"
virt.return_value.getVersion.return_value = "VERSION 1337"
self.run_virt(config, datastore)
result = datastore.get(config.name)
for host in result.association['hypervisors']:
self.assertTrue(host.name is None)
def test_mapping_has_no_hostname_when_unavailible(self, virt):
config = Config('test', 'libvirt', server='abc://server/test')
datastore = Datastore()
virt.return_value.getCapabilities.return_value = LIBVIRT_CAPABILITIES_NO_HOSTNAME_XML
virt.return_value.getType.return_value = "LIBVIRT_TYPE"
virt.return_value.getVersion.return_value = "VERSION 1337"
self.run_virt(config, datastore)
result = datastore.get(config.name)
for host in result.association['hypervisors']:
self.assertTrue(host.name is None)
def test_mapping_hypervisor_has_system_uuid(self, virt):
config = self.create_config('test', None, type='libvirt', server='abc://server/test')
datastore = Datastore()
virt.return_value.getCapabilities.return_value = LIBVIRT_CAPABILITIES_XML
virt.return_value.getType.return_value = "LIBVIRT_TYPE"
virt.return_value.getVersion.return_value = "VERSION 1337"
self.run_virt(config, datastore)
result = datastore.get(config.name)
for host in result.association['hypervisors']:
self.assertEqual(host.facts['dmi.system.uuid'], 'this-is-uuid')
def test_mapping_hypervisor_has_system_uuid(self, virt):
config = self.create_config('test',
None,
type='libvirt',
server='abc://server/test')
datastore = Datastore()
virt.return_value.getCapabilities.return_value = LIBVIRT_CAPABILITIES_XML
virt.return_value.getType.return_value = "LIBVIRT_TYPE"
virt.return_value.getVersion.return_value = "VERSION 1337"
self.run_virt(config, datastore)
result = datastore.get(config.name)
for host in result.association['hypervisors']:
self.assertEqual(host.facts['dmi.system.uuid'], 'this-is-uuid')
def test_mapping_has_hostname_when_availible(self, virt):
config = self.create_config('test',
None,
type='libvirt',
server='abc://server/test')
datastore = Datastore()
virt.return_value.getCapabilities.return_value = LIBVIRT_CAPABILITIES_XML
virt.return_value.getType.return_value = "LIBVIRT_TYPE"
virt.return_value.getVersion.return_value = "VERSION 1337"
virt.return_value.getHostname.return_value = "test_host"
self.run_virt(config, datastore)
result = datastore.get(config.name)
for host in result.association['hypervisors']:
self.assertTrue(host.name == "test_host")
def test_oneshot(self, mock_client):
expected_assoc = '"well formed HostGuestMapping"'
expected_report = HostGuestAssociationReport(self.esx.config, expected_assoc)
updateSet = Mock()
updateSet.version = 'some_new_version_string'
updateSet.truncated = False
mock_client.return_value.service.WaitForUpdatesEx.return_value = updateSet
datastore = Datastore()
self.esx.applyUpdates = Mock()
getHostGuestMappingMock = Mock()
getHostGuestMappingMock.return_value = expected_assoc
self.esx.getHostGuestMapping = getHostGuestMappingMock
self.run_once(datastore)
result_report = datastore.get(self.esx.config.name)
self.assertEqual(expected_report.config.name, result_report.config.name)
self.assertEqual(expected_report.config._values, result_report.config._values)
self.assertEqual(expected_report._assoc, result_report._assoc)
def test_oneshot(self, mock_client):
expected_assoc = '"well formed HostGuestMapping"'
expected_report = HostGuestAssociationReport(self.esx.config,
expected_assoc)
updateSet = Mock()
updateSet.version = 'some_new_version_string'
updateSet.truncated = False
mock_client.return_value.service.WaitForUpdatesEx.return_value = updateSet
datastore = Datastore()
self.esx.applyUpdates = Mock()
getHostGuestMappingMock = Mock()
getHostGuestMappingMock.return_value = expected_assoc
self.esx.getHostGuestMapping = getHostGuestMappingMock
self.run_once(datastore)
result_report = datastore.get(self.esx.config.name)
self.assertEqual(expected_report.config.hash,
result_report.config.hash)
self.assertEqual(expected_report._assoc, result_report._assoc)
class Executor(object):
def __init__(self, logger, options):
"""
Executor class provides bridge between virtualization supervisor and
Subscription Manager.
logger - logger instance
options - options for virt-who, parsed from command line arguments
"""
self.logger = logger
self.options = options
self.terminate_event = Event()
self.virts = []
self.destinations = []
# Queue for getting events from virt backends
self.datastore = Datastore()
self.reloading = False
self.dest_to_source_mapper = DestinationToSourceMapper(options)
for name, config in self.dest_to_source_mapper.configs:
logger.info("Using config named '%s'" % name)
def _create_virt_backends(self):
"""
Create virts list with virt backend threads
"""
virts = []
for name, config in self.dest_to_source_mapper.configs:
try:
virt = Virt.from_config(self.logger, config, self.datastore,
terminate_event=self.terminate_event,
interval=self.options[VW_GLOBAL]['interval'],
oneshot=self.options[VW_GLOBAL]['oneshot'])
except Exception as e:
self.logger.error('Unable to use configuration "%s": %s', name, str(e))
continue
virts.append(virt)
return virts
def _create_destinations(self):
"""Populate self.destinations with a list of list with them
@param reset: Whether to kill existing destinations or not, defaults
to false
@type: bool
"""
dests = []
for info in self.dest_to_source_mapper.dests:
# Dests should already include all destinations we want created
# at this time. This method will make no assumptions of creating
# defaults of any kind.
source_keys = self.dest_to_source_mapper.dest_to_sources_map[info]
info.name = "destination_%s" % hash(info)
logger = log.getLogger(name=info.name)
manager = Manager.fromInfo(logger, self.options, info)
dest_class = info_to_destination_class[type(info)]
dest = dest_class(config=info, logger=logger,
source_keys=source_keys,
options=self.options,
source=self.datastore, dest=manager,
terminate_event=self.terminate_event,
interval=self.options[VW_GLOBAL]['interval'],
oneshot=self.options[VW_GLOBAL]['oneshot'])
dests.append(dest)
return dests
@staticmethod
def wait_on_threads(threads, max_wait_time=None, kill_on_timeout=False):
"""
Wait for each of the threads in the list to be terminated
@param threads: A list of IntervalThread objects to wait on
@type threads: list
@param max_wait_time: An optional max amount of seconds to wait
@type max_wait_time: int
@param kill_on_timeout: An optional arg that, if truthy and
max_wait_time is defined and exceeded, cause this method to attempt
to terminate and join the threads given it.
@type kill_on_timeout: bool
@return: A list of threads that have not quit yet. Without a
max_wait_time this list is always empty (or we are stuck waiting).
With a max_wait_time this list will include those threads that have
not quit yet.
@rtype: list
"""
delta_time = 1.0
total_waited = 0
threads_not_terminated = list(threads)
while len(threads_not_terminated) > 0:
if max_wait_time is not None and total_waited > max_wait_time:
if kill_on_timeout:
Executor.terminate_threads(threads_not_terminated)
return []
return threads_not_terminated
for thread in threads_not_terminated:
if thread.is_terminated():
threads_not_terminated.remove(thread)
if not threads_not_terminated:
break
time.sleep(delta_time)
if max_wait_time is not None:
total_waited += 1 * 1.0/delta_time
return threads_not_terminated
@staticmethod
def terminate_threads(threads):
for thread in threads:
thread.stop()
if thread.ident:
thread.join()
def run_oneshot(self):
# Start all sources
self.virts = self._create_virt_backends()
if len(self.virts) == 0:
err = "virt-who can't be started: no suitable virt backend found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
self.destinations = self._create_destinations()
if len(self.destinations) == 0:
err = "virt-who can't be started: no suitable destinations found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
for thread in self.virts:
thread.start()
Executor.wait_on_threads(self.virts)
if self.options[VW_GLOBAL]['print']:
to_print = {}
for source in self.dest_to_source_mapper.sources:
try:
report = self.datastore.get(source)
config = report.config
to_print[config.name] = report
except KeyError:
self.logger.info('Unable to retrieve report for source '
'\"%s\" for printing' % source)
return to_print
for thread in self.destinations:
thread.start()
Executor.wait_on_threads(self.destinations)
def run(self):
self.logger.debug("Starting infinite loop with %d seconds interval",
self.options[VW_GLOBAL]['interval'])
# Need to update the dest to source mapping of the dest_to_source_mapper object
# here because of the way that main reads the config from the command
# line
# TODO: Update dests to source map on addition or removal of configs
self.dest_to_source_mapper.update_dest_to_source_map()
# Start all sources
self.virts = self._create_virt_backends()
if len(self.virts) == 0:
err = "virt-who can't be started: no suitable virt backend found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
self.destinations = self._create_destinations()
if len(self.destinations) == 0:
err = "virt-who can't be started: no suitable destinations found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
for thread in self.virts:
thread.start()
for thread in self.destinations:
thread.start()
# Interruptibly wait on the other threads to be terminated
self.wait_on_threads(self.destinations)
raise ExitRequest(code=0)
def stop_threads(self):
self.terminate_event.set()
self.terminate_threads(self.virts)
self.terminate_threads(self.destinations)
def terminate(self):
self.logger.debug("virt-who is shutting down")
self.stop_threads()
self.virts = []
self.destinations = []
self.datastore = None
def reload(self):
"""
Causes all threads to be terminated in preparation for running again
"""
self.stop_threads()
self.terminate_event.clear()
self.datastore = Datastore()
class Executor(object):
def __init__(self, logger, options):
"""
Executor class provides bridge between virtualization supervisor and
Subscription Manager.
logger - logger instance
options - options for virt-who, parsed from command line arguments
"""
self.logger = logger
self.options = options
self.terminate_event = Event()
self.virts = []
self.destinations = []
# Queue for getting events from virt backends
self.datastore = Datastore()
self.reloading = False
self.dest_to_source_mapper = DestinationToSourceMapper(options)
for name, config in self.dest_to_source_mapper.configs:
logger.info("Using config named '%s'" % name)
def _create_virt_backends(self):
"""
Create virts list with virt backend threads
"""
virts = []
config_names = []
for name, config in self.dest_to_source_mapper.configs:
config_names.append(name)
try:
virt = Virt.from_config(
self.logger,
config,
self.datastore,
terminate_event=self.terminate_event,
interval=self.options[VW_GLOBAL]['interval'],
oneshot=self.options[VW_GLOBAL]['oneshot'],
status=self.options[VW_GLOBAL]['status'])
except Exception as e:
self.logger.error('Unable to use configuration "%s": %s', name,
str(e))
continue
virts.append(virt)
self._init_run_status(config_names)
return virts
def _init_run_status(self, config_names=[]):
# ensure that there is an entry in the status file for each config. We don't care if they work, we
# need to record their existence
need_write = False
try:
with FileLock(STATUS_LOCK):
os.makedirs(STATUS_DATA_DIR, exist_ok=True)
if os.path.exists(STATUS_DATA):
with open(STATUS_DATA, "r") as json_status:
try:
status_dict = json.load(json_status)
except JSONDecodeError:
status_dict = {}
else:
status_dict = {}
if 'sources' not in status_dict:
status_dict['sources'] = {}
need_write = True
if 'destinations' not in status_dict:
status_dict['destinations'] = {}
need_write = True
for name in config_names:
if name not in status_dict['sources']:
status_dict['sources'][name] = {
"last_successful_retrieve": None
}
need_write = True
if name not in status_dict['destinations']:
status_dict['destinations'][name] = {
"last_successful_send": None,
"last_job_id": None
}
need_write = True
if need_write:
# need to create the file if it does not exist
with open(STATUS_DATA, "w+") as json_status:
json.dump(status_dict, json_status)
except IOError:
self.logger.error(
"Unable to record run data. Cannot get lock on file.")
def _create_destinations(self):
"""Populate self.destinations with a list of list with them
@param reset: Whether to kill existing destinations or not, defaults
to false
@type: bool
"""
dests = []
for info in self.dest_to_source_mapper.dests:
# Dests should already include all destinations we want created
# at this time. This method will make no assumptions of creating
# defaults of any kind.
source_keys = self.dest_to_source_mapper.dest_to_sources_map[info]
info.name = "destination_%s" % hash(info)
logger = log.getLogger(name=info.name)
manager = Manager.fromInfo(logger, self.options, info)
dest_class = info_to_destination_class[type(info)]
dest = dest_class(config=info,
logger=logger,
source_keys=source_keys,
options=self.options,
source=self.datastore,
dest=manager,
terminate_event=self.terminate_event,
interval=self.options[VW_GLOBAL]['interval'],
oneshot=self.options[VW_GLOBAL]['oneshot'],
status=self.options[VW_GLOBAL]['status'])
dests.append(dest)
return dests
@staticmethod
def wait_on_threads(threads, max_wait_time=None, kill_on_timeout=False):
"""
Wait for each of the threads in the list to be terminated
@param threads: A list of IntervalThread objects to wait on
@type threads: list
@param max_wait_time: An optional max amount of seconds to wait
@type max_wait_time: int
@param kill_on_timeout: An optional arg that, if truthy and
max_wait_time is defined and exceeded, cause this method to attempt
to terminate and join the threads given it.
@type kill_on_timeout: bool
@return: A list of threads that have not quit yet. Without a
max_wait_time this list is always empty (or we are stuck waiting).
With a max_wait_time this list will include those threads that have
not quit yet.
@rtype: list
"""
delta_time = 1.0
total_waited = 0
threads_not_terminated = list(threads)
while len(threads_not_terminated) > 0:
if max_wait_time is not None and total_waited > max_wait_time:
if kill_on_timeout:
Executor.terminate_threads(threads_not_terminated)
return []
return threads_not_terminated
for thread in threads_not_terminated:
if thread.is_terminated():
threads_not_terminated.remove(thread)
if not threads_not_terminated:
break
time.sleep(delta_time)
if max_wait_time is not None:
total_waited += 1 * 1.0 / delta_time
return threads_not_terminated
@staticmethod
def terminate_threads(threads):
for thread in threads:
thread.stop()
if thread.ident:
thread.join()
def run_oneshot(self):
# Start all sources
self.virts = self._create_virt_backends()
if len(self.virts) == 0:
err = "virt-who can't be started: no suitable virt backend found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
self.destinations = self._create_destinations()
if len(self.destinations) == 0:
err = "virt-who can't be started: no suitable destinations found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
for thread in self.virts:
thread.start()
Executor.wait_on_threads(self.virts)
if self.options[VW_GLOBAL]['print']:
to_print = {}
for source in self.dest_to_source_mapper.sources:
try:
report = self.datastore.get(source)
config = report.config
to_print[config.name] = report
except KeyError:
self.logger.info(
f"Unable to retrieve report for source '{source}' for printing"
)
return to_print
for thread in self.destinations:
thread.start()
Executor.wait_on_threads(self.destinations)
if self.options[VW_GLOBAL]['status']:
output = {}
for source in self.dest_to_source_mapper.sources:
try:
report = self.datastore.get(source)
except KeyError:
self.logger.info(
f"Unable to retrieve report for source '{source}' for printing"
)
else:
output[report.config.name] = report
return output
def run(self):
self.logger.debug("Starting infinite loop with %d seconds interval",
self.options[VW_GLOBAL]['interval'])
# Need to update the dest to source mapping of the dest_to_source_mapper object
# here because of the way that main reads the config from the command
# line
# TODO: Update dests to source map on addition or removal of configs
self.dest_to_source_mapper.update_dest_to_source_map()
# Start all sources
self.virts = self._create_virt_backends()
if len(self.virts) == 0:
err = "virt-who can't be started: no suitable virt backend found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
self.destinations = self._create_destinations()
if len(self.destinations) == 0:
err = "virt-who can't be started: no suitable destinations found"
self.logger.error(err)
raise ExitRequest(code=1, message=err)
for thread in self.virts:
thread.start()
for thread in self.destinations:
thread.start()
# Interruptibly wait on the other threads to be terminated
self.wait_on_threads(self.destinations)
raise ExitRequest(code=0)
def stop_threads(self):
self.terminate_event.set()
self.terminate_threads(self.virts)
self.terminate_threads(self.destinations)
def terminate(self):
self.logger.debug("virt-who is shutting down")
self.stop_threads()
self.virts = []
self.destinations = []
self.datastore = None
def reload(self):
"""
Causes all threads to be terminated in preparation for running again
"""
self.stop_threads()
self.terminate_event.clear()
self.datastore = Datastore()
def test_get_nonexistant_item_with_default_returns_default(self):
# Ensures the default is returned if there is one provided and the
# key does not exist
datastore = Datastore()
result = datastore.get("NONEXISTANT", default=sentinel.default_value)
self.assertTrue(result == sentinel.default_value)
def test_get_nonexistant_item_with_default_returns_default(self):
# Ensures the default is returned if there is one provided and the
# key does not exist
datastore = Datastore()
result = datastore.get("NONEXISTANT", default=sentinel.default_value)
self.assertTrue(result == sentinel.default_value)
