def setUp(self):
self.config = Config('config',
'esx',
server='localhost',
username='******',
password='******',
owner='owner',
env='env',
log_dir='',
log_file='')
self.second_config = Config('second_config',
'esx',
server='localhost',
username='******',
password='******',
owner='owner',
env='env',
log_dir='',
log_file='')
fake_virt = Mock()
fake_virt.CONFIG_TYPE = 'esx'
guests = [Guest('guest1', fake_virt, 1)]
test_hypervisor = Hypervisor('test',
guestIds=[Guest('guest1', fake_virt, 1)])
assoc = {'hypervisors': [test_hypervisor]}
self.fake_domain_list = DomainListReport(self.second_config, guests)
self.fake_report = HostGuestAssociationReport(self.config, assoc)
def test_send_data_batch_hypervisor_checkin(self):
# This tests that reports of the right type are batched into one
# and that the hypervisorCheckIn method of the destination is called
# with the right parameters
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
d1['exclude_hosts'] = []
d1['filter_hosts'] = []
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
d2['exclude_hosts'] = []
d2['filter_hosts'] = []
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
data_to_send = {'source1': report1, 'source2': report2}
source_keys = ['source1', 'source2']
report1 = Mock()
report2 = Mock()
report1.hash = "report1_hash"
report2.hash = "report2_hash"
datastore = {'source1': report1, 'source2': report2}
manager = Mock()
options = Mock()
options.print_ = False
def check_hypervisorCheckIn(report, options=None):
self.assertEqual(report.association['hypervisors'],
data_to_send.values)
manager.hypervisorCheckIn = Mock(side_effect=check_hypervisorCheckIn)
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
def test_status_pending_hypervisor_async_result(self):
# This test's that when we have an async result from the server,
# we poll for the status on the interval until we get completed result
# Setup the test data
config1, d1 = self.create_fake_config('source1', **self.default_config_args)
config2, d2 = self.create_fake_config('source2', **self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
report1.job_id = 'job1'
report2.job_id = 'job2'
data_to_send = {'source1': report1,
'source2': report2}
source_keys = ['source1', 'source2']
batch_report1 = Mock() # The "report" to check status
batch_report1.state = AbstractVirtReport.STATE_CREATED
datastore = {'source1': report1, 'source2': report2}
manager = Mock()
items = [AbstractVirtReport.STATE_PROCESSING, AbstractVirtReport.STATE_PROCESSING,
AbstractVirtReport.STATE_PROCESSING, AbstractVirtReport.STATE_FINISHED,
AbstractVirtReport.STATE_FINISHED]
manager.check_report_state = Mock(side_effect=self.check_report_state_closure(items))
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger, config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False, options=self.options)
# In this test we want to see that the wait method is called when we
# expect and with what parameters we expect
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.submitted_report_and_hash_for_source ={'source1':(report1, 'hash1'),'source2':(report2, 'hash2')}
reports = destination_thread._get_data_common(source_keys)
self.assertEqual(0, len(reports))
reports = destination_thread._get_data_common(source_keys)
self.assertEqual(1, len(reports))
reports = destination_thread._get_data_common(source_keys)
self.assertEqual(2, len(reports))
def test_send_data_poll_hypervisor_async_result(self):
# This test's that when we have an async result from the server,
# we poll for the result
# Setup the test data
config1, d1 = self.create_fake_config('source1', **self.default_config_args)
config2, d2 = self.create_fake_config('source2', **self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
data_to_send = {'source1': report1,
'source2': report2}
source_keys = ['source1', 'source2']
batch_report1 = Mock() # The "report" to check status
batch_report1.state = AbstractVirtReport.STATE_CREATED
datastore = {'source1': report1, 'source2': report2}
manager = Mock()
items = [ManagerThrottleError(), ManagerThrottleError(), ManagerThrottleError(), AbstractVirtReport.STATE_FINISHED]
manager.check_report_state = Mock(side_effect=self.check_report_state_closure(items))
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger, config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False, options=self.options)
# In this test we want to see that the wait method is called when we
# expect and with what parameters we expect
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.check_report_status(batch_report1)
# There should be three waits, one after the job is submitted with duration of
# MinimumJobPollingInterval. The second and third with duration MinimumJobPollInterval * 2
# (and all subsequent calls as demonstrated by the third wait)
destination_thread.wait.assert_has_calls([
call(wait_time=MinimumJobPollInterval),
call(wait_time=MinimumJobPollInterval * 2),
call(wait_time=MinimumJobPollInterval * 2)])
class TestManager(TestBase):
""" Test of all available subscription managers. """
guest1 = Guest('9c927368-e888-43b4-9cdb-91b10431b258', xvirt, Guest.STATE_RUNNING)
guest2 = Guest('d5ffceb5-f79d-41be-a4c1-204f836e144a', xvirt, Guest.STATE_SHUTOFF)
guestInfo = [guest1]
hypervisor_id = "HYPERVISOR_ID"
config = Config('test', 'libvirt', owner='OWNER', env='ENV')
host_guest_report = HostGuestAssociationReport(config, {
'hypervisors': [
Hypervisor('9c927368-e888-43b4-9cdb-91b10431b258', []),
Hypervisor('ad58b739-5288-4cbc-a984-bd771612d670', [guest1, guest2])
]
})
domain_report = DomainListReport(config, [guest1], hypervisor_id)
def test_send_data_poll_async_429(self):
# This test's that when a 429 is detected during async polling
# we wait for the amount of time specified
source_keys = ['source1', 'source2']
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
datastore = {'source1': report1, 'source2': report2}
data_to_send = {'source1': report1, 'source2': report2}
config, d = self.create_fake_config('test', **self.default_config_args)
error_to_throw = ManagerThrottleError(retry_after=62)
manager = Mock()
manager.hypervisorCheckIn = Mock(side_effect=[error_to_throw, report1])
expected_wait_calls = [call(wait_time=error_to_throw.retry_after)]
logger = Mock()
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
destination_thread.wait.assert_has_calls(expected_wait_calls)
def test_sending_guests(self, parseFile, fromOptions, fromConfig,
getLogger):
self.setUpParseFile(parseFile)
options = Mock()
options.oneshot = True
options.interval = 0
options.print_ = False
fake_virt = Mock()
fake_virt.CONFIG_TYPE = 'esx'
test_hypervisor = Hypervisor('test',
guestIds=[Guest('guest1', fake_virt, 1)])
association = {'hypervisors': [test_hypervisor]}
options.log_dir = ''
options.log_file = ''
getLogger.return_value = sentinel.logger
fromConfig.return_value.config.name = 'test'
virtwho = Executor(self.logger, options, config_dir="/nonexistant")
config = Config("test",
"esx",
server="localhost",
username="******",
password="******",
owner="owner",
env="env")
virtwho.configManager.addConfig(config)
virtwho.queue = Queue()
virtwho.queue.put(HostGuestAssociationReport(config, association))
virtwho.run()
fromConfig.assert_called_with(sentinel.logger, config)
self.assertTrue(fromConfig.return_value.start.called)
fromOptions.assert_called_with(self.logger, options, ANY)
def test_new_status(self, get):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = uuids['host']
expected_guestId = uuids['vm']
expected_guest_state = Guest.STATE_RUNNING
get.side_effect = [
MagicMock(content=CLUSTERS_XML),
MagicMock(content=HOSTS_XML),
MagicMock(content=VMS_XML_STATUS),
]
expected_result = Hypervisor(
hypervisorId=expected_hypervisorId,
name=expected_hostname,
guestIds=[
Guest(
expected_guestId,
self.rhevm.CONFIG_TYPE,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT: '1',
Hypervisor.HYPERVISOR_TYPE_FACT: 'qemu',
Hypervisor.HYPERVISOR_VERSION_FACT: '1.2.3',
Hypervisor.HYPERVISOR_CLUSTER: 'Cetus',
Hypervisor.SYSTEM_UUID_FACT: 'db5a7a9f-6e33-3bfd-8129-c8010e4e1497',
}
)
result = self.rhevm.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def test_milicpu(self):
client = Mock()
client.get_nodes.return_value = self.new_nodes()
client.get_vms.return_value = self.vms()
config = self.create_config(name='test',
wrapper=None,
type='kubevirt',
owner='owner',
kubeconfig='/etc/hosts')
with patch.dict('os.environ', {'KUBECONFIG': '/dev/null'}):
kubevirt = Virt.from_config(self.logger, config, Datastore())
kubevirt._client = client
expected_result = Hypervisor(
hypervisorId='52c01ad890e84b15a1be4be18bd64ecd',
name='main',
guestIds=[
Guest(
'f83c5f73-5244-4bd1-90cf-02bac2dda608',
kubevirt.CONFIG_TYPE,
Guest.STATE_RUNNING,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT: '7',
Hypervisor.HYPERVISOR_TYPE_FACT: 'qemu',
Hypervisor.SYSTEM_UUID_FACT:
'52c01ad890e84b15a1be4be18bd64ecd',
Hypervisor.HYPERVISOR_VERSION_FACT: 'v1.18.0-rc.1',
})
result = kubevirt.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def test_getHostGuestMapping(self, mock_client):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = 'Fake_uuid'
expected_guestId = 'guest1UUID'
expected_guest_state = Guest.STATE_RUNNING
fake_parent = MagicMock()
fake_parent.value = 'fake_parent_id'
fake_parent._type = 'ClusterComputeResource'
fake_vm_id = MagicMock()
fake_vm_id.value = 'guest1'
fake_vm = MagicMock()
fake_vm.ManagedObjectReference = [fake_vm_id]
fake_vms = {
'guest1': {
'runtime.powerState': 'poweredOn',
'config.uuid': expected_guestId
}
}
self.esx.vms = fake_vms
fake_host = {
'hardware.systemInfo.uuid': expected_hypervisorId,
'config.network.dnsConfig.hostName': 'hostname',
'config.network.dnsConfig.domainName': 'domainname',
'config.product.version': '1.2.3',
'hardware.cpuInfo.numCpuPackages': '1',
'name': expected_hostname,
'parent': fake_parent,
'vm': fake_vm,
}
fake_hosts = {'random-host-id': fake_host}
self.esx.hosts = fake_hosts
fake_cluster = {'name': 'Fake_cluster_name'}
self.esx.clusters = {'fake_parent_id': fake_cluster}
expected_result = Hypervisor(hypervisorId=expected_hypervisorId,
name=expected_hostname,
guestIds=[
Guest(
expected_guestId,
self.esx.CONFIG_TYPE,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT:
'1',
Hypervisor.HYPERVISOR_TYPE_FACT:
'vmware',
Hypervisor.HYPERVISOR_VERSION_FACT:
'1.2.3',
Hypervisor.HYPERVISOR_CLUSTER:
'Fake_cluster_name',
})
result = self.esx.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def test_getHostGuestMapping(self):
kube_api = Mock()
kube_api.list_node.return_value = self.nodes()
kubevirt_api = Mock()
kubevirt_api.list_virtual_machine_instance_for_all_namespaces.return_value = self.vms(
)
self.kubevirt.kube_api = kube_api
self.kubevirt.kubevirt_api = kubevirt_api
expected_result = Hypervisor(
hypervisorId='52c01ad890e84b15a1be4be18bd64ecd',
name='master',
guestIds=[
Guest(
'default/win-2016',
self.kubevirt.CONFIG_TYPE,
Guest.STATE_RUNNING,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT: '2',
Hypervisor.HYPERVISOR_TYPE_FACT: 'qemu',
Hypervisor.HYPERVISOR_VERSION_FACT: 'v1.9.1+a0ce1bc657',
})
result = self.kubevirt.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def test_getHostGuestMapping(self, get):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = uuids['host']
expected_guestId = uuids['vm']
expected_guest_state = Guest.STATE_SHUTOFF
get.side_effect = [
MagicMock(text=CLUSTERS_XML),
MagicMock(text=HOSTS_XML),
MagicMock(text=VMS_XML),
]
expected_result = Hypervisor(hypervisorId=expected_hypervisorId,
name=expected_hostname,
guestIds=[
Guest(
expected_guestId,
self.rhevm,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT:
'1',
Hypervisor.HYPERVISOR_TYPE_FACT:
'qemu',
Hypervisor.HYPERVISOR_VERSION_FACT:
'1.2.3',
})
result = self.rhevm.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def test_getHostGuestMapping(self, session):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = '12345678-90AB-CDEF-1234-567890ABCDEF'
expected_guestId = '12345678-90AB-CDEF-1234-567890ABCDEF'
expected_guest_state = Guest.STATE_UNKNOWN
session.return_value.post.side_effect = HyperVMock.post
expected_result = Hypervisor(hypervisorId=expected_hypervisorId,
name=expected_hostname,
guestIds=[
Guest(
expected_guestId,
self.hyperv,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT:
'1',
Hypervisor.HYPERVISOR_TYPE_FACT:
'hyperv',
Hypervisor.HYPERVISOR_VERSION_FACT:
'0.1.2345.67890',
})
result = self.hyperv.getHostGuestMapping()['hypervisors'][0]
assert expected_result.toDict() == result.toDict()
def test_getHostGuestMapping_with_hm(self):
client = Mock()
client.get_nodes.return_value = self.nodes()
client.get_vms.return_value = self.vms()
config = self.create_config(name='test', wrapper=None, type='kubevirt',
owner='owner', kubeconfig='/etc/hosts',
hypervisor_id='hostname')
with patch.dict('os.environ', {'KUBECONFIG':'/dev/null'}):
kubevirt = Virt.from_config(self.logger, config, Datastore())
kubevirt._client = client
expected_result = Hypervisor(
hypervisorId='minikube',
name='master',
guestIds=[
Guest(
'f83c5f73-5244-4bd1-90cf-02bac2dda608',
kubevirt.CONFIG_TYPE,
Guest.STATE_RUNNING,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT: '2',
Hypervisor.HYPERVISOR_TYPE_FACT: 'qemu',
Hypervisor.HYPERVISOR_VERSION_FACT: 'v1.9.1+a0ce1bc657',
}
)
result = kubevirt.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def filter_hosts(self, filter_something, filter_type=''):
config_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, config_dir)
with open(os.path.join(config_dir, "test.conf"), "w") as f:
f.write("""
[test]
type=esx
server=does.not.exist
username=username
password=password
owner=owner
env=env
{filter_something}
{filter_type}
""".format(filter_something=filter_something, filter_type=filter_type))
conf = parse_file(os.path.join(config_dir, "test.conf"))
test_conf_values = conf.pop('test')
effective_config = EffectiveConfig()
effective_config['test'] = VirtConfigSection.from_dict(
test_conf_values, 'test', effective_config)
effective_config.validate()
config_manager = DestinationToSourceMapper(effective_config)
self.assertEqual(len(config_manager.configs), 1)
config = config_manager.configs[0][1]
included_hypervisor = Hypervisor('12345',
guestIds=[
Guest('guest-2',
xvirt.CONFIG_TYPE,
Guest.STATE_RUNNING),
])
excluded_hypervisor = Hypervisor('00000',
guestIds=[
Guest('guest-1',
xvirt.CONFIG_TYPE,
Guest.STATE_RUNNING),
])
assoc = {
'hypervisors': [
excluded_hypervisor,
included_hypervisor,
]
}
report = HostGuestAssociationReport(config, assoc)
assert report.association == {'hypervisors': [included_hypervisor]}
def test_multiple_hosts(self, session):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = 'Fake_uuid'
expected_guestId = 'guest1UUID'
expected_guest_state = Guest.STATE_UNKNOWN
xenapi = session.return_value.xenapi
hosts = []
for i in range(3):
hosts.append({
'uuid': expected_hypervisorId + str(i),
'hostname': expected_hostname + str(i),
'cpu_info': {
'socket_count': '1'
},
'software_version': {
'product_brand': 'XenServer',
'product_version': '1.2.3',
},
})
guest = {
'uuid': expected_guestId,
'power_state': 'unknown',
}
xenapi.host.get_all.return_value = hosts
xenapi.host.get_resident_VMs.return_value = [
guest,
]
xenapi.host.get_record = lambda x: x
xenapi.VM.get_record = lambda x: x
expected_result = [
Hypervisor(
hypervisorId=expected_hypervisorId + str(i),
name=expected_hostname + str(i),
guestIds=[
Guest(
expected_guestId,
self.xen.CONFIG_TYPE,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT: '1',
Hypervisor.HYPERVISOR_TYPE_FACT: 'XenServer',
Hypervisor.HYPERVISOR_VERSION_FACT: '1.2.3',
Hypervisor.SYSTEM_UUID_FACT: expected_hypervisorId + str(i)
}
) for i in range(3)]
self.xen._prepare()
result = self.xen.getHostGuestMapping()['hypervisors']
self.assertEqual(len(result), 3, "3 hosts should be reported")
self.assertEqual([x.toDict() for x in expected_result], [x.toDict() for x in result])
def test_duplicate_reports_are_ignored(self):
"""
Test that duplicate reports are filtered out when retrieving items
from the data store
"""
source_keys = ['source1', 'source2']
interval = 1
terminate_event = Mock()
options = Mock()
options.print_ = False
config1 = Config('source1', 'esx')
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
config = Mock()
manager = Mock()
guest1 = Guest('GUUID1', virt1, Guest.STATE_RUNNING)
report1 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_1')
report2 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_2')
report3 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_3')
datastore = {
'source1': report1, # Not changing, should be excluded later
'source2': report2, # Will change the report sent for source2
}
data_to_send = {
'source1': report1,
'source2': report2,
}
logger = Mock()
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=options)
destination_thread._send_data(data_to_send=data_to_send)
expected_hashes = {}
for source_key, report in data_to_send.iteritems():
expected_hashes[source_key] = report.hash
self.assertEqual(destination_thread.last_report_for_source,
expected_hashes)
# Pretend there were updates to the datastore from elsewhere
destination_thread.source['source2'] = report3
next_data_to_send = destination_thread._get_data()
expected_next_data_to_send = {'source2': report3}
self.assertEqual(next_data_to_send, expected_next_data_to_send)
def getHostGuestMapping(self):
"""
Get a dict of host to uvm mapping.
Args:
None.
Returns:
None.
"""
mapping = {'hypervisors': []}
host_uvm_map = self._interface.build_host_to_uvm_map(self.version)
for host_uuid in host_uvm_map:
host = host_uvm_map[host_uuid]
try:
if self.config['hypervisor_id'] == 'uuid':
hypervisor_id = host_uuid
elif self.config['hypervisor_id'] == 'hostname':
hypervisor_id = host['name']
except KeyError:
self.logger.debug("Host '%s' doesn't have hypervisor_id property",
host_uuid)
continue
guests = []
if 'guest_list' in host and len(host['guest_list']) > 0:
for guest_vm in host['guest_list']:
try:
state = guest_vm['power_state']
except KeyError:
self.logger.warning("Guest %s is missing power state. Perhaps they"
" are powered off", guest_vm['uuid'])
continue
guests.append(Guest(guest_vm['uuid'], self.CONFIG_TYPE, state))
else:
self.logger.debug("Host '%s' doesn't have any vms", host_uuid)
cluster_uuid = self._interface.get_host_cluster_uuid(host)
host_version = self._interface.get_host_version(host)
host_name = host['name']
facts = {
Hypervisor.CPU_SOCKET_FACT: str(host['num_cpu_sockets']),
Hypervisor.HYPERVISOR_TYPE_FACT: host.get('hypervisor_type', 'AHV'),
Hypervisor.HYPERVISOR_VERSION_FACT: str(host_version),
Hypervisor.HYPERVISOR_CLUSTER: str(cluster_uuid)}
mapping['hypervisors'].append(virt.Hypervisor(hypervisorId=hypervisor_id,
guestIds=guests,
name=host_name,
facts=facts))
return mapping
def test_getHostGuestMapping_incomplete_data(self, mock_client):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = 'Fake_uuid'
expected_guestId = 'guest1UUID'
expected_guest_state = Guest.STATE_UNKNOWN
fake_parent = MagicMock()
fake_parent.value = 'Fake_parent'
fake_vm_id = MagicMock()
fake_vm_id.value = 'guest1'
fake_vm = MagicMock()
fake_vm.ManagedObjectReference = [fake_vm_id]
fake_vms = {
'guest1': {
'runtime.powerState': 'BOGUS_STATE',
'config.uuid': expected_guestId
}
}
self.esx.vms = fake_vms
fake_host = {
'hardware.systemInfo.uuid': expected_hypervisorId,
'config.network.dnsConfig.hostName': 'hostname',
'config.network.dnsConfig.domainName': 'domainname',
'config.product.version': '1.2.3',
'hardware.cpuInfo.numCpuPackages': '1',
'parent': fake_parent,
'vm': fake_vm
}
fake_hosts = {'random-host-id': fake_host}
self.esx.hosts = fake_hosts
expected_result = Hypervisor(hypervisorId=expected_hypervisorId,
name=expected_hostname,
guestIds=[
Guest(
expected_guestId,
self.esx,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT:
'1',
Hypervisor.HYPERVISOR_TYPE_FACT:
'vmware',
Hypervisor.HYPERVISOR_VERSION_FACT:
'1.2.3',
})
result = self.esx.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())
def listDomains(self):
domains = []
response = self.server.list(True)
if response['status']['code'] != 0:
self.logger.error("Unable to list virtual machines: %s",
response['status']['message'])
else:
for vm in response['vmList']:
status = VDSM_STATE_TO_GUEST_STATE.get(vm['status'],
Guest.STATE_UNKNOWN)
domains.append(Guest(vm['vmId'], self, status))
return domains
def listDomains(self):
domains = []
vm_list = None
if self.jsonrpc_client is not None:
vm_list = self._get_vm_list_jsonrpc()
elif self.xmlrpc_client is not None:
vm_list = self._get_vm_list_xmlrpc()
if vm_list:
for vm in vm_list:
status = VDSM_STATE_TO_GUEST_STATE.get(vm['status'],
Guest.STATE_UNKNOWN)
domains.append(Guest(vm['vmId'], self.CONFIG_TYPE, status))
return domains
def filter_hosts(self, config):
config_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, config_dir)
with open(os.path.join(config_dir, "test.conf"), "w") as f:
f.write("""
[test]
type=esx
server=does.not.exist
username=username
password=password
owner=owner
env=env
{config}
""".format(config=config))
config_manager = ConfigManager(self.logger, config_dir)
self.assertEqual(len(config_manager.configs), 1)
config = config_manager.configs[0]
included_hypervisor = Hypervisor('12345',
guestIds=[
Guest('guest-2', xvirt,
Guest.STATE_RUNNING),
])
excluded_hypervisor = Hypervisor('00000',
guestIds=[
Guest('guest-1', xvirt,
Guest.STATE_RUNNING),
])
assoc = {
'hypervisors': [
excluded_hypervisor,
included_hypervisor,
]
}
report = HostGuestAssociationReport(config, assoc)
assert report.association == {'hypervisors': [included_hypervisor]}
def test_send_data_429_during_send_virt_guests(self):
# Show that when a 429 is encountered during the sending of a
# DomainListReport that we retry after waiting the appropriate
# amount of time
source_keys = ['source1']
config1 = Config('source1', 'esx')
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1, Guest.STATE_RUNNING)
report1 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
config = Mock()
config.polling_interval = 10
logger = Mock()
error_to_throw = ManagerThrottleError(retry_after=21)
manager = Mock()
manager.sendVirtGuests = Mock(side_effect=[error_to_throw, report1])
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=options)
destination_thread.wait = Mock()
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls([
call(report1, options=destination_thread.options),
call(report1, options=destination_thread.options)
])
destination_thread.wait.assert_has_calls(
[call(wait_time=error_to_throw.retry_after)])
def test_send_data_429_during_send_virt_guests(self):
# Show that when a 429 is encountered during the sending of a
# DomainListReport that we retry after waiting the appropriate
# amount of time
config, d = self.create_fake_config('test', **self.default_config_args)
source_keys = ['source1']
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
report1 = DomainListReport(config, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
logger = Mock()
error_to_throw = ManagerThrottleError(retry_after=62)
manager = Mock()
manager.sendVirtGuests = Mock(side_effect=[error_to_throw, report1])
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.wait = Mock()
destination_thread.record_status = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls(
[call(report1, options=destination_thread.options)])
destination_thread.wait.assert_has_calls(
[call(wait_time=error_to_throw.retry_after)])
def test_send_data_domain_list_reports(self):
# Show that DomainListReports are sent using the sendVirtGuests
# method of the destination
source_keys = ['source1']
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
report1 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
config, d = self.create_fake_config('test', **self.default_config_args)
logger = Mock()
manager = Mock()
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options)
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls(
[call(report1, options=destination_thread.options)])
def test_send_data_domain_list_reports(self):
# Show that DomainListReports are sent using the sendVirtGuests
# method of the destination
source_keys = ['source1']
config1 = Config('source1', 'esx')
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1, Guest.STATE_RUNNING)
report1 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
config = Mock()
config.polling_interval = 10
logger = Mock()
manager = Mock()
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=options)
destination_thread.wait = Mock()
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls(
[call(report1, options=destination_thread.options)])
def test_getHostGuestMapping(self, host_to_uvm_map):
host_to_uvm_map.return_value = HOST_UVM_MAP
expected_result = []
for host_uuid in HOST_UVM_MAP:
host = HOST_UVM_MAP[host_uuid]
hypervisor_id = host_uuid
host_name = host['name']
cluster_uuid = host['cluster_uuid']
guests = []
for guest_vm in host['guest_list']:
state = virt.Guest.STATE_RUNNING
guests.append(
Guest(guest_vm['uuid'], self.ahv.CONFIG_TYPE, state))
facts = {
Hypervisor.CPU_SOCKET_FACT: '2',
Hypervisor.HYPERVISOR_TYPE_FACT: u'kKvm',
Hypervisor.HYPERVISOR_VERSION_FACT: 'Nutanix 20180802.100874',
Hypervisor.HYPERVISOR_CLUSTER: str(cluster_uuid),
Hypervisor.SYSTEM_UUID_FACT: str(host_uuid)
}
expected_result.append(
Hypervisor(name=host_name,
hypervisorId=hypervisor_id,
guestIds=guests,
facts=facts))
result = self.ahv.getHostGuestMapping()['hypervisors']
self.assertEqual(len(result), len(expected_result),
'lists length do not match')
for index in range(0, len(result)):
self.assertEqual(expected_result[index].toDict(),
result[index].toDict())
class TestSatellite(TestBase):
mapping = {
'hypervisors': [
Hypervisor('host-1', [
Guest('guest1-1', xvirt.CONFIG_TYPE, Guest.STATE_RUNNING),
Guest('guest1-2', xvirt.CONFIG_TYPE, Guest.STATE_SHUTOFF)
]),
Hypervisor('host-2', [
Guest('guest2-1', xvirt.CONFIG_TYPE, Guest.STATE_RUNNING),
Guest('guest2-2', xvirt.CONFIG_TYPE, Guest.STATE_SHUTOFF),
Guest('guest2-3', xvirt.CONFIG_TYPE, Guest.STATE_RUNNING)
])
]
}
# Used as the default configuration
# Override in other tests if need be
default_config_args = {
'type': 'libvirt',
'hypervisor_id': 'uuid',
'simplified_vim': True,
'sat_server': "http://localhost:%s" % TEST_PORT,
'sat_username': '******',
'sat_password': '******',
}
@classmethod
def setUpClass(cls):
super(TestSatellite, cls).setUpClass()
cls.fake_server = FakeSatellite()
cls.thread = threading.Thread(target=cls.fake_server.serve_forever)
cls.thread.start()
@classmethod
def tearDownClass(cls):
cls.fake_server.shutdown()
def test_wrong_server(self):
options = Mock()
s = Satellite(self.logger, options)
config, d = self.create_fake_config('test', **TestSatellite.default_config_args)
d['sat_server'] = "wrong_server"
d['sat_username'] = "******"
d['sat_password'] = "******"
report = HostGuestAssociationReport(config, self.mapping)
self.assertRaises(SatelliteError, s.hypervisorCheckIn, report, options)
def test_wrong_username(self):
options = Mock()
options.force_register = True
s = Satellite(self.logger, options)
config, d = self.create_fake_config('test', **TestSatellite.default_config_args)
d['sat_server'] = "http://*****:*****@patch('virtwho.password.Password._can_write')
def test_per_config_options_encrypted(self, can_write):
options = Mock()
options.force_register = True
can_write.return_value = True
with tempfile.NamedTemporaryFile() as tmp:
password.Password.KEYFILE = tmp.name
config_dict = {
"sat_server": "http://localhost:%s" % TEST_PORT,
"sat_username": "******",
"sat_encrypted_password": hexlify(password.Password.encrypt('password')),
"type": "libvirt",
}
config = VirtConfigSection.from_dict(config_dict, 'test', None)
config.validate()
dests = DestinationToSourceMapper.parse_dests_from_dict(config._values)
self.assertEqual(len(dests), 1)
dest_info = dests.pop()
s = Manager.fromInfo(self.logger, options, dest_info)
self.assertTrue(isinstance(s, Satellite))
report = HostGuestAssociationReport(config, self.mapping)
result = s.hypervisorCheckIn(report, options)
self.assertTrue("failedUpdate" in result)
self.assertTrue("created" in result)
self.assertTrue("updated" in result)
class TestSubscriptionManager(TestBase):
guestList = [
Guest('222', xvirt.CONFIG_TYPE, Guest.STATE_RUNNING),
Guest('111', xvirt.CONFIG_TYPE, Guest.STATE_RUNNING),
Guest('333', xvirt.CONFIG_TYPE, Guest.STATE_RUNNING),
]
mapping = {
'hypervisors': [
Hypervisor('123', guestList, name='TEST_HYPERVISOR'),
Hypervisor('123', guestList, name='TEST_HYPERVISOR2')
]
}
hypervisor_id = "HYPERVISOR_ID"
uep_connection = None
@classmethod
@patch('rhsm.config.initConfig')
@patch('rhsm.certificate.create_from_file')
def setUpClass(cls, rhsmcert, rhsmconfig):
super(TestSubscriptionManager, cls).setUpClass()
config = VirtConfigSection.from_dict({'type': 'libvirt'}, 'test', None)
cls.tempdir = tempfile.mkdtemp()
with open(os.path.join(cls.tempdir, 'cert.pem'), 'w') as f:
f.write("\n")
rhsmcert.return_value.subject = {'CN': 123}
rhsmconfig.return_value.get.side_effect = lambda group, key: {
'consumerCertDir': cls.tempdir
}.get(key, DEFAULT)
cls.sm = SubscriptionManager(cls.logger, config)
cls.sm.connection = MagicMock()
cls.sm.connection.return_value.has_capability = MagicMock(
return_value=False)
cls.sm.connection.return_value.getConsumer = MagicMock(return_value={})
cls.sm.connection.return_value.getOwner = MagicMock(
return_value={'key': 'owner'})
cls.uep_connection = patch('rhsm.connection.UEPConnection',
cls.sm.connection)
cls.uep_connection.start()
cls.sm.cert_uuid = 123
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tempdir)
cls.uep_connection.stop()
def test_sendVirtGuests(self):
config = VirtConfigSection.from_dict({'type': 'libvirt'}, 'test', None)
report = DomainListReport(config, self.guestList, self.hypervisor_id)
self.sm.sendVirtGuests(report)
self.sm.connection.updateConsumer.assert_called_with(
123,
guest_uuids=[g.toDict() for g in self.guestList],
hypervisor_id=self.hypervisor_id)
def test_hypervisorCheckIn(self):
owner = "owner"
config = VirtConfigSection.from_dict(
{
'type': 'libvirt',
'owner': owner
}, 'test', None)
# Ensure the data takes the proper for for the old API
self.sm.connection.return_value.has_capability = MagicMock(
return_value=False)
self.sm.logger = MagicMock()
report = HostGuestAssociationReport(config, self.mapping)
self.sm.hypervisorCheckIn(report)
self.sm.connection.hypervisorCheckIn.assert_called_with(
owner,
'',
dict((host.hypervisorId, [g.toDict() for g in host.guestIds])
for host in self.mapping['hypervisors']),
options=None)
self.sm.logger.warning.assert_called_with(
"The hypervisor id '123' is assigned to 2 different systems. "
"Only one will be recorded at the server.")
@patch('rhsm.connection.UEPConnection')
# def test_hypervisorCheckInAsync(self):
def test_hypervisorCheckInAsync(self, rhsmconnection):
owner = 'owner'
config = VirtConfigSection.from_dict(
{
'type': 'libvirt',
'owner': owner
}, 'test', None)
# Ensure we try out the new API
rhsmconnection.return_value.has_capability.return_value = True
self.sm.logger = MagicMock()
report = HostGuestAssociationReport(config, self.mapping)
self.sm.hypervisorCheckIn(report)
expected = {
'hypervisors': [h.toDict() for h in self.mapping['hypervisors']]
}
self.sm.connection.hypervisorCheckIn.assert_called_with('owner',
'',
expected,
options=None)
self.sm.logger.warning.assert_called_with(
"The hypervisor id '123' is assigned to 2 different systems. "
"Only one will be recorded at the server.")
self.sm.connection.return_value.has_capability = MagicMock(
return_value=False)
@patch('rhsm.connection.UEPConnection')
# def test_hypervisorHeartbeat(self):
def test_hypervisorHeartbeat(self, rhsmconnection):
owner = 'owner'
config = VirtConfigSection.from_dict(
{
'type': 'libvirt',
'owner': owner
}, 'test', None)
# Ensure we try out the new API
rhsmconnection.return_value.has_capability.return_value = True
self.sm.logger = MagicMock()
self.sm.hypervisorHeartbeat(config, options=None)
self.sm.connection.hypervisorHeartbeat.assert_called_with(owner, None)
@patch('rhsm.connection.UEPConnection')
def test_job_status(self, rhsmconnection):
rhsmconnection.return_value.has_capability.return_value = True
config = VirtConfigSection.from_dict(
{
'type': 'libvirt',
'owner': 'owner'
}, 'test', None)
report = HostGuestAssociationReport(config, self.mapping)
self.sm.hypervisorCheckIn(report)
rhsmconnection.return_value.getJob.return_value = {
'state': 'RUNNING',
}
self.sm.check_report_state(report)
self.assertEqual(report.state, AbstractVirtReport.STATE_PROCESSING)
def host(_host):
return {'uuid': _host}
# self.sm.connection.return_value.getJob.return_value = {
rhsmconnection.return_value.getJob.return_value = {
'state': 'FINISHED',
'resultData': {
'failedUpdate': ["failed"],
'updated': [host('123')],
'created': [host('456')],
'unchanged': [host('789')]
}
}
self.sm.logger = MagicMock()
self.sm.check_report_state(report)
# calls: authenticating + checking job status + 1 line about the number of unchanged
self.assertEqual(self.sm.logger.debug.call_count, 3)
self.assertEqual(report.state, AbstractVirtReport.STATE_FINISHED)
def test_getHostGuestMapping(self, session):
expected_hostname = 'hostname.domainname'
expected_hypervisorId = 'Fake_uuid'
expected_guestId = 'guest1UUID'
expected_guest_state = Guest.STATE_UNKNOWN
xenapi = session.return_value.xenapi
host = {
'uuid': expected_hypervisorId,
'hostname': expected_hostname,
'cpu_info': {
'socket_count': '1'
},
'software_version': {
'product_brand': 'XenServer',
'product_version': '1.2.3',
},
}
xenapi.host.get_all.return_value = [
host
]
xenapi.host.get_record.return_value = host
control_domain = {
'uuid': '0',
'is_control_domain': True,
}
guest = {
'uuid': expected_guestId,
'power_state': 'unknown',
}
snapshot = {
'uuid': '12345678-90AB-CDEF-1234-567890ABCDEF',
'is_a_snapshot': True,
'power_state': 'unknown',
}
xenapi.host.get_resident_VMs.return_value = [
control_domain,
snapshot,
guest,
]
xenapi.VM.get_record = lambda x: x
expected_result = Hypervisor(
hypervisorId=expected_hypervisorId,
name=expected_hostname,
guestIds=[
Guest(
expected_guestId,
self.xen,
expected_guest_state,
)
],
facts={
Hypervisor.CPU_SOCKET_FACT: '1',
Hypervisor.HYPERVISOR_TYPE_FACT: 'XenServer',
Hypervisor.HYPERVISOR_VERSION_FACT: '1.2.3',
}
)
self.xen._prepare()
result = self.xen.getHostGuestMapping()['hypervisors'][0]
self.assertEqual(expected_result.toDict(), result.toDict())