def test_send_data_batch_hypervisor_checkin(self):
# This tests that reports of the right type are batched into one
# and that the hypervisorCheckIn method of the destination is called
# with the right parameters
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
d1['exclude_hosts'] = []
d1['filter_hosts'] = []
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
d2['exclude_hosts'] = []
d2['filter_hosts'] = []
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
data_to_send = {'source1': report1, 'source2': report2}
source_keys = ['source1', 'source2']
report1 = Mock()
report2 = Mock()
report1.hash = "report1_hash"
report2.hash = "report2_hash"
datastore = {'source1': report1, 'source2': report2}
manager = Mock()
options = Mock()
options.print_ = False
def check_hypervisorCheckIn(report, options=None):
self.assertEqual(report.association['hypervisors'],
data_to_send.values)
manager.hypervisorCheckIn = Mock(side_effect=check_hypervisorCheckIn)
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
def test_duplicate_reports_are_ignored(self):
"""
Test that duplicate reports are filtered out when retrieving items
from the data store
"""
source_keys = ['source1', 'source2']
interval = 1
terminate_event = Mock()
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
config, d = self.create_fake_config('test', **self.default_config_args)
manager = Mock()
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
report1 = DomainListReport(config, [guest1],
hypervisor_id='hypervisor_id_1')
report2 = DomainListReport(config, [guest1],
hypervisor_id='hypervisor_id_2')
report3 = DomainListReport(config, [guest1],
hypervisor_id='hypervisor_id_3')
datastore = {
'source1': report1, # Not changing, should be excluded later
'source2': report2, # Will change the report sent for source2
}
data_to_send = {
'source1': report1,
'source2': report2,
}
logger = Mock()
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.is_initial_run = False
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send=data_to_send)
expected_hashes = {}
for source_key, report in data_to_send.items():
expected_hashes[source_key] = report.hash
self.assertEqual(destination_thread.last_report_for_source,
expected_hashes)
# Pretend there were updates to the datastore from elsewhere
destination_thread.source['source2'] = report3
next_data_to_send = destination_thread._get_data()
expected_next_data_to_send = {'source2': report3}
self.assertEqual(next_data_to_send, expected_next_data_to_send)
def test_send_data_poll_async_429(self):
# This test's that when a 429 is detected during async polling
# we wait for the amount of time specified
source_keys = ['source1', 'source2']
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
datastore = {'source1': report1, 'source2': report2}
data_to_send = {'source1': report1, 'source2': report2}
config, d = self.create_fake_config('test', **self.default_config_args)
error_to_throw = ManagerThrottleError(retry_after=62)
manager = Mock()
manager.hypervisorCheckIn = Mock(side_effect=[error_to_throw, report1])
expected_wait_calls = [call(wait_time=error_to_throw.retry_after)]
logger = Mock()
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
destination_thread.wait.assert_has_calls(expected_wait_calls)
def test_send_data_429_during_send_virt_guests(self):
# Show that when a 429 is encountered during the sending of a
# DomainListReport that we retry after waiting the appropriate
# amount of time
config, d = self.create_fake_config('test', **self.default_config_args)
source_keys = ['source1']
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
report1 = DomainListReport(config, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
logger = Mock()
error_to_throw = ManagerThrottleError(retry_after=62)
manager = Mock()
manager.sendVirtGuests = Mock(side_effect=[error_to_throw, report1])
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.wait = Mock()
destination_thread.record_status = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls(
[call(report1, options=destination_thread.options)])
destination_thread.wait.assert_has_calls(
[call(wait_time=error_to_throw.retry_after)])
def test_send_data_domain_list_reports(self):
# Show that DomainListReports are sent using the sendVirtGuests
# method of the destination
source_keys = ['source1']
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
report1 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
config, d = self.create_fake_config('test', **self.default_config_args)
logger = Mock()
manager = Mock()
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options)
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls(
[call(report1, options=destination_thread.options)])
def test_send_data_clear_no_hypervisors(self):
# BZ 1943486
# Show that a run of _send_data with zero hypervisors
# for a config will clear the previous report hash
source_keys = ['source1', 'source2']
report1 = Mock()
report2 = Mock()
report1.hash = "report1_hash"
report2.hash = "report2_hash"
datastore = {'source1': report1, 'source2': report2}
last_report_for_source = {
'source1': report1.hash,
'source2': report2.hash
}
manager = Mock()
logger = Mock()
config, d = self.create_fake_config('source1',
**self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options)
destination_thread.is_initial_run = False
destination_thread.last_report_for_source = last_report_for_source
destination_thread.record_status = Mock()
report = HostGuestAssociationReport(config, {'hypervisors': []})
destination_thread._send_data({'source1': report})
self.assertEqual(destination_thread.last_report_for_source,
{'source2': report2.hash})
def test_merge_status(self):
self.tmp_dir = tempfile.mkdtemp()
status_pid_file_name = self.tmp_dir + os.path.sep + 'virt-who-status.pid'
status_pid_file_patcher = patch('virtwho.virt.virt.STATUS_LOCK',
status_pid_file_name)
status_pid_file_patcher.start()
self.addCleanup(status_pid_file_patcher.stop)
status_file_name = self.tmp_dir + os.path.sep + 'run_data.json'
status_data_file_patcher = patch('virtwho.virt.virt.STATUS_DATA',
status_file_name)
status_data_file_patcher.start()
self.addCleanup(status_data_file_patcher.stop)
self.addCleanup(shutil.rmtree, self.tmp_dir)
with open(status_file_name, "w+") as f:
f.write("""
{
"sources": {
"source1": {
"last_successful_retrieve": "2020-02-28 07:25:25 UTC",
"hypervisors": 20,
"guests": 37
},
"source2": {
"last_successful_retrieve": null
}
},
"destinations": {
"source1": {
"last_successful_send": "2020-02-28 07:25:27 UTC",
"last_job_id": "hypervisor12345"
},
"source2": {
"last_successful_send": null,
"last_job_id": null
}
}
}
""")
os.chmod(status_file_name, 444)
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
report1 = StatusReport(config1)
report2 = StatusReport(config2)
data_to_send = {'source1': report1, 'source2': report2}
source_keys = ['source1', 'source2']
datastore = {'source1': Mock(), 'source2': Mock()}
manager = Mock()
options = Mock()
options.print_ = False
manager.hypervisorCheckIn = Mock()
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
config.owner = 'test_owner'
terminate_event = Mock()
interval = 10 # Arbitrary for this test
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options,
status=True)
def check_report_status(report, status_call):
self.assertTrue(status_call)
report.last_job_status = "FINISHED"
destination_thread.record_status = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.check_report_status = Mock(
side_effect=check_report_status)
destination_thread._send_data(data_to_send)
self.assertEqual(report1.data['source']['last_successful_retrieve'],
"2020-02-28 07:25:25 UTC")
self.assertEqual(report1.data['source']['hypervisors'], 20)
self.assertEqual(report1.data['source']['guests'], 37)
self.assertEqual(report1.data['destination']["last_successful_send"],
"2020-02-28 07:25:27 UTC")
self.assertEqual(
report1.data['destination']["last_successful_send_job_status"],
"FINISHED")
self.assertEqual(report2.data['source']['last_successful_retrieve'],
None)
self.assertEqual(report2.data['source']['hypervisors'], None)
self.assertEqual(report2.data['source']['guests'], None)
self.assertEqual(report2.data['destination']["last_successful_send"],
None)
self.assertEqual(
report2.data['destination']["last_successful_send_job_status"],
None)
def test_record_status(self):
# This tests that reports of the right type are batched into one
# and that the hypervisorCheckIn method of the destination is called
# with the right parameters
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
data_to_send = {'source1': report1, 'source2': report2}
source_keys = ['source1', 'source2']
report1 = Mock()
report2 = Mock()
report1.hash = "report1_hash"
report2.hash = "report2_hash"
datastore = {'source1': report1, 'source2': report2}
manager = Mock()
options = Mock()
options.print_ = False
def check_hypervisorCheckIn(report, options=None):
report.job_id = '123456789'
return Mock()
manager.hypervisorCheckIn = Mock(side_effect=check_hypervisorCheckIn)
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=self.options)
def check_record_status(source_key, type, json_info):
if type == 'sources':
self.assertEqual(json_info['hypervisors'], 1)
elif type == 'destinations':
self.assertEqual(json_info['last_job_id'], '123456789')
destination_thread.record_status = Mock(
side_effect=check_record_status)
destination_thread.is_terminated = Mock(return_value=False)
destination_thread._send_data(data_to_send)