def hypervisorHeartbeat(self, config, options=None):
"""
Send heart beat to candlepin server
:param config: reference on configuration
:param options: other options
"""
if options:
named_options = NamedOptions()
for key, value in options['global'].items():
setattr(named_options, key, value)
else:
named_options = None
try:
connection = self._connect(config)
result = connection.hypervisorHeartbeat(config['owner'],
named_options)
except BadStatusLine:
raise ManagerError(
"Communication with subscription manager interrupted")
except rhsm_connection.RateLimitExceededException as e:
raise ManagerThrottleError(e.retry_after)
except rhsm_connection.GoneException:
raise ManagerError(
"Communication with subscription manager failed: consumer no longer exists"
)
except rhsm_connection.ConnectionException as e:
if hasattr(e, 'code'):
raise ManagerError(
"Communication with subscription manager failed with code %d: %s"
% (e.code, str(e)))
raise ManagerError(
"Communication with subscription manager failed: %s" % str(e))
return result
def sendVirtGuests(self, report, options=None):
"""
Update consumer facts with info about virtual guests.
`guests` is a list of `Guest` instances (or it children).
"""
guests = report.guests
self._connect(report.config)
# Sort the list
guests.sort(key=lambda item: item.uuid)
serialized_guests = [guest.toDict() for guest in guests]
self.logger.info(
'Sending update in guests lists for config '
'"%s": %d guests found', report.config.name, len(guests))
self.logger.debug("Domain info: %s",
json.dumps(serialized_guests, indent=4))
# Send list of guest uuids to the server
try:
self.connection.updateConsumer(self.uuid(),
guest_uuids=serialized_guests,
hypervisor_id=report.hypervisor_id)
except rhsm_connection.GoneException:
raise ManagerError(
"Communication with subscription manager failed: consumer no longer exists"
)
except rhsm_connection.RateLimitExceededException as e:
raise ManagerThrottleError(e.retry_after)
report.state = AbstractVirtReport.STATE_FINISHED
def hypervisorCheckIn(self, report, options=None):
""" Send hosts to guests mapping to subscription manager. """
connection = self._connect(report.config)
is_async = self._is_rhsm_server_async(report, connection)
serialized_mapping = self._hypervisor_mapping(report, is_async,
connection)
self.logger.debug(
"Host-to-guest mapping being sent to '{owner}': {mapping}".format(
owner=report.config['owner'],
mapping=json.dumps(serialized_mapping, indent=4)))
# All subclasses of ConfigSection use dictionary like notation,
# but RHSM uses attribute like notation
if options:
named_options = NamedOptions()
for key, value in options['global'].items():
setattr(named_options, key, value)
else:
named_options = None
try:
try:
result = self.connection.hypervisorCheckIn(
report.config['owner'],
report.config['env'],
serialized_mapping,
options=named_options) # pylint:disable=unexpected-keyword-arg
except TypeError:
# This is temporary workaround until the options parameter gets implemented
# in python-rhsm
self.logger.debug(
"hypervisorCheckIn method in python-rhsm doesn't understand options parameter, ignoring"
)
result = self.connection.hypervisorCheckIn(
report.config['owner'], report.config['env'],
serialized_mapping)
except BadStatusLine:
raise ManagerError(
"Communication with subscription manager interrupted")
except rhsm_connection.RateLimitExceededException as e:
raise ManagerThrottleError(e.retry_after)
except rhsm_connection.GoneException:
raise ManagerError(
"Communication with subscription manager failed: consumer no longer exists"
)
except rhsm_connection.ConnectionException as e:
if hasattr(e, 'code'):
raise ManagerError(
"Communication with subscription manager failed with code %d: %s"
% (e.code, str(e)))
raise ManagerError(
"Communication with subscription manager failed: %s" % str(e))
if is_async is True:
report.state = AbstractVirtReport.STATE_CREATED
report.job_id = result['id']
else:
report.state = AbstractVirtReport.STATE_FINISHED
return result
def test_send_data_poll_async_429(self):
# This test's that when a 429 is detected during async polling
# we wait for the amount of time specified
source_keys = ['source1', 'source2']
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
datastore = {'source1': report1, 'source2': report2}
data_to_send = {'source1': report1, 'source2': report2}
config, d = self.create_fake_config('test', **self.default_config_args)
error_to_throw = ManagerThrottleError(retry_after=62)
manager = Mock()
manager.hypervisorCheckIn = Mock(side_effect=[error_to_throw, report1])
expected_wait_calls = [call(wait_time=error_to_throw.retry_after)]
logger = Mock()
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.record_status = Mock()
destination_thread._send_data(data_to_send)
destination_thread.wait.assert_has_calls(expected_wait_calls)
def test_send_data_429_during_send_virt_guests(self):
# Show that when a 429 is encountered during the sending of a
# DomainListReport that we retry after waiting the appropriate
# amount of time
source_keys = ['source1']
config1 = Config('source1', 'esx')
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1, Guest.STATE_RUNNING)
report1 = DomainListReport(config1, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
config = Mock()
config.polling_interval = 10
logger = Mock()
error_to_throw = ManagerThrottleError(retry_after=21)
manager = Mock()
manager.sendVirtGuests = Mock(side_effect=[error_to_throw, report1])
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=options)
destination_thread.wait = Mock()
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls([
call(report1, options=destination_thread.options),
call(report1, options=destination_thread.options)
])
destination_thread.wait.assert_has_calls(
[call(wait_time=error_to_throw.retry_after)])
def check_report_state(self, report):
# BZ 1554228
job_id = str(report.job_id)
self._connect(report.config)
self.logger.debug('Checking status of job %s', job_id)
try:
result = self.connection.getJob(job_id)
except BadStatusLine:
raise ManagerError(
"Communication with subscription manager interrupted")
except rhsm_connection.RateLimitExceededException as e:
raise ManagerThrottleError(e.retry_after)
except rhsm_connection.ConnectionException as e:
if hasattr(e, 'code'):
raise ManagerError(
"Communication with subscription manager failed with code %d: %s"
% (e.code, str(e)))
raise ManagerError(
"Communication with subscription manager failed: %s" % str(e))
state = STATE_MAPPING.get(result['state'],
AbstractVirtReport.STATE_FAILED)
report.state = state
if state not in (AbstractVirtReport.STATE_FINISHED,
AbstractVirtReport.STATE_CANCELED,
AbstractVirtReport.STATE_FAILED):
self.logger.debug('Job %s not finished', job_id)
else:
# log completed job status
result_data = result.get('resultData', {})
if not result_data:
self.logger.warning("Job status report without resultData: %s",
result)
return
if isinstance(result_data, string_types):
self.logger.warning(
"Job status report encountered the following error: %s",
result_data)
return
for fail in result_data.get('failedUpdate', []):
self.logger.error("Error during update list of guests: %s",
str(fail))
self.logger.debug("Number of mappings unchanged: %d",
len(result_data.get('unchanged', [])))
self.logger.info("Mapping for config \"%s\" updated",
report.config.name)
def test_send_data_429_during_send_virt_guests(self):
# Show that when a 429 is encountered during the sending of a
# DomainListReport that we retry after waiting the appropriate
# amount of time
config, d = self.create_fake_config('test', **self.default_config_args)
source_keys = ['source1']
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
report1 = DomainListReport(config, [guest1],
hypervisor_id='hypervisor_id_1')
datastore = {'source1': report1}
data_to_send = {'source1': report1}
logger = Mock()
error_to_throw = ManagerThrottleError(retry_after=62)
manager = Mock()
manager.sendVirtGuests = Mock(side_effect=[error_to_throw, report1])
terminate_event = Mock()
interval = 10
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
destination_thread.wait = Mock()
destination_thread.record_status = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread._send_data(data_to_send)
manager.sendVirtGuests.assert_has_calls(
[call(report1, options=destination_thread.options)])
destination_thread.wait.assert_has_calls(
[call(wait_time=error_to_throw.retry_after)])
def test_send_data_poll_hypervisor_async_result(self):
# This test's that when we have an async result from the server,
# we poll for the result
# Setup the test data
config1, d1 = self.create_fake_config('source1',
**self.default_config_args)
config2, d2 = self.create_fake_config('source2',
**self.default_config_args)
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1.CONFIG_TYPE, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2.CONFIG_TYPE, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
data_to_send = {'source1': report1, 'source2': report2}
source_keys = ['source1', 'source2']
batch_report1 = Mock() # The "report" to check status
batch_report1.state = AbstractVirtReport.STATE_CREATED
datastore = {'source1': report1, 'source2': report2}
manager = Mock()
items = [
ManagerThrottleError(),
ManagerThrottleError(),
ManagerThrottleError(), AbstractVirtReport.STATE_FINISHED
]
manager.check_report_state = Mock(
side_effect=self.check_report_state_closure(items))
logger = Mock()
config, d = self.create_fake_config('test', **self.default_config_args)
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=False,
options=self.options)
# In this test we want to see that the wait method is called when we
# expect and with what parameters we expect
destination_thread.wait = Mock()
destination_thread.is_terminated = Mock(return_value=False)
destination_thread.check_report_status(batch_report1)
# There should be three waits, one after the job is submitted with duration of
# MinimumJobPollingInterval. The second and third with duration MinimumJobPollInterval * 2
# (and all subsequent calls as demonstrated by the third wait)
destination_thread.wait.assert_has_calls([
call(wait_time=MinimumJobPollInterval),
call(wait_time=MinimumJobPollInterval * 2),
call(wait_time=MinimumJobPollInterval * 2)
])
def test_send_data_poll_async_429(self):
# This test's that when a 429 is detected during async polling
# we wait for the amount of time specified
source_keys = ['source1', 'source2']
config1 = Config('source1', 'esx')
config2 = Config('source2', 'esx')
virt1 = Mock()
virt1.CONFIG_TYPE = 'esx'
virt2 = Mock()
virt2.CONFIG_TYPE = 'esx'
guest1 = Guest('GUUID1', virt1, Guest.STATE_RUNNING)
guest2 = Guest('GUUID2', virt2, Guest.STATE_RUNNING)
assoc1 = {'hypervisors': [Hypervisor('hypervisor_id_1', [guest1])]}
assoc2 = {'hypervisors': [Hypervisor('hypervisor_id_2', [guest2])]}
report1 = HostGuestAssociationReport(config1, assoc1)
report2 = HostGuestAssociationReport(config2, assoc2)
datastore = {'source1': report1, 'source2': report2}
data_to_send = {'source1': report1, 'source2': report2}
config = Mock()
config.polling_interval = 10
error_to_throw = ManagerThrottleError(retry_after=20)
manager = Mock()
manager.hypervisorCheckIn.return_value = report1
# A closure to allow us to have a function that "modifies" the given
# report in a predictable way.
# In this case I want to set the state of the report to STATE_FINISHED
# after the first try
def check_report_state_closure(items):
item_iterator = iter(items)
def mock_check_report_state(report):
item = next(item_iterator)
if isinstance(item, Exception):
raise item
report.state = item
return report
return mock_check_report_state
states = [error_to_throw, AbstractVirtReport.STATE_FINISHED]
expected_wait_calls = [call(wait_time=error_to_throw.retry_after)]
check_report_mock = check_report_state_closure(states)
manager.check_report_state = Mock(side_effect=check_report_mock)
logger = Mock()
terminate_event = Mock()
interval = 10 # Arbitrary for this test
options = Mock()
options.print_ = False
destination_thread = DestinationThread(logger,
config,
source_keys=source_keys,
source=datastore,
dest=manager,
interval=interval,
terminate_event=terminate_event,
oneshot=True,
options=options)
destination_thread.wait = Mock()
destination_thread._send_data(data_to_send)
destination_thread.wait.assert_has_calls(expected_wait_calls)
def _connect(self, config=None):
""" Connect to the subscription-manager. """
kwargs = {
'host': self.rhsm_config.get('server', 'hostname'),
'ssl_port': int(self.rhsm_config.get('server', 'port')),
'handler': self.rhsm_config.get('server', 'prefix'),
'proxy_hostname': self.rhsm_config.get('server', 'proxy_hostname'),
'proxy_port': self.rhsm_config.get('server', 'proxy_port'),
'proxy_user': self.rhsm_config.get('server', 'proxy_user'),
'proxy_password': self.rhsm_config.get('server', 'proxy_password'),
'no_proxy': self.rhsm_config.get('server', 'no_proxy'),
'insecure': self.rhsm_config.get('server', 'insecure')
}
kwargs_to_config = {
'host': 'rhsm_hostname',
'ssl_port': 'rhsm_port',
'handler': 'rhsm_prefix',
'proxy_hostname': 'rhsm_proxy_hostname',
'proxy_port': 'rhsm_proxy_port',
'proxy_user': '******',
'proxy_password': '******',
'no_proxy': 'rhsm_no_proxy',
'insecure': 'rhsm_insecure'
}
rhsm_username = None
rhsm_password = None
if config:
try:
rhsm_username = config['rhsm_username']
rhsm_password = config['rhsm_password']
except KeyError:
pass
if rhsm_username == NotSetSentinel:
rhsm_username = None
if rhsm_password == NotSetSentinel:
rhsm_password = None
# Testing for None is necessary, it might be an empty string
for key, value in kwargs.items():
try:
from_config = config[kwargs_to_config[key]]
if from_config != NotSetSentinel and from_config is \
not None:
if key == 'ssl_port':
from_config = int(from_config)
kwargs[key] = from_config
except KeyError:
continue
if rhsm_username and rhsm_password:
self.logger.debug("Authenticating with RHSM username %s", rhsm_username)
kwargs['username'] = rhsm_username
kwargs['password'] = rhsm_password
else:
self.logger.debug("Authenticating with certificate: %s", self.cert_file)
if not os.access(self.cert_file, os.R_OK):
raise SubscriptionManagerUnregisteredError(
"Unable to read certificate, system is not registered or you are not root")
kwargs['cert_file'] = self.cert_file
kwargs['key_file'] = self.key_file
self.logger.info("X-Correlation-ID: %s", self.correlation_id)
if self.correlation_id:
kwargs['correlation_id'] = self.correlation_id
self.connection = rhsm_connection.UEPConnection(**kwargs)
# add version to user_agent header on connection BZ 1844506
self.connection.conn.user_agent += " " + parser.get_version().replace(" ", "/")
try:
if not self.connection.ping()['result']:
raise SubscriptionManagerError(
"Unable to obtain status from server, UEPConnection is likely not usable."
)
except rhsm_connection.RateLimitExceededException as e:
raise ManagerThrottleError(e.retry_after)
except BadStatusLine:
raise ManagerError("Communication with subscription manager interrupted")
return self.connection
def hypervisorCheckIn(self, report, options=None):
""" Send hosts to guests mapping to subscription manager. """
mapping = report.association
serialized_mapping = {}
self._connect(report.config)
self.logger.debug(
"Checking if server has capability 'hypervisor_async'")
is_async = hasattr(
self.connection, 'has_capability'
) and self.connection.has_capability('hypervisors_async')
if is_async and os.environ.get('VIRTWHO_DISABLE_ASYNC',
'').lower() in ['1', 'yes', 'true']:
self.logger.info(
"Async reports are supported but explicitly disabled")
is_async = False
if is_async:
self.logger.debug("Server has capability 'hypervisors_async'")
# Transform the mapping into the async version
serialized_mapping = {
'hypervisors': [h.toDict() for h in mapping['hypervisors']]
}
else:
self.logger.debug(
"Server does not have 'hypervisors_async' capability")
# Reformat the data from the mapping to make it fit with
# the old api.
for hypervisor in mapping['hypervisors']:
guests = [g.toDict() for g in hypervisor.guestIds]
serialized_mapping[hypervisor.hypervisorId] = guests
hypervisor_count = len(mapping['hypervisors'])
guest_count = sum(
len(hypervisor.guestIds) for hypervisor in mapping['hypervisors'])
self.logger.info(
'Sending update in hosts-to-guests mapping for config '
'"%s": %d hypervisors and %d guests found', report.config.name,
hypervisor_count, guest_count)
self.logger.debug("Host-to-guest mapping: %s",
json.dumps(serialized_mapping, indent=4))
try:
try:
result = self.connection.hypervisorCheckIn(report.config.owner,
report.config.env,
serialized_mapping,
options=options) # pylint:disable=unexpected-keyword-arg
except TypeError:
# This is temporary workaround until the options parameter gets implemented
# in python-rhsm
self.logger.debug(
"hypervisorCheckIn method in python-rhsm doesn't understand options parameter, ignoring"
)
result = self.connection.hypervisorCheckIn(
report.config.owner, report.config.env, serialized_mapping)
except BadStatusLine:
raise ManagerError(
"Communication with subscription manager interrupted")
except rhsm_connection.RateLimitExceededException as e:
retry_after = int(
getattr(e, 'headers', {}).get('Retry-After', '60'))
raise ManagerThrottleError(retry_after)
except rhsm_connection.GoneException:
raise ManagerError(
"Communication with subscription manager failed: consumer no longer exists"
)
except rhsm_connection.ConnectionException as e:
if hasattr(e, 'code'):
raise ManagerError(
"Communication with subscription manager failed with code %d: %s"
% (e.code, str(e)))
raise ManagerError(
"Communication with subscription manager failed: %s" % str(e))
if is_async is True:
report.state = AbstractVirtReport.STATE_PROCESSING
report.job_id = result['id']
else:
report.state = AbstractVirtReport.STATE_FINISHED
return result