def process(self):
"""Process the message."""
if self.event_type in (KAFKA_APPLICATION_CREATE,):
storage.create_source_event(self.source_id, self.auth_header, self.offset)
if storage.is_known_source(self.source_id):
if self.event_type in (KAFKA_APPLICATION_CREATE,):
self.save_sources_details()
self.save_source_info(bill=True)
# _Authentication_ messages are responsible for saving credentials.
# However, OCP does not send an Auth message. Therefore, we need
# to run the following branch for OCP which completes the source
# creation cycle for an OCP source.
if storage.get_source_type(self.source_id) == Provider.PROVIDER_OCP:
self.save_source_info(auth=True)
if self.event_type in (KAFKA_APPLICATION_UPDATE,):
if storage.get_source_type(self.source_id) == Provider.PROVIDER_AZURE:
# Because azure auth is split in Sources backend, we need to check both
# auth and billing when we recieve either auth update or app update event
updated = self.save_source_info(auth=True, bill=True)
else:
updated = self.save_source_info(bill=True)
if updated:
LOG.info(f"[ApplicationMsgProcessor] source_id {self.source_id} updated")
storage.enqueue_source_create_or_update(self.source_id)
else:
LOG.info(f"[ApplicationMsgProcessor] source_id {self.source_id} not updated. No changes detected.")
if self.event_type in (KAFKA_APPLICATION_DESTROY,):
storage.enqueue_source_delete(self.source_id, self.offset, allow_out_of_order=True)
def test_get_source_type(self):
"""Test to source type from source."""
test_source_id = 3
ocp_obj = Sources(source_id=test_source_id,
auth_header=self.test_header,
offset=3,
source_type=Provider.PROVIDER_OCP,
name='Test OCP Source',
authentication={'resource_name': 'arn:test'},
billing_source={'bucket': 'test-bucket'})
ocp_obj.save()
response = storage.get_source_type(test_source_id)
self.assertEquals(response, Provider.PROVIDER_OCP)
self.assertEquals(storage.get_source_type(test_source_id + 1), None)
def save_billing_source(self):
"""Store Sources billing information."""
LOG.info(f"[save_billing_source] starting for source_id {self.source_id} ...")
source_type = storage.get_source_type(self.source_id)
if not source_type:
LOG.info(f"[save_billing_source] source_type not found for source_id: {self.source_id}")
return
if source_type == Provider.PROVIDER_OCP:
# OCP sources do not have billing sources, so skip running thru this function
LOG.info("[save_billing_source] skipping for OCP source")
return
sources_network = self.get_sources_client()
try:
data_source = {"data_source": sources_network.get_data_source(source_type)}
except SourcesHTTPClientError as error:
LOG.info(f"[save_billing_source] billing info not available for source_id: {self.source_id}")
sources_network.set_source_status(error)
raise error
else:
if not data_source.get("data_source"):
return
result = bool(storage.add_provider_sources_billing_info(self.source_id, data_source))
LOG.info(f"[save_billing_source] completed for source_id: {self.source_id}: {result}")
return result
def process(self):
"""Process the message."""
if self.event_type in (KAFKA_AUTHENTICATION_CREATE):
LOG.debug(
f"[AuthenticationMsgProcessor] creating source for source_id: {self.source_id}"
)
storage.create_source_event(self.source_id, self.account_number,
self.auth_header, self.offset)
if storage.is_known_source(self.source_id):
if self.event_type in (KAFKA_AUTHENTICATION_CREATE):
self.save_source_info(auth=True)
if self.event_type in (KAFKA_AUTHENTICATION_UPDATE):
if storage.get_source_type(
self.source_id) == Provider.PROVIDER_AZURE:
# Because azure auth is split in Sources backend, we need to check both
# auth and billing when we recieve either auth update or app update event
updated = self.save_source_info(auth=True, bill=True)
else:
updated = self.save_source_info(auth=True)
if updated:
LOG.info(
f"[AuthenticationMsgProcessor] source_id {self.source_id} updated"
)
storage.enqueue_source_create_or_update(self.source_id)
else:
LOG.info(
f"[AuthenticationMsgProcessor] source_id {self.source_id} not updated. No changes detected."
)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f'Source ID not found for ID: {source_id}')
return
try:
if source_type == 'OCP':
source_details = sources_network.get_source_details()
# Check for imported to maintain temporary backwards compatibility
# until the Sources Front End creates 'imported' entry with OCP Cluster ID.
if source_details.get('source_ref'):
authentication = {
'resource_name': source_details.get('source_ref')
}
else:
uid = source_details.get('uid')
LOG.info(
f'OCP is using fallback Source UID ({str(uid)} for authentication.'
' Update frontend to add Cluster ID to the source_ref field on the Source.'
)
authentication = {'resource_name': uid}
elif source_type == 'AWS':
authentication = {
'resource_name': sources_network.get_aws_role_arn()
}
elif source_type == 'AZURE':
authentication = {
'credentials': sources_network.get_azure_credentials()
}
else:
LOG.error(f'Unexpected source type: {source_type}')
return
storage.add_provider_sources_auth_info(source_id, authentication)
except SourcesHTTPClientError:
LOG.info(
f'Authentication info not available for Source ID: {source_id}')
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f"Source ID not found for ID: {source_id}")
return
try:
if source_type == Provider.PROVIDER_OCP:
source_details = sources_network.get_source_details()
if source_details.get("source_ref"):
authentication = {
"resource_name": source_details.get("source_ref")
}
else:
raise SourcesHTTPClientError("Unable to find Cluster ID")
elif source_type in (Provider.PROVIDER_AWS,
Provider.PROVIDER_AWS_LOCAL):
authentication = {
"resource_name": sources_network.get_aws_role_arn()
}
elif source_type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
authentication = {
"credentials": sources_network.get_azure_credentials()
}
else:
LOG.error(f"Unexpected source type: {source_type}")
return
storage.add_provider_sources_auth_info(source_id, authentication)
storage.clear_update_flag(source_id)
LOG.info(f"Authentication attached to Source ID: {source_id}")
except SourcesHTTPClientError as error:
LOG.info(
f"Authentication info not available for Source ID: {source_id}")
sources_network.set_source_status(str(error))
def test_get_source_type(self):
"""Test to source type from source."""
test_source_id = 3
ocp_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
offset=3,
source_type=Provider.PROVIDER_OCP,
name="Test OCP Source",
authentication={"role_arn": "arn:test"},
billing_source={"bucket": "test-bucket"},
)
ocp_obj.save()
response = storage.get_source_type(test_source_id)
self.assertEqual(response, Provider.PROVIDER_OCP)
self.assertEqual(storage.get_source_type(test_source_id + 1), None)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if source_type:
sources_network = SourcesHTTPClient(auth_header, source_id)
else:
LOG.info(f'Source ID not found for ID: {source_id}')
return
try:
if source_type == 'OCP':
source_details = sources_network.get_source_details()
authentication = {'resource_name': source_details.get('uid')}
elif source_type == 'AWS':
authentication = {
'resource_name': sources_network.get_aws_role_arn()
}
elif source_type == 'AZURE':
authentication = {
'credentials': sources_network.get_azure_credentials()
}
else:
LOG.error(f'Unexpected source type: {source_type}')
return
storage.add_provider_sources_auth_info(source_id, authentication)
except SourcesHTTPClientError:
LOG.info(
f'Authentication info not available for Source ID: {source_id}')
def process(self):
"""Process the message."""
# We have no `self.event_type in (Source.X,)` statements here because we will only
# process Source.update. All non-update events are filtered in `msg_for_cost_mgmt`
if not storage.is_known_source(self.source_id):
LOG.info(
"[SourceMsgProcessor] update event for unknown source_id, skipping..."
)
return
updated = self.save_sources_details()
if storage.get_source_type(self.source_id) == Provider.PROVIDER_OCP:
updated |= self.save_source_info(auth=True)
if updated:
LOG.info(
f"[SourceMsgProcessor] source_id {self.source_id} updated")
storage.enqueue_source_create_or_update(self.source_id)
else:
LOG.info(
f"[SourceMsgProcessor] source_id {self.source_id} not updated. No changes detected."
)
def save_auth_info(auth_header, source_id):
"""
Store Sources Authentication information given an Source ID.
This method is called when a Cost Management application is
attached to a given Source as well as when an Authentication
is created. We have to handle both cases since an
Authentication.create event can occur before a Source is
attached to the Cost Management application.
Authentication is stored in the Sources database table.
Args:
source_id (Integer): Platform Sources ID.
auth_header (String): Authentication Header.
Returns:
None
"""
source_type = storage.get_source_type(source_id)
if not source_type:
LOG.info(f"Source ID not found for ID: {source_id}")
return
sources_network = SourcesHTTPClient(auth_header, source_id)
try:
authentication = get_authentication(source_type, sources_network)
except SourcesHTTPClientError as error:
LOG.info(
f"Authentication info not available for Source ID: {source_id}")
sources_network.set_source_status(error)
else:
if not authentication:
return
storage.add_provider_sources_auth_info(source_id, authentication)
storage.clear_update_flag(source_id)
LOG.info(f"Authentication attached to Source ID: {source_id}")
def save_credentials(self):
"""Store Sources Authentication information."""
LOG.info(f"[save_credentials] starting for source_id {self.source_id} ...")
source_type = storage.get_source_type(self.source_id)
if not source_type:
LOG.info(f"[save_credentials] source_type not found for source_id: {self.source_id}")
return
sources_network = self.get_sources_client()
try:
authentication = {"credentials": sources_network.get_credentials(source_type)}
except SourcesHTTPClientError as error:
LOG.info(f"[save_credentials] authentication info not available for source_id: {self.source_id}")
sources_network.set_source_status(error)
raise error
else:
if not authentication.get("credentials"): # TODO: is this check needed?
return
result = bool(storage.add_provider_sources_auth_info(self.source_id, authentication))
LOG.info(f"[save_credentials] complete for source_id: {self.source_id}: {result}")
return result
