def process(self): """Process the message.""" if self.event_type in (KAFKA_APPLICATION_CREATE,): storage.create_source_event(self.source_id, self.auth_header, self.offset) if storage.is_known_source(self.source_id): if self.event_type in (KAFKA_APPLICATION_CREATE,): self.save_sources_details() self.save_source_info(bill=True) # _Authentication_ messages are responsible for saving credentials. # However, OCP does not send an Auth message. Therefore, we need # to run the following branch for OCP which completes the source # creation cycle for an OCP source. if storage.get_source_type(self.source_id) == Provider.PROVIDER_OCP: self.save_source_info(auth=True) if self.event_type in (KAFKA_APPLICATION_UPDATE,): if storage.get_source_type(self.source_id) == Provider.PROVIDER_AZURE: # Because azure auth is split in Sources backend, we need to check both # auth and billing when we recieve either auth update or app update event updated = self.save_source_info(auth=True, bill=True) else: updated = self.save_source_info(bill=True) if updated: LOG.info(f"[ApplicationMsgProcessor] source_id {self.source_id} updated") storage.enqueue_source_create_or_update(self.source_id) else: LOG.info(f"[ApplicationMsgProcessor] source_id {self.source_id} not updated. No changes detected.") if self.event_type in (KAFKA_APPLICATION_DESTROY,): storage.enqueue_source_delete(self.source_id, self.offset, allow_out_of_order=True)
def test_get_source_type(self): """Test to source type from source.""" test_source_id = 3 ocp_obj = Sources(source_id=test_source_id, auth_header=self.test_header, offset=3, source_type=Provider.PROVIDER_OCP, name='Test OCP Source', authentication={'resource_name': 'arn:test'}, billing_source={'bucket': 'test-bucket'}) ocp_obj.save() response = storage.get_source_type(test_source_id) self.assertEquals(response, Provider.PROVIDER_OCP) self.assertEquals(storage.get_source_type(test_source_id + 1), None)
def save_billing_source(self): """Store Sources billing information.""" LOG.info(f"[save_billing_source] starting for source_id {self.source_id} ...") source_type = storage.get_source_type(self.source_id) if not source_type: LOG.info(f"[save_billing_source] source_type not found for source_id: {self.source_id}") return if source_type == Provider.PROVIDER_OCP: # OCP sources do not have billing sources, so skip running thru this function LOG.info("[save_billing_source] skipping for OCP source") return sources_network = self.get_sources_client() try: data_source = {"data_source": sources_network.get_data_source(source_type)} except SourcesHTTPClientError as error: LOG.info(f"[save_billing_source] billing info not available for source_id: {self.source_id}") sources_network.set_source_status(error) raise error else: if not data_source.get("data_source"): return result = bool(storage.add_provider_sources_billing_info(self.source_id, data_source)) LOG.info(f"[save_billing_source] completed for source_id: {self.source_id}: {result}") return result
def process(self): """Process the message.""" if self.event_type in (KAFKA_AUTHENTICATION_CREATE): LOG.debug( f"[AuthenticationMsgProcessor] creating source for source_id: {self.source_id}" ) storage.create_source_event(self.source_id, self.account_number, self.auth_header, self.offset) if storage.is_known_source(self.source_id): if self.event_type in (KAFKA_AUTHENTICATION_CREATE): self.save_source_info(auth=True) if self.event_type in (KAFKA_AUTHENTICATION_UPDATE): if storage.get_source_type( self.source_id) == Provider.PROVIDER_AZURE: # Because azure auth is split in Sources backend, we need to check both # auth and billing when we recieve either auth update or app update event updated = self.save_source_info(auth=True, bill=True) else: updated = self.save_source_info(auth=True) if updated: LOG.info( f"[AuthenticationMsgProcessor] source_id {self.source_id} updated" ) storage.enqueue_source_create_or_update(self.source_id) else: LOG.info( f"[AuthenticationMsgProcessor] source_id {self.source_id} not updated. No changes detected." )
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f'Source ID not found for ID: {source_id}') return try: if source_type == 'OCP': source_details = sources_network.get_source_details() # Check for imported to maintain temporary backwards compatibility # until the Sources Front End creates 'imported' entry with OCP Cluster ID. if source_details.get('source_ref'): authentication = { 'resource_name': source_details.get('source_ref') } else: uid = source_details.get('uid') LOG.info( f'OCP is using fallback Source UID ({str(uid)} for authentication.' ' Update frontend to add Cluster ID to the source_ref field on the Source.' ) authentication = {'resource_name': uid} elif source_type == 'AWS': authentication = { 'resource_name': sources_network.get_aws_role_arn() } elif source_type == 'AZURE': authentication = { 'credentials': sources_network.get_azure_credentials() } else: LOG.error(f'Unexpected source type: {source_type}') return storage.add_provider_sources_auth_info(source_id, authentication) except SourcesHTTPClientError: LOG.info( f'Authentication info not available for Source ID: {source_id}')
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f"Source ID not found for ID: {source_id}") return try: if source_type == Provider.PROVIDER_OCP: source_details = sources_network.get_source_details() if source_details.get("source_ref"): authentication = { "resource_name": source_details.get("source_ref") } else: raise SourcesHTTPClientError("Unable to find Cluster ID") elif source_type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): authentication = { "resource_name": sources_network.get_aws_role_arn() } elif source_type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): authentication = { "credentials": sources_network.get_azure_credentials() } else: LOG.error(f"Unexpected source type: {source_type}") return storage.add_provider_sources_auth_info(source_id, authentication) storage.clear_update_flag(source_id) LOG.info(f"Authentication attached to Source ID: {source_id}") except SourcesHTTPClientError as error: LOG.info( f"Authentication info not available for Source ID: {source_id}") sources_network.set_source_status(str(error))
def test_get_source_type(self): """Test to source type from source.""" test_source_id = 3 ocp_obj = Sources( source_id=test_source_id, auth_header=self.test_header, offset=3, source_type=Provider.PROVIDER_OCP, name="Test OCP Source", authentication={"role_arn": "arn:test"}, billing_source={"bucket": "test-bucket"}, ) ocp_obj.save() response = storage.get_source_type(test_source_id) self.assertEqual(response, Provider.PROVIDER_OCP) self.assertEqual(storage.get_source_type(test_source_id + 1), None)
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if source_type: sources_network = SourcesHTTPClient(auth_header, source_id) else: LOG.info(f'Source ID not found for ID: {source_id}') return try: if source_type == 'OCP': source_details = sources_network.get_source_details() authentication = {'resource_name': source_details.get('uid')} elif source_type == 'AWS': authentication = { 'resource_name': sources_network.get_aws_role_arn() } elif source_type == 'AZURE': authentication = { 'credentials': sources_network.get_azure_credentials() } else: LOG.error(f'Unexpected source type: {source_type}') return storage.add_provider_sources_auth_info(source_id, authentication) except SourcesHTTPClientError: LOG.info( f'Authentication info not available for Source ID: {source_id}')
def process(self): """Process the message.""" # We have no `self.event_type in (Source.X,)` statements here because we will only # process Source.update. All non-update events are filtered in `msg_for_cost_mgmt` if not storage.is_known_source(self.source_id): LOG.info( "[SourceMsgProcessor] update event for unknown source_id, skipping..." ) return updated = self.save_sources_details() if storage.get_source_type(self.source_id) == Provider.PROVIDER_OCP: updated |= self.save_source_info(auth=True) if updated: LOG.info( f"[SourceMsgProcessor] source_id {self.source_id} updated") storage.enqueue_source_create_or_update(self.source_id) else: LOG.info( f"[SourceMsgProcessor] source_id {self.source_id} not updated. No changes detected." )
def save_auth_info(auth_header, source_id): """ Store Sources Authentication information given an Source ID. This method is called when a Cost Management application is attached to a given Source as well as when an Authentication is created. We have to handle both cases since an Authentication.create event can occur before a Source is attached to the Cost Management application. Authentication is stored in the Sources database table. Args: source_id (Integer): Platform Sources ID. auth_header (String): Authentication Header. Returns: None """ source_type = storage.get_source_type(source_id) if not source_type: LOG.info(f"Source ID not found for ID: {source_id}") return sources_network = SourcesHTTPClient(auth_header, source_id) try: authentication = get_authentication(source_type, sources_network) except SourcesHTTPClientError as error: LOG.info( f"Authentication info not available for Source ID: {source_id}") sources_network.set_source_status(error) else: if not authentication: return storage.add_provider_sources_auth_info(source_id, authentication) storage.clear_update_flag(source_id) LOG.info(f"Authentication attached to Source ID: {source_id}")
def save_credentials(self): """Store Sources Authentication information.""" LOG.info(f"[save_credentials] starting for source_id {self.source_id} ...") source_type = storage.get_source_type(self.source_id) if not source_type: LOG.info(f"[save_credentials] source_type not found for source_id: {self.source_id}") return sources_network = self.get_sources_client() try: authentication = {"credentials": sources_network.get_credentials(source_type)} except SourcesHTTPClientError as error: LOG.info(f"[save_credentials] authentication info not available for source_id: {self.source_id}") sources_network.set_source_status(error) raise error else: if not authentication.get("credentials"): # TODO: is this check needed? return result = bool(storage.add_provider_sources_auth_info(self.source_id, authentication)) LOG.info(f"[save_credentials] complete for source_id: {self.source_id}: {result}") return result