def delete_source(source_id, auth_header, koku_uuid):
"""Delete Provider and Source."""
LOG.info(f"Deactivating Provider {koku_uuid}")
mark_provider_as_inactive(koku_uuid)
LOG.info(f"Deleting Provider {koku_uuid} for Source ID: {source_id}")
coordinator = SourcesProviderCoordinator(source_id, auth_header)
coordinator.destroy_account(koku_uuid)
def test_execute_koku_provider_op_destroy_provider_not_found(self):
"""Test to execute Koku Operations to sync with Sources for destruction with provider missing.
First, raise ProviderBuilderError. Check that provider and source still exists.
Then, re-call provider destroy without exception, then see both source and provider are gone.
"""
source_id = self.source_ids.get(Provider.PROVIDER_AWS)
provider = Sources(**self.sources.get(Provider.PROVIDER_AWS))
provider.save()
# check that the source exists
self.assertTrue(Sources.objects.filter(source_id=source_id).exists())
with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True):
builder = SourcesProviderCoordinator(source_id, provider.auth_header)
builder.create_account(provider)
self.assertTrue(Provider.objects.filter(uuid=provider.source_uuid).exists())
provider = Sources.objects.get(source_id=source_id)
msg = {"operation": "destroy", "provider": provider, "offset": provider.offset}
with patch.object(SourcesHTTPClient, "set_source_status"):
with patch.object(ProviderBuilder, "destroy_provider", side_effect=raise_provider_manager_error):
source_integration.execute_koku_provider_op(msg)
self.assertTrue(Provider.objects.filter(uuid=provider.source_uuid).exists())
self.assertTrue(Sources.objects.filter(source_uuid=provider.source_uuid).exists())
self.assertTrue(Sources.objects.filter(koku_uuid=provider.source_uuid).exists())
with patch.object(SourcesHTTPClient, "set_source_status"):
source_integration.execute_koku_provider_op(msg)
self.assertFalse(Provider.objects.filter(uuid=provider.source_uuid).exists())
def update_source_name(self):
"""Update source name if it is out of sync with platform."""
source_details = self.sources_client.get_source_details()
if source_details.get("name") != self.source.name:
self.source.name = source_details.get("name")
self.source.save()
builder = SourcesProviderCoordinator(self.source_id, self.source.auth_header)
builder.update_account(self.source)
def test_execute_koku_provider_op_update(self):
"""Test to execute Koku Operations to sync with Sources for update."""
def set_status_helper(*args, **kwargs):
"""helper to clear update flag."""
storage.clear_update_flag(source_id)
source_id = self.source_ids.get(Provider.PROVIDER_AWS)
provider = Sources(**self.sources.get(Provider.PROVIDER_AWS))
provider.save()
msg = {"operation": "create", "provider": provider, "offset": provider.offset}
with patch.object(SourcesHTTPClient, "set_source_status"):
with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True):
source_integration.execute_koku_provider_op(msg)
builder = SourcesProviderCoordinator(source_id, provider.auth_header)
source = storage.get_source_instance(source_id)
uuid = source.koku_uuid
with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True):
builder.update_account(source)
self.assertEqual(
Provider.objects.get(uuid=uuid).billing_source.data_source,
self.sources.get(Provider.PROVIDER_AWS).get("billing_source").get("data_source"),
)
provider.billing_source = {"data_source": {"bucket": "new-bucket"}}
provider.koku_uuid = uuid
provider.pending_update = True
provider.save()
msg = {"operation": "update", "provider": provider, "offset": provider.offset}
with patch.object(SourcesHTTPClient, "set_source_status", side_effect=set_status_helper):
with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True):
source_integration.execute_koku_provider_op(msg)
response = Sources.objects.get(source_id=source_id)
self.assertEqual(response.pending_update, False)
self.assertEqual(response.billing_source, {"data_source": {"bucket": "new-bucket"}})
response = Provider.objects.get(uuid=uuid)
self.assertEqual(response.billing_source.data_source.get("bucket"), "new-bucket")
def push_status(self):
"""Push status_msg to platform sources."""
try:
status_obj = self.status()
if self.source.source_type in (Provider.PROVIDER_GCP,
Provider.PROVIDER_GCP_LOCAL):
builder = SourcesProviderCoordinator(self.source.source_id,
self.source.auth_header)
if not status_obj:
if self.source.koku_uuid:
builder.update_account(self.source)
elif self.source.billing_source.get("data_source",
{}).get("table_id"):
builder.create_account(self.source)
self.sources_client.set_source_status(status_obj)
self.update_source_name()
LOG.info(
f"Source status for Source ID: {str(self.source_id)}: Status: {str(status_obj)}"
)
except SkipStatusPush as error:
LOG.info(
f"Platform sources status push skipped. Reason: {str(error)}")
except SourcesHTTPClientError as error:
err_msg = "Unable to push source status. Reason: {}".format(
str(error))
LOG.warning(err_msg)
def execute_koku_provider_op(msg):
"""
Execute the 'create' or 'destroy Koku-Provider operations.
'create' operations:
Koku POST /providers is executed along with updating the Sources database table with
the Koku Provider uuid.
'destroy' operations:
Koku DELETE /providers is executed along with removing the Sources database entry.
Two types of exceptions are handled for Koku HTTP operations. Recoverable client and
Non-Recoverable client errors. If the error is recoverable the calling function
(synchronize_sources) will re-queue the operation.
Args:
msg (Asyncio msg): Dictionary messages containing operation,
provider and offset.
example: {'operation': 'create', 'provider': SourcesModelObj, 'offset': 3}
Returns:
None
"""
provider = msg.get("provider")
operation = msg.get("operation")
account_coordinator = SourcesProviderCoordinator(provider.source_id, provider.auth_header)
sources_client = SourcesHTTPClient(provider.auth_header, provider.source_id)
try:
if operation == "create":
LOG.info(f"Creating Koku Provider for Source ID: {str(provider.source_id)}")
instance = account_coordinator.create_account(provider)
LOG.info(f"Creating provider {instance.uuid} for Source ID: {provider.source_id}")
elif operation == "update":
instance = account_coordinator.update_account(provider)
LOG.info(f"Updating provider {instance.uuid} for Source ID: {provider.source_id}")
elif operation == "destroy":
account_coordinator.destroy_account(provider)
LOG.info(f"Destroying provider {provider.koku_uuid} for Source ID: {provider.source_id}")
else:
LOG.error(f"unknown operation: {operation}")
sources_client.set_source_status(None)
except SourcesProviderCoordinatorError as account_error:
raise SourcesIntegrationError("Koku provider error: ", str(account_error))
except ValidationError as account_error:
err_msg = (
f"Unable to {operation} provider for Source ID: {str(provider.source_id)}. Reason: {str(account_error)}"
)
LOG.warning(err_msg)
sources_client.set_source_status(account_error)
def push_status(self):
"""Push status_msg to platform sources."""
try:
status_obj = self.status()
if self._gcp_bigquery_table_found():
builder = SourcesProviderCoordinator(self.source.source_id,
self.source.auth_header)
if self.source.koku_uuid:
builder.update_account(self.source)
else:
builder.create_account(self.source)
self.sources_client.set_source_status(status_obj)
self.update_source_name()
LOG.info(
f"Source status for Source ID: {str(self.source_id)}: Status: {str(status_obj)}"
)
except SkipStatusPush as error:
LOG.info(
f"Platform sources status push skipped. Reason: {str(error)}")
except SourcesHTTPClientError as error:
err_msg = "Unable to push source status. Reason: {}".format(
str(error))
LOG.warning(err_msg)