def setUp(self):
"""Setup tests."""
super().setUp()
self.test_account = '10001'
user_data = self._create_user_data()
customer = self._create_customer_data(account=self.test_account)
self.request_context = self._create_request_context(
customer, user_data, create_customer=True, is_admin=False)
self.test_source_id = 1
self.azure_obj = Sources(
source_id=self.test_source_id,
auth_header=self.request_context['request'].META,
account_id=customer.get('account_id'),
offset=1,
source_type='AZURE',
name='Test Azure Source',
authentication={
'credentials': {
'client_id': 'test_client',
'tenant_id': 'test_tenant',
'client_secret': 'test_secret'
}
})
self.azure_obj.save()
mock_url = PropertyMock(
return_value='http://www.sourcesclient.com/api/v1/sources/')
SourcesViewSet.url = mock_url
def create_source_event(source_id, auth_header, offset):
"""Create a Sources database object."""
LOG.info(f"[create_source_event] starting for source_id {source_id} ...")
try:
decoded_rh_auth = b64decode(auth_header)
json_rh_auth = json_loads(decoded_rh_auth)
account_id = json_rh_auth.get("identity", {}).get("account_number")
except (binascii.Error, JSONDecodeError) as error:
LOG.error(str(error))
return
try:
source = Sources.objects.filter(source_id=source_id).first()
if source:
LOG.info(
f"[create_source_event] source_id: {str(source_id)} already exists."
)
if source.out_of_order_delete:
LOG.info(
f"[create_source_event] source_id: {source_id} destroy event already occurred."
)
source.delete()
else:
new_event = Sources(source_id=source_id,
auth_header=auth_header,
offset=offset,
account_id=account_id)
new_event.save()
LOG.info(f"[create_source_event] created source_id: {source_id}")
except (InterfaceError, OperationalError) as error:
LOG.error(f"[create_source_event] {type(error).__name__}: {error}")
raise error
def test_sources_network_info_sync_ocp(self):
"""Test to get additional Source context from Sources API for OCP."""
test_source_id = 1
test_auth_header = Config.SOURCES_FAKE_HEADER
source_name = 'OCP Source'
source_uid = faker.uuid4()
ocp_source = Sources(source_id=test_source_id,
auth_header=test_auth_header,
offset=1)
ocp_source.save()
source_type_id = 3
resource_id = 2
authentication_id = 4
mock_source_name = 'openshift'
with requests_mock.mock() as m:
m.get(f'http://www.sources.com/api/v1.0/sources/{test_source_id}',
status_code=200, json={'name': source_name, 'source_type_id': source_type_id, 'uid': source_uid})
m.get(f'http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}',
status_code=200, json={'data': [{'name': mock_source_name}]})
m.get(f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={test_source_id}',
status_code=200, json={'data': [{'id': resource_id}]})
m.get((f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint'
f'&[authtype]=token&[resource_id]={resource_id}'),
status_code=200, json={'data': [{'id': authentication_id}]})
source_integration.sources_network_info(test_source_id, test_auth_header)
source_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(source_obj.name, source_name)
self.assertEqual(source_obj.source_type, 'OCP')
self.assertEqual(source_obj.authentication, {'resource_name': source_uid})
def test_sources_network_info_sync_aws(self):
"""Test to get additional Source context from Sources API for AWS."""
test_source_id = 2
test_auth_header = Config.SOURCES_FAKE_HEADER
source_name = 'AWS Source'
source_uid = faker.uuid4()
authentication = 'roleARNhere'
aws_source = Sources(source_id=test_source_id,
auth_header=test_auth_header,
offset=1)
aws_source.save()
source_type_id = 1
mock_source_name = 'amazon'
resource_id = 2
authentication_id = 3
with requests_mock.mock() as m:
m.get(f'http://www.sources.com/api/v1.0/sources/{test_source_id}',
status_code=200, json={'name': source_name, 'source_type_id': source_type_id, 'uid': source_uid})
m.get(f'http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}',
status_code=200, json={'data': [{'name': mock_source_name}]})
m.get(f'http://www.sources.com/api/v1.0/endpoints?filter[source_id]={test_source_id}',
status_code=200, json={'data': [{'id': resource_id}]})
m.get((f'http://www.sources.com/api/v1.0/authentications?filter[resource_type]=Endpoint'
f'&[authtype]=arn&[resource_id]={resource_id}'),
status_code=200, json={'data': [{'id': authentication_id}]})
m.get((f'http://www.sources.com/internal/v1.0/authentications/{authentication_id}'
f'?expose_encrypted_attribute[]=password'),
status_code=200, json={'password': authentication})
source_integration.sources_network_info(test_source_id, test_auth_header)
source_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(source_obj.name, source_name)
self.assertEqual(source_obj.source_type, 'AWS')
self.assertEqual(source_obj.authentication, {'resource_name': authentication})
def setUp(self):
"""Set up tests."""
super().setUp()
self.test_account = "10001"
user_data = self._create_user_data()
customer = self._create_customer_data(account=self.test_account)
self.request_context = self._create_request_context(customer, user_data, create_customer=True, is_admin=True)
self.test_source_id = 1
name = "Test Azure Source"
customer_obj = Customer.objects.get(account_id=customer.get("account_id"))
self.azure_provider = Provider(name=name, type=Provider.PROVIDER_AZURE, customer=customer_obj)
self.azure_provider.save()
self.azure_obj = Sources(
source_id=self.test_source_id,
auth_header=self.request_context["request"].META,
account_id=customer.get("account_id"),
offset=1,
source_type=Provider.PROVIDER_AZURE,
name=name,
authentication={
"credentials": {"client_id": "test_client", "tenant_id": "test_tenant", "client_secret": "test_secret"}
},
source_uuid=self.azure_provider.uuid,
)
self.azure_obj.save()
mock_url = PropertyMock(return_value="http://www.sourcesclient.com/api/v1/sources/")
SourcesViewSet.url = mock_url
def test_sources_network_auth_info_ocp_with_cluster_id(self):
"""Test to get authentication information from Sources backend for OCP with cluster_id."""
test_source_id = 2
test_resource_id = 1
application_type = 2
cluster_id = faker.uuid4()
test_auth_header = Config.SOURCES_FAKE_HEADER
ocp_source = Sources(
source_id=test_source_id,
auth_header=test_auth_header,
endpoint_id=test_resource_id,
source_type=Provider.PROVIDER_OCP,
offset=1,
)
ocp_source.save()
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/sources/{test_source_id}",
status_code=200,
json={"source_ref": cluster_id},
)
m.get(
f"http://www.sources.com/api/v1.0/application_types?filter[name]=/insights/platform/cost-management",
status_code=200,
json={"data": [{"id": application_type}]},
)
source_integration.sources_network_auth_info(test_resource_id, test_auth_header)
source_obj = Sources.objects.get(source_id=test_source_id)
self.assertEquals(source_obj.authentication, {"resource_name": cluster_id})
def test_execute_koku_provider_op_destroy(self):
"""Test to execute Koku Operations to sync with Sources for destruction."""
source_id = 1
auth_header = Config.SOURCES_FAKE_HEADER
offset = 2
mock_koku_uuid = faker.uuid4()
provider = Sources(source_id=source_id,
auth_header=auth_header,
offset=offset,
koku_uuid=mock_koku_uuid)
provider.save()
with requests_mock.mock() as m:
m.delete(
f'http://www.koku.com/api/cost-management/v1/providers/{mock_koku_uuid}/',
status_code=204)
msg = {
'operation': 'destroy',
'provider': provider,
'offset': provider.offset
}
source_integration.execute_koku_provider_op(msg)
self.assertEqual(
Sources.objects.filter(source_id=source_id).exists(), False)
def test_sources_network_auth_info_error(self):
"""Test to get authentication information from Sources backend with error."""
test_source_id = 2
test_resource_id = 1
application_type_id = 2
app_id = 1
test_auth_header = Config.SOURCES_FAKE_HEADER
ocp_source = Sources(
source_id=test_source_id,
auth_header=test_auth_header,
endpoint_id=test_resource_id,
source_type=Provider.PROVIDER_OCP,
offset=1,
)
ocp_source.save()
with requests_mock.mock() as m:
m.get(f"http://www.sources.com/api/v1.0/sources/{test_source_id}", status_code=400)
m.get(
SOURCES_APPS.format(application_type_id, test_source_id),
status_code=200,
json={"data": [{"id": app_id}]},
)
m.get(
f"http://www.sources.com/api/v1.0/application_types?filter[name]=/insights/platform/cost-management",
status_code=200,
json={"data": [{"id": application_type_id}]},
)
m.patch(f"http://www.sources.com/api/v1.0/applications/{app_id}", status_code=204)
source_integration.sources_network_auth_info(test_resource_id, test_auth_header)
source_obj = Sources.objects.get(source_id=test_source_id)
self.assertEquals(source_obj.authentication, {})
def test_post_authentications_malformed_json(self):
"""Test the POT authentication endpoint for a Azure provider with bad json data."""
subscription_id = {'not-subscription': 'test-subscription-id'}
params = {'source_id': 1, 'credentials': subscription_id}
test_name = 'Azure Test'
credential_dict = {
'credentials': {
'client_id': 'test_client',
'tenant_id': 'test_tenant',
'client_secret': 'test_secret'
}
}
azure_obj = Sources(source_id=self.test_source_id,
auth_header=self.test_header,
offset=self.test_offset,
source_type='AZURE',
name=test_name,
authentication=credential_dict,
billing_source={
'data_source': {
'resource_group': 'RG1',
'storage_account': 'test_storage'
}
})
azure_obj.save()
response = self.client.post(reverse('authentication'),
json.dumps(params),
content_type='application/json')
body = response.json()
self.assertEqual(response.status_code, 400)
self.assertIn(str('Subscription ID not found'), str(body))
def test_sources_network_info_no_endpoint(self):
"""Test to get additional Source context from Sources API with no endpoint found."""
test_source_id = 1
mock_source_name = "source name"
source_type_id = 1
source_uid = faker.uuid4()
test_auth_header = Config.SOURCES_FAKE_HEADER
ocp_source = Sources(source_id=test_source_id, auth_header=test_auth_header, offset=1)
ocp_source.save()
with requests_mock.mock() as m:
m.get(
f"http://www.sources.com/api/v1.0/sources/{test_source_id}",
status_code=200,
json={"name": mock_source_name, "source_type_id": source_type_id, "uid": source_uid},
)
m.get(
f"http://www.sources.com/api/v1.0/source_types?filter[id]={source_type_id}",
status_code=200,
json={"data": [{"name": mock_source_name}]},
)
m.get(
f"http://www.sources.com/api/v1.0/endpoints?filter[source_id]={test_source_id}",
status_code=200,
json={"data": []},
)
source_integration.sources_network_info(test_source_id, test_auth_header)
source_obj = Sources.objects.get(source_id=test_source_id)
self.assertIsNone(source_obj.name)
self.assertEquals(source_obj.source_type, "")
self.assertEquals(source_obj.authentication, {})
def test_execute_koku_provider_op_destroy_non_recoverable_error(self):
"""Test to execute Koku Operations to sync with Sources with non-recoverable error."""
source_id = 1
app_id = 1
application_type_id = 2
auth_header = Config.SOURCES_FAKE_HEADER
offset = 2
provider = Sources(source_id=source_id, auth_header=auth_header, offset=offset)
provider.save()
logging.disable(logging.NOTSET)
with requests_mock.mock() as m:
m.post('http://www.koku.com/api/cost-management/v1/providers/',
status_code=400,
json={'errors': [{'detail': 'koku check failed'}]})
m.get('http://www.sources.com/api/v1.0/applications?filter[application_type_id]={}&filter[source_id]={}'.
format(application_type_id, source_id),
status_code=200, json={'data': [{'id': app_id}]})
m.patch(f'http://www.sources.com/api/v1.0/applications/{app_id}',
status_code=204)
with self.assertLogs('sources.kafka_listener', level='ERROR') as logger:
msg = {'operation': 'create', 'provider': provider, 'offset': provider.offset}
source_integration.execute_koku_provider_op(msg, application_type_id)
self.assertIn(':Unable to create provider for Source ID: 1', logger.output[0])
def test_execute_koku_provider_op_update(self):
"""Test to execute Koku Operations to sync with Sources for destruction."""
source_id = 1
app_id = 1
auth_header = Config.SOURCES_FAKE_HEADER
offset = 2
mock_koku_uuid = faker.uuid4()
application_type_id = 2
provider = Sources(
source_id=source_id, auth_header=auth_header, offset=offset, koku_uuid=mock_koku_uuid, pending_update=True
)
provider.save()
with requests_mock.mock() as m:
m.put(f"http://www.koku.com/api/cost-management/v1/providers/{mock_koku_uuid}/", status_code=200, json={})
m.get(
f"http://www.sources.com/api/v1.0/applications?filter[source_id]={source_id}",
status_code=200,
json={"data": [{"id": app_id}]},
)
m.patch(f"http://www.sources.com/api/v1.0/applications/{app_id}", status_code=204)
msg = {"operation": "update", "provider": provider, "offset": provider.offset}
source_integration.execute_koku_provider_op(msg, application_type_id)
response = Sources.objects.get(source_id=source_id)
self.assertEquals(response.pending_update, False)
def test_execute_koku_provider_op_destroy_non_recoverable_error(self):
"""Test to execute Koku Operations to sync with Sources with non-recoverable error."""
source_id = 1
app_id = 1
application_type_id = 2
auth_header = Config.SOURCES_FAKE_HEADER
offset = 2
provider = Sources(source_id=source_id, auth_header=auth_header, offset=offset)
provider.save()
logging.disable(logging.NOTSET)
with requests_mock.mock() as m:
m.post(
"http://www.koku.com/api/cost-management/v1/providers/",
status_code=400,
json={"errors": [{"detail": "koku check failed"}]},
)
m.get(
SOURCES_APPS.format(application_type_id, source_id), status_code=200, json={"data": [{"id": app_id}]}
)
m.patch(f"http://www.sources.com/api/v1.0/applications/{app_id}", status_code=204)
with self.assertLogs("sources.kafka_listener", level="ERROR") as logger:
msg = {"operation": "create", "provider": provider, "offset": provider.offset}
source_integration.execute_koku_provider_op(msg, application_type_id)
self.assertIn(":Unable to create provider for Source ID: 1", logger.output[0])
def test_get_query_from_api_data(self):
"""Test helper method to get query based on API request_data."""
test_source_id = 3
test_source_name = 'Test AWS Source'
test_endpoint_id = 4
aws_obj = Sources(source_id=test_source_id,
auth_header=self.test_header,
offset=3,
endpoint_id=test_endpoint_id,
source_type='AWS',
name=test_source_name,
billing_source={'bucket': 'test-bucket'})
aws_obj.save()
test_matrix = [{'request_data': {'source_id': test_source_id}, 'expected_exception': None},
{'request_data': {'source_name': test_source_name}, 'expected_exception': None},
{'request_data': {'source_id': test_source_id, 'source_name': test_source_name},
'expected_exception': SourcesStorageError}]
for test in test_matrix:
if not test.get('expected_exception'):
response = storage.get_query_from_api_data(test.get('request_data'))
self.assertEquals(response.source_id, test_source_id)
else:
with self.assertRaises(test.get('expected_exception')):
storage.get_query_from_api_data(test.get('request_data'))
def test_add_subscription_id_to_credentials(self):
"""Test to add subscription_id to AZURE credentials."""
test_source_id = 2
subscription_id = 'test_sub_id'
azure_obj = Sources(source_id=test_source_id,
auth_header=self.test_header,
offset=2,
source_type='AZURE',
name='Test Azure Source',
authentication={
'credentials': {
'client_id': 'test_client',
'tenant_id': 'test_tenant',
'client_secret': 'test_secret'
}
},
billing_source={
'data_source': {
'resource_group': 'RG1',
'storage_account': 'test_storage'
}
})
azure_obj.save()
storage.add_subscription_id_to_credentials(test_source_id,
subscription_id)
response_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(
response_obj.authentication.get('credentials').get(
'subscription_id'), subscription_id)
def test_update_application_settings(self):
"""Test to update application settings."""
test_source_id = 3
resource_group = "testrg"
subscription_id = "testsubid"
settings = {
"billing_source": {
"data_source": {
"resource_group": resource_group
}
},
"authentication": {
"subscription_id": subscription_id
},
}
azure_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
offset=3,
source_type=Provider.PROVIDER_AZURE,
name="Test AZURE Source",
)
azure_obj.save()
storage.update_application_settings(test_source_id, settings)
db_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(db_obj.authentication.get("subscription_id"),
subscription_id)
self.assertEqual(
db_obj.billing_source.get("data_source").get("resource_group"),
resource_group)
def test_save_status(self):
"""Test to verify source status is saved."""
test_source_id = 3
status = "unavailable"
user_facing_string = "Missing credential and billing source"
mock_status = {
"availability_status": status,
"availability_status_error": user_facing_string
}
azure_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
offset=3,
source_type=Provider.PROVIDER_AZURE,
name="Test AZURE Source",
)
azure_obj.save()
return_code = storage.save_status(test_source_id, mock_status)
db_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(db_obj.status, mock_status)
self.assertTrue(return_code)
# Save again and verify return_code is False
return_code = storage.save_status(test_source_id, mock_status)
db_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(db_obj.status, mock_status)
self.assertFalse(return_code)
def test_execute_koku_provider_op_destroy_provider_not_found(self):
"""Test to execute Koku Operations to sync with Sources for destruction with provider missing.
First, raise ProviderBuilderError. Check that provider and source still exists.
Then, re-call provider destroy without exception, then see both source and provider are gone.
"""
source_id = self.source_ids.get(Provider.PROVIDER_AWS)
provider = Sources(**self.sources.get(Provider.PROVIDER_AWS))
provider.save()
# check that the source exists
self.assertTrue(Sources.objects.filter(source_id=source_id).exists())
with patch.object(ProviderAccessor, "cost_usage_source_ready", returns=True):
builder = SourcesProviderCoordinator(source_id, provider.auth_header)
builder.create_account(provider)
self.assertTrue(Provider.objects.filter(uuid=provider.source_uuid).exists())
provider = Sources.objects.get(source_id=source_id)
msg = {"operation": "destroy", "provider": provider, "offset": provider.offset}
with patch.object(SourcesHTTPClient, "set_source_status"):
with patch.object(ProviderBuilder, "destroy_provider", side_effect=raise_provider_manager_error):
source_integration.execute_koku_provider_op(msg)
self.assertTrue(Provider.objects.filter(uuid=provider.source_uuid).exists())
self.assertTrue(Sources.objects.filter(source_uuid=provider.source_uuid).exists())
self.assertTrue(Sources.objects.filter(koku_uuid=provider.source_uuid).exists())
with patch.object(SourcesHTTPClient, "set_source_status"):
source_integration.execute_koku_provider_op(msg)
self.assertFalse(Provider.objects.filter(uuid=provider.source_uuid).exists())
def test_clear_update_flag(self):
"""Test for clearing source update flag."""
test_matrix = [
{"koku_uuid": None, "pending_update": False, "expected_pending_update": False},
{"koku_uuid": faker.uuid4(), "pending_update": False, "expected_pending_update": False},
{"koku_uuid": faker.uuid4(), "pending_update": True, "expected_pending_update": False},
]
test_source_id = 3
for test in test_matrix:
aws_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
koku_uuid=test.get("koku_uuid"),
pending_update=test.get("pending_update"),
offset=3,
source_type=Provider.PROVIDER_AWS,
name="Test AWS Source",
billing_source={"bucket": "test-bucket"},
)
aws_obj.save()
storage.clear_update_flag(test_source_id)
response = Sources.objects.get(source_id=test_source_id)
self.assertEquals(test.get("expected_pending_update"), response.pending_update)
test_source_id += 1
def test_execute_koku_provider_op_destroy_non_recoverable_error(self):
"""Test to execute Koku Operations to sync with Sources with non-recoverable error."""
source_id = 1
auth_header = Config.SOURCES_FAKE_HEADER
offset = 2
provider = Sources(source_id=source_id,
auth_header=auth_header,
offset=offset)
provider.save()
logging.disable(logging.NOTSET)
with requests_mock.mock() as m:
m.post('http://www.koku.com/api/cost-management/v1/providers/',
status_code=400,
json={'uuid': faker.uuid4()})
with self.assertLogs('sources.kafka_listener',
level='ERROR') as logger:
msg = {
'operation': 'create',
'provider': provider,
'offset': provider.offset
}
source_integration.execute_koku_provider_op(msg)
self.assertIn(':Unable to create provider for Source ID: 1',
logger.output[0])
def test_add_provider_sources_auth_info_with_sub_id(self):
"""Test to add authentication to a source with subscription_id."""
test_source_id = 3
test_endpoint_id = 4
test_authentication = {"credentials": {"client_id": "new-client-id"}}
azure_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
offset=3,
endpoint_id=test_endpoint_id,
source_type=Provider.PROVIDER_AZURE,
name="Test AZURE Source",
authentication={
"credentials": {
"subscription_id": "orig-sub-id",
"client_id": "test-client-id"
}
},
)
azure_obj.save()
storage.add_provider_sources_auth_info(test_source_id,
test_authentication)
response = Sources.objects.filter(source_id=test_source_id).first()
self.assertEquals(
response.authentication.get("credentials").get("subscription_id"),
"orig-sub-id")
self.assertEquals(
response.authentication.get("credentials").get("client_id"),
"new-client-id")
def test_execute_koku_provider_op_update(self):
"""Test to execute Koku Operations to sync with Sources for destruction."""
source_id = 1
auth_header = Config.SOURCES_FAKE_HEADER
offset = 2
mock_koku_uuid = faker.uuid4()
provider = Sources(source_id=source_id,
auth_header=auth_header,
offset=offset,
koku_uuid=mock_koku_uuid,
pending_update=True)
provider.save()
with requests_mock.mock() as m:
m.put(
f'http://www.koku.com/api/cost-management/v1/providers/{mock_koku_uuid}/',
status_code=200,
json={})
msg = {
'operation': 'update',
'provider': provider,
'offset': provider.offset
}
source_integration.execute_koku_provider_op(msg)
response = Sources.objects.get(source_id=source_id)
self.assertEquals(response.pending_update, False)
def test_clear_update_flag(self):
"""Test for clearing source update flag."""
test_matrix = [{
'koku_uuid': None,
'pending_update': False,
'expected_pending_update': False
}, {
'koku_uuid': faker.uuid4(),
'pending_update': False,
'expected_pending_update': False
}, {
'koku_uuid': faker.uuid4(),
'pending_update': True,
'expected_pending_update': False
}]
test_source_id = 3
for test in test_matrix:
aws_obj = Sources(source_id=test_source_id,
auth_header=self.test_header,
koku_uuid=test.get('koku_uuid'),
pending_update=test.get('pending_update'),
offset=3,
endpoint_id=4,
source_type=Provider.PROVIDER_AWS,
name='Test AWS Source',
billing_source={'bucket': 'test-bucket'})
aws_obj.save()
storage.clear_update_flag(test_source_id)
response = Sources.objects.get(source_id=test_source_id)
self.assertEquals(test.get('expected_pending_update'),
response.pending_update)
test_source_id += 1
def setUp(self):
"""Set up tests."""
super().setUp()
customer = self._create_customer_data()
self.azure_name = 'Test Azure Source'
azure_user_data = self._create_user_data()
self.azure_request_context = self._create_request_context(customer, azure_user_data, create_customer=True,
is_admin=False)
self.test_azure_source_id = 1
self.azure_obj = Sources(source_id=self.test_azure_source_id,
auth_header=self.azure_request_context['request'].META,
account_id=customer.get('account_id'),
offset=1,
source_type=Provider.PROVIDER_AZURE,
name=self.azure_name,
authentication={'credentials': {'client_id': 'test_client',
'tenant_id': 'test_tenant',
'client_secret': 'test_secret'}})
self.azure_obj.save()
self.aws_name = 'Test AWS Source'
aws_user_data = self._create_user_data()
self.aws_request_context = self._create_request_context(customer, aws_user_data, create_customer=True,
is_admin=False)
self.test_aws_source_id = 2
self.aws_obj = Sources(source_id=self.test_aws_source_id,
auth_header=self.aws_request_context['request'].META,
account_id=customer.get('account_id'),
offset=2,
source_type=Provider.PROVIDER_AWS,
name=self.aws_name)
self.aws_obj.save()
def test_add_provider_sources_auth_info_with_sub_id(self):
"""Test to add authentication to a source with subscription_id."""
test_source_id = 3
test_endpoint_id = 4
test_authentication = {'credentials': {'client_id': 'new-client-id'}}
azure_obj = Sources(source_id=test_source_id,
auth_header=self.test_header,
offset=3,
endpoint_id=test_endpoint_id,
source_type=Provider.PROVIDER_AZURE,
name='Test AZURE Source',
authentication={
'credentials': {
'subscription_id': 'orig-sub-id',
'client_id': 'test-client-id'
}
})
azure_obj.save()
storage.add_provider_sources_auth_info(test_source_id,
test_authentication)
response = Sources.objects.filter(source_id=test_source_id).first()
self.assertEquals(
response.authentication.get('credentials').get('subscription_id'),
'orig-sub-id')
self.assertEquals(
response.authentication.get('credentials').get('client_id'),
'new-client-id')
def create_provider_event(source_id, auth_header, offset):
"""
Create a Sources database object.
Args:
source_id (Integer) - Platform-Sources identifier
auth_header (String) - HTTP Authentication Header
offset (Integer) - Kafka offset
Returns:
None
"""
try:
decoded_rh_auth = b64decode(auth_header)
json_rh_auth = json_loads(decoded_rh_auth)
account_id = json_rh_auth.get('identity', {}).get('account_number')
except (binascii.Error, JSONDecodeError) as error:
LOG.error(str(error))
return
try:
Sources.objects.get(source_id=source_id)
LOG.debug(f'Source ID {str(source_id)} already exists.')
except Sources.DoesNotExist:
new_event = Sources(source_id=source_id, auth_header=auth_header,
offset=offset, account_id=account_id)
new_event.save()
def enqueue_source_delete(source_id, offset, allow_out_of_order=False):
"""
Queues a source destroy event to be processed by the synchronize_sources method.
Args:
queue (Asyncio Queue) - process_queue containing all pending Souces-koku events.
source_id (Integer) - Platform-Sources identifier.
allow_out_of_order (Bool) - Allow for out of order delete events (Application or Source).
Returns:
None
"""
try:
source = Sources.objects.get(source_id=source_id)
if not source.pending_delete and not source.out_of_order_delete:
source.pending_delete = True
source.save()
except Sources.DoesNotExist:
if allow_out_of_order:
LOG.info(
f"Source ID: {source_id} not known. Marking as out of order delete."
)
new_event = Sources(source_id=source_id,
offset=offset,
out_of_order_delete=True)
new_event.save()
LOG.info(
f"source.storage.create_source_event created Source ID as pending delete: {source_id}"
)
except (InterfaceError, OperationalError) as error:
LOG.error(
f"Accessing sources resulted in {type(error).__name__}: {error}")
raise error
def create_source_event(source_id, auth_header, offset):
"""
Create a Sources database object.
Args:
source_id (Integer) - Platform-Sources identifier
auth_header (String) - HTTP Authentication Header
offset (Integer) - Kafka offset
Returns:
None
"""
try:
decoded_rh_auth = b64decode(auth_header)
json_rh_auth = json_loads(decoded_rh_auth)
account_id = json_rh_auth.get("identity", {}).get("account_number")
except (binascii.Error, JSONDecodeError) as error:
LOG.error(str(error))
return
try:
source = Sources.objects.filter(source_id=source_id).first()
if source:
LOG.debug(f"Source ID {str(source_id)} already exists.")
if source.out_of_order_delete:
LOG.info(f"Source ID: {source_id} destroy event already occurred.")
source.delete()
else:
new_event = Sources(source_id=source_id, auth_header=auth_header, offset=offset, account_id=account_id)
new_event.save()
LOG.info(f"source.storage.create_source_event created Source ID: {source_id}")
except (InterfaceError, OperationalError) as error:
LOG.error(f"source.storage.create_provider_event {type(error).__name__}: {error}")
raise error
def test_update_application_settings_billing_and_auth_starting_with_billing(
self):
"""Test to update application settings with both billing and auth where billing already exists."""
test_source_id = 3
subscription_id = "testsubid"
tenant_id = "testtenant"
client_id = "myclientid"
client_secret = "mysecret"
resource_group = "myrg"
storage_account = "mysa"
settings = {
"billing_source": {
"data_source": {
"storage_account": storage_account
}
},
"authentication": {
"credentials": {
"subscription_id": subscription_id,
"client_id": client_id,
"tenant_id": tenant_id,
"client_secret": client_secret,
}
},
}
azure_obj = Sources(
source_id=test_source_id,
auth_header=self.test_header,
offset=3,
billing_source={"data_source": {
"resource_group": resource_group
}},
source_type=Provider.PROVIDER_AZURE,
name="Test AZURE Source",
)
azure_obj.save()
storage.update_application_settings(test_source_id, settings)
db_obj = Sources.objects.get(source_id=test_source_id)
self.assertEqual(
db_obj.authentication.get("credentials").get("subscription_id"),
subscription_id)
self.assertEqual(
db_obj.authentication.get("credentials").get("client_secret"),
client_secret)
self.assertEqual(
db_obj.authentication.get("credentials").get("client_id"),
client_id)
self.assertEqual(
db_obj.authentication.get("credentials").get("tenant_id"),
tenant_id)
self.assertEqual(
db_obj.billing_source.get("data_source").get("resource_group"),
resource_group)
self.assertEqual(
db_obj.billing_source.get("data_source").get("storage_account"),
storage_account)
def test_storage_callback_create(self):
"""Test storage callback puts create task onto queue."""
local_source = Sources(**self.aws_local_source, pending_update=True)
local_source.save()
with patch("sources.kafka_listener.execute_process_queue"):
storage_callback("", local_source)
_, msg = PROCESS_QUEUE.get_nowait()
self.assertEqual(msg.get("operation"), "create")
