def test_create_tag_non_existent_template_should_not_raise_error(
self,
create_fileset_enricher_tag_template,
get_fileset_enricher_tag_template,
list_tags, # noqa: E125
create_tag):
datacatalog_helper = DataCatalogHelper('test_project')
entry = MockedObject()
entry.name = 'fileset_entry'
tag_template = MockedObject()
tag_template.name = 'projects/my-project/locations/my-location/' \
'tagTemplates/fileset_template'
create_fileset_enricher_tag_template.return_value = tag_template
get_fileset_enricher_tag_template.side_effect = PermissionDenied(
'entry does not exist')
list_tags.return_value = []
stats = self.__create_full_stats_obj()
datacatalog_helper.create_tag_from_stats(entry, stats)
create_fileset_enricher_tag_template.assert_called_once()
get_fileset_enricher_tag_template.assert_called_once()
list_tags.assert_called_once()
create_tag.assert_called_once()
def test_limited_project_access(self, MockResourceManagerClient, MockLoggingClient):
MockResourceManagerClient.return_value.list_projects.return_value = [
MagicMock(project_id='proj1'),
MagicMock(project_id='proj2'),
MagicMock(project_id='proj3'),
]
MockLoggingClient.return_value.project = 'proj1'
MockLoggingClient.return_value.list_entries.side_effect = [
PermissionDenied(''),
iter(SAMPLE_ENTRIES),
NotFound(''),
]
reader = Reader(
start_time=datetime(2018, 4, 3, 9, 51, 22),
end_time=datetime(2018, 4, 3, 10, 51, 33),
collect_multiple_projects=True,
)
self.assertEqual(
reader.log_list,
[
BASE_LOG_NAME.format('proj1'),
BASE_LOG_NAME.format('proj2'),
BASE_LOG_NAME.format('proj3'),
],
)
self.assertEqual(list(reader), [FlowRecord(x) for x in SAMPLE_ENTRIES])
def test_get_no_permissions(self, mock_client_info, mock_secrets_client):
mock_client = mock.MagicMock()
mock_client_info.return_value = mock.MagicMock()
mock_secrets_client.return_value = mock_client
mock_client.secret_version_path.return_value = "full-path"
# No permissions for requested secret id
mock_client.access_secret_version.side_effect = PermissionDenied('test-msg')
secrets_client = _SecretManagerClient(credentials="credentials")
secret = secrets_client.get_secret(secret_id="missing", project_id="project_id")
mock_client.secret_version_path.assert_called_once_with("project_id", 'missing', 'latest')
assert secret is None
mock_client.access_secret_version.assert_called_once_with('full-path')
def test_create_tag_providing_tag_fields_with_key_error_should_leak_the_error(
self,
create_fileset_enricher_tag_template,
get_fileset_enricher_tag_template,
list_tags, # noqa: E125
create_tag): # noqa:E125
datacatalog_helper = DataCatalogHelper('test_project')
entry = MockedObject()
entry.name = 'fileset_entry'
tag_template = MockedObject()
tag_template.name = 'projects/my-project/locations/my-location/' \
'tagTemplates/fileset_template'
create_fileset_enricher_tag_template.return_value = tag_template
get_fileset_enricher_tag_template.side_effect = PermissionDenied(
'entry does not exist')
list_tags.return_value = []
stats = {
'prefix': 'gs://my_bucket/*.csv',
'count': 0,
'files_by_bucket': '',
'buckets_found': 0,
'execution_time': pd.Timestamp.utcnow()
}
fields = ['files', 'prefix']
datacatalog_helper.create_tag_from_stats(entry, stats, fields)
tag = create_tag.call_args_list[0][1]['tag']
fields_dict = MessageToDict(tag)['fields']
self.assertTrue(set(fields).issubset(fields_dict))
self.assertIn('execution_time', fields_dict)
self.assertNotIn('files_by_bucket', fields_dict)
self.assertNotIn('buckets_found', fields_dict)
self.assertNotIn('bucket_prefix', fields_dict)
self.assertNotIn('min_file_size', fields_dict)
self.assertNotIn('max_file_size', fields_dict)
self.assertNotIn('avg_file_size', fields_dict)
self.assertNotIn('first_created_date', fields_dict)
self.assertNotIn('last_created_date', fields_dict)
self.assertNotIn('last_updated_date', fields_dict)
self.assertNotIn('created_files_by_day', fields_dict)
self.assertNotIn('updated_files_by_day', fields_dict)
create_fileset_enricher_tag_template.assert_called_once()
get_fileset_enricher_tag_template.assert_called_once()
list_tags.assert_called_once()
create_tag.assert_called_once()
def get_response(self, file_path, report_type):
try:
with io.open(file_path, 'rb') as image_file:
content = image_file.read()
image = vision.types.Image(content=content)
except:
try:
image = vision.types.Image()
image.source.image_uri = self.file_path
except PermissionDenied:
raise PermissionDenied("Ensure Vision API is enabled")
objects = self.REPORTS[report_type](self.client,image)
return parsers[report_type](objects)
def test_tag_template_exists_should_return_false_nonexistent(
self, mock_get_tag_template):
mock_get_tag_template.side_effect = PermissionDenied(message='')
tag_template_exists = DataCatalogFacade().tag_template_exists(None)
self.assertFalse(tag_template_exists)
mock_get_tag_template.assert_called_once()
def test_delete_tag_template_should_handle_nonexistent(
self, mock_delete_tag_template):
mock_delete_tag_template.side_effect = PermissionDenied(message='')
DataCatalogFacade().delete_tag_template(None)
mock_delete_tag_template.assert_called_once()