def test_table_retention(self):
"""Test yaml file that includes more than one rules that works on the same project"""
rule_yaml = """
rules:
- name: project 5 max 90 on table
applies_to:
- bigquery_table
resource:
- type: table
resource_ids:
- "def-project-5:ds01.fake-table-01"
maximum_retention: 100
"""
table_test_data = [
frsd.FakeTableDataInput(
table_id='fake-table-01',
dataset=frsd.DATASET1,
expiration_time=frsd.DEFAULT_TABLE_CREATE_TIME +
110 * rre._MS_PER_DAY),
frsd.FakeTableDataInput(
table_id='fake-table-02',
dataset=frsd.DATASET1,
expiration_time=frsd.DEFAULT_TABLE_CREATE_TIME +
89 * rre._MS_PER_DAY),
frsd.FakeTableDataInput(
table_id='fake-table-03',
dataset=frsd.DATASET1,
expiration_time=frsd.DEFAULT_TABLE_CREATE_TIME +
91 * rre._MS_PER_DAY),
]
_mock_bucket = get_mock_table_retention(table_test_data)
with tempfile.NamedTemporaryFile(suffix='.yaml') as f:
f.write(rule_yaml.encode())
f.flush()
_fake_bucket_list = _mock_bucket(resource_type='bigquery_table')
self.scanner = retention_scanner.RetentionScanner({}, {},
mock.MagicMock(),
'', '', f.name)
mock_data_access = mock.MagicMock()
mock_data_access.scanner_iter.side_effect = _mock_bucket
mock_service_config = mock.MagicMock()
mock_service_config.model_manager = mock.MagicMock()
mock_service_config.model_manager.get.return_value = (
mock.MagicMock(), mock_data_access)
self.scanner.service_config = mock_service_config
all_lifecycle_info = self.scanner.run()
def test_invalid_rule_with_no_resource(self):
"""Test that a rule without resource cannot be created"""
yaml_str_no_resource="""
rules:
- name: No resource
applies_to:
- bucket
minimum_retention: 365
maximum_retention: 365
"""
with tempfile.NamedTemporaryFile(suffix='.yaml') as f:
f.write(yaml_str_no_resource.encode())
f.flush()
rules_local_path = get_datafile_path(__file__, f.name)
with self.assertRaises(InvalidRulesSchemaError):
self.scanner = retention_scanner.RetentionScanner(
{}, {}, mock.MagicMock(), '', '', rules_local_path)
def test_invalid_rule_lack_of_min_max(self):
"""Test that a rule with neither minimum_retention nor maximum_retention
cannot be created"""
yaml_str_lack_min_max="""
rules:
- name: Lack of min and max retention
applies_to:
- bucket
resource:
- type: bucket
resource_ids:
- some-resource-id
"""
with tempfile.NamedTemporaryFile(suffix='.yaml') as f:
f.write(yaml_str_lack_min_max.encode())
f.flush()
rules_local_path = get_datafile_path(__file__, f.name)
with self.assertRaises(InvalidRulesSchemaError):
self.scanner = retention_scanner.RetentionScanner(
{}, {}, mock.MagicMock(), '', '', rules_local_path)
def test_invalid_rule_with_duplicate_applies_to(self):
"""Test that a rule with duplicate applies_to cannot be created"""
yaml_str_duplicate_applies_to = """
rules:
- name: Duplicate applies_to
applies_to:
- bucket
- bucket
resource:
- type: bucket
resource_ids:
- some-resource-id
minimum_retention: 365
maximum_retention: 365
"""
with tempfile.NamedTemporaryFile(suffix='.yaml') as f:
f.write(yaml_str_duplicate_applies_to)
f.flush()
rules_local_path = get_datafile_path(__file__, f.name)
with self.assertRaises(InvalidRulesSchemaError):
self.scanner = retention_scanner.RetentionScanner(
{}, {}, mock.MagicMock(), '', '', rules_local_path)
def test_bucket_retention_on_multi_buckets(self):
"""Test a rule that includes more than one bucket IDs"""
rule_yaml = """
rules:
- name: multiple buckets in a single rule
applies_to:
- bucket
resource:
- type: bucket
resource_ids:
- fake-bucket-1
- fake-bucket-2
- fake-bucket-3
minimum_retention: 90
"""
bucket_test_data=[
frsd.FakeBucketDataInput(
id='fake-bucket-1',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':89})]
),
frsd.FakeBucketDataInput(
id='fake-bucket-2',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':90})]
),
frsd.FakeBucketDataInput(
id='fake-bucket-3',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':89})]
),
frsd.FakeBucketDataInput(
id='fake-bucket-4',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':89})]
),
]
_mock_bucket = get_mock_bucket_retention(bucket_test_data)
with tempfile.NamedTemporaryFile(suffix='.yaml') as f:
f.write(rule_yaml)
f.flush()
_fake_bucket_list = _mock_bucket(None, 'bucket')
self.scanner = retention_scanner.RetentionScanner(
{}, {}, mock.MagicMock(), '', '', f.name)
mock_data_access = mock.MagicMock()
mock_data_access.scanner_iter.side_effect = _mock_bucket
mock_service_config = mock.MagicMock()
mock_service_config.model_manager = mock.MagicMock()
mock_service_config.model_manager.get.return_value = (
mock.MagicMock(), mock_data_access)
self.scanner.service_config = mock_service_config
all_lifecycle_info = self.scanner._retrieve()
all_violations = self.scanner._find_violations(all_lifecycle_info)
res_map = {}
for i in _fake_bucket_list:
res_map[i.id] = i
expected_violations = set([
get_expect_violation_item(res_map, 'fake-bucket-1',
'multiple buckets in a single rule', 0),
get_expect_violation_item(res_map, 'fake-bucket-3',
'multiple buckets in a single rule', 0)
])
self.assertEqual(expected_violations, set(all_violations))
def test_bucket_retention_on_multi_rules_on_a_project(self):
"""Test yaml file that includes more than one rules that works on the same project"""
rule_yaml = """
rules:
- name: project 1 min 90
applies_to:
- bucket
resource:
- type: project
resource_ids:
- def-project-1
minimum_retention: 90
- name: project 1 min 100
applies_to:
- bucket
resource:
- type: project
resource_ids:
- def-project-1
minimum_retention: 100
- name: project 1 min 110
applies_to:
- bucket
resource:
- type: project
resource_ids:
- def-project-1
minimum_retention: 110
"""
bucket_test_data=[
frsd.FakeBucketDataInput(
id='fake-bucket-11',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':90})]
),
frsd.FakeBucketDataInput(
id='fake-bucket-12',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':100})]
),
frsd.FakeBucketDataInput(
id='fake-bucket-13',
project=frsd.PROJECT1,
lifecycles=[frsd.LifecycleInput(action='Delete', conditions={'age':110})]
),
]
_mock_bucket = get_mock_bucket_retention(bucket_test_data)
with tempfile.NamedTemporaryFile(suffix='.yaml') as f:
f.write(rule_yaml)
f.flush()
_fake_bucket_list = _mock_bucket(resource_type='bucket')
self.scanner = retention_scanner.RetentionScanner(
{}, {}, mock.MagicMock(), '', '', f.name)
mock_data_access = mock.MagicMock()
mock_data_access.scanner_iter.side_effect = _mock_bucket
mock_service_config = mock.MagicMock()
mock_service_config.model_manager = mock.MagicMock()
mock_service_config.model_manager.get.return_value = (
mock.MagicMock(), mock_data_access)
self.scanner.service_config = mock_service_config
all_lifecycle_info = self.scanner._retrieve()
all_violations = self.scanner._find_violations(all_lifecycle_info)
res_map = {}
for i in _fake_bucket_list:
res_map[i.id] = i
expected_violations = set([
get_expect_violation_item(res_map, 'fake-bucket-11',
'project 1 min 100', 1),
get_expect_violation_item(res_map, 'fake-bucket-11',
'project 1 min 110', 2),
get_expect_violation_item(res_map, 'fake-bucket-12',
'project 1 min 110', 2)
])
self.assertEqual(expected_violations, set(all_violations))