def test_update_policies_ignore_policy_if_failed_to_create_content_hash(
self, get_blob_client_mock, _1, _2, _3, _4):
client_mock = Mock()
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob("blob1.yml", None), # no hash
]
client_mock.get_blob_to_path = self.download_policy_blob
get_blob_client_mock.return_value = (client_mock, None, None)
def create_blob_from_bytes(_1, _2, _3, **kwargs):
raise AzureHttpError("Failed to create blob", 403)
client_mock.create_blob_from_bytes = create_blob_from_bytes
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
# cleanup
self.addCleanup(lambda: shutil.rmtree(host.policy_cache))
self.assertEqual({}, host.policies)
# run
host.update_policies()
# no policies were loaded
self.assertEqual(0, len(host.policies.items()))
# jobs were created
jobs = host.scheduler.get_jobs()
self.assertEqual(0, len([j for j in jobs if j.id == 'blob1.yml']))
def test_update_policies(self, get_blob_client_mock, _1, _2, _3, _4):
# mock blob list call
client_mock = Mock()
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob("blob1.yml", "hash1"),
ContainerHostTest.get_mock_blob("blob2.YAML", "hash2"),
ContainerHostTest.get_mock_blob("blob3.md", "hash3")
]
client_mock.get_blob_to_path = self.download_policy_blob
get_blob_client_mock.return_value = (client_mock, None, None)
# init
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
# cleanup
self.addCleanup(lambda: shutil.rmtree(host.policy_cache))
self.assertEqual({}, host.policies)
# run
host.update_policies()
# both policies were loaded
self.assertEqual(2, len(host.policies.items()))
# jobs were created
jobs = host.scheduler.get_jobs()
self.assertEqual(1, len([j for j in jobs if j.id == 'blob1.yml']))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob2.YAML']))
def test_update_policies_create_content_hash(self, get_blob_client_mock,
_1, _2, _3, _4):
client_mock = Mock()
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob("blob1.yml", None), # no hash
]
client_mock.get_blob_to_path = self.download_policy_blob
get_blob_client_mock.return_value = (client_mock, None, None)
def get_blob_properties(_, blob_name):
return ContainerHostTest.get_mock_blob(blob_name,
'hash') # now with hash
client_mock.get_blob_properties = get_blob_properties
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
# cleanup
self.addCleanup(lambda: shutil.rmtree(host.policy_cache))
self.assertEqual({}, host.policies)
# run
host.update_policies()
# both policies were loaded
self.assertEqual(1, len(host.policies.items()))
# jobs were created
jobs = host.scheduler.get_jobs()
self.assertEqual(1, len([j for j in jobs if j.id == 'blob1.yml']))
def test_update_policies_list_blobs_azure_http_error(
self, get_blob_client_mock, _2, _3, _4, _5):
client_mock = Mock()
client_mock.list_blobs.side_effect = AzureHttpError(
"failed to list blobs", 400)
get_blob_client_mock.return_value = (client_mock, None, None)
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
with self.assertRaises(AzureHttpError):
host.update_policies()
client_mock.list_blobs.assert_called()
def test_update_policies_missing_mode(self, get_blob_client_mock, _2, _3,
_4, _5):
client_mock = Mock()
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob('no-mode-blob.yaml',
'no-mode-blob')
]
client_mock.get_blob_to_path = self.download_missing_mode_policy_blob
get_blob_client_mock.return_value = (client_mock, None, None)
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
self.addCleanup(lambda: shutil.rmtree(host.policy_cache))
host.update_policies()
self.assertEqual(1, len(host.policies.items()))
def test_update_policies_in_subfolders(self, makedirs_mock,
get_blob_client_mock, _2, _3, _4,
_5):
blob_name = "path/to/blob.yml"
client_mock = Mock()
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob(blob_name, 'blob')
]
client_mock.get_blob_to_path = self.download_policy_blob
get_blob_client_mock.return_value = (client_mock, None, None)
self.addCleanup(lambda: shutil.rmtree(host.policy_cache))
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
host.update_policies()
self.assertEqual(1, len(host.policies.items()))
jobs = host.scheduler.get_jobs()
self.assertEqual(1, len([j for j in jobs if j.id == blob_name]))
blob_dir = os.path.join(host.policy_cache, blob_name)
makedirs_mock.assert_any_call(os.path.dirname(blob_dir))
def test_update_policies_add_remove(self, get_blob_client_mock, _1, _2, _3,
_4):
"""
Run a series of add/update/removal of policy blobs
and verify jobs and caches are updated correctly
"""
# mock blob list call
client_mock = Mock()
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob("blob1.yml", "hash1")
]
client_mock.get_blob_to_path = self.download_policy_blob
get_blob_client_mock.return_value = (client_mock, None, None)
# init
host = Host(DEFAULT_EVENT_QUEUE_ID, DEFAULT_EVENT_QUEUE_NAME,
DEFAULT_POLICY_STORAGE)
# cleanup
self.addCleanup(lambda: shutil.rmtree(host.policy_cache))
self.assertEqual({}, host.policies)
# Initial load
host.update_policies()
self.assertEqual(1, len(host.policies.items()))
##################
# Add two policies
##################
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob("blob1.yml", "hash1"),
ContainerHostTest.get_mock_blob("blob2.yml", "hash2"),
ContainerHostTest.get_mock_blob("blob3.yml", "hash3")
]
host.update_policies()
self.assertEqual(3, len(host.policies.items()))
self.assertIsNotNone(host.policies['blob1.yml'])
self.assertIsNotNone(host.policies['blob2.yml'])
self.assertIsNotNone(host.policies['blob3.yml'])
# jobs were updated
jobs = host.scheduler.get_jobs()
self.assertEqual(3,
len([j for j in jobs if j.func == host.run_policy]))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob1.yml']))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob2.yml']))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob3.yml']))
##############################################
# Add one, remove one, update one
##############################################
client_mock.list_blobs.return_value = [
ContainerHostTest.get_mock_blob("blob1.yml", "hash1"),
ContainerHostTest.get_mock_blob("blob4.yml", "hash4"),
ContainerHostTest.get_mock_blob("blob3.yml", "hash3_new")
]
host.update_policies()
self.assertEqual(3, len(host.policies.items()))
self.assertIsNotNone(host.policies['blob1.yml'])
self.assertIsNotNone(host.policies['blob4.yml'])
self.assertIsNotNone(host.policies['blob3.yml'])
self.assertEqual('hash3_new', host.blob_cache['blob3.yml'])
# jobs were updated
jobs = host.scheduler.get_jobs()
self.assertEqual(3,
len([j for j in jobs if j.func == host.run_policy]))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob1.yml']))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob4.yml']))
self.assertEqual(1, len([j for j in jobs if j.id == 'blob3.yml']))
############
# remove all
############
client_mock.list_blobs.return_value = []
host.update_policies()
self.assertEqual(0, len(host.policies.items()))
# jobs were updated
jobs = host.scheduler.get_jobs()
self.assertEqual(0,
len([j for j in jobs if j.func == host.run_policy]))