def test_delete_cluster_with_cleanup_disk_left(gke_mock, mocker):
"""Test that disk is deleted even if k8s did not delete it"""
def mocked_get_disks(cfg, dry_run):
"""Mocked getting GCP disks"""
return GCP_DISKS
def mocked_delete_disk(name, cfg):
"""Mocked GCP disk deletion"""
pass
def mocked_delete_cluster(cfg):
"""Mocked deletion of GKE cluster"""
return GKE_CLUSTERS[0]
def mocked_get_persistent_disks(dry_run):
"""Mocked listing of kubernets persistent disks"""
# persistent disk to delete
return [GCP_DISKS[0]]
mocker.patch('elb.gcp.get_disks', side_effect=mocked_get_disks)
mocker.patch('elb.gcp.delete_disk', side_effect=mocked_delete_disk)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
mocker.patch('elb.kubernetes.get_persistent_disks',
side_effect=mocked_get_persistent_disks)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
kubernetes.get_persistent_disks.assert_called()
# test that GCP disk deletion was called for the appropriate disk
gcp.delete_disk.assert_called_with(GCP_DISKS[0], cfg)
# test that cluster deletion was called
gcp.delete_cluster.assert_called_with(cfg)
def test_delete_cluster_with_cleanup_cluster_reconciling(gke_mock, mocker):
"""Test that cluster status RECONCILING is handled when deleting the
cluster. The code should wait until cluster status is RUNNING and delete
it then."""
class GKEStatusMock:
"""Class to mock changin GKE cluster status"""
def __init__(self):
self.status = 'RECONCILING'
def mocked_check_cluster(self, cfg):
"""Mocked check cluster status. Returns RECONCILING the first time
and RUNNING after that"""
if self.status == 'RECONCILING':
self.status = 'RUNNING'
return 'RECONCILING'
return self.status
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion"""
return cfg.cluster.name
mocked_cluster = GKEStatusMock()
mocker.patch('elb.gcp.check_cluster',
side_effect=mocked_cluster.mocked_check_cluster)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
# test that gcp.check_cluster was called more than once
assert gcp.check_cluster.call_count > 1
# test that cluster deletion was called
gcp.delete_cluster.assert_called()
def test_delete_cluster_with_cleanup_no_cluster(gke_mock):
"""Test deleting GKE cluster with cleanup when no cluster is present"""
# no cluster found in GKE
gke_mock.set_options(['no-cluster'])
cfg = get_mocked_config()
with pytest.raises(SafeExecError):
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
def test_get_gke_clusters_empty(mocker):
"""Test listing GKE clusters for an empty list"""
def safe_exec_empty(cmd):
"""Mocked safe_exec returning an emty JSON list"""
return MockedCompletedProcess('[]')
mocker.patch('elb.gcp.safe_exec', side_effect=safe_exec_empty)
cfg = get_mocked_config()
assert len(gcp.get_gke_clusters(cfg)) == 0
gcp.safe_exec.assert_called()
def test_get_disks_bad_output(mocker):
"""Test that gcp.get_disks raises RuntimeError for bad gcloud output"""
def safe_exec_bad_gcloud(cmd):
"""Mocked util.safe_exec function that returns incorrect JSON"""
if not cmd.startswith('gcloud compute disks list --format json'):
raise ValueError(f'Bad gcloud command line: {cmd}')
return MockedCompletedProcess('some-non-json-string')
mocker.patch('elb.gcp.safe_exec', side_effect=safe_exec_bad_gcloud)
cfg = get_mocked_config()
with pytest.raises(RuntimeError):
gcp.get_disks(cfg)
gcp.safe_exec.assert_called()
def test_delete_cluster_with_cleanup_cluster_stopping(gke_mock, mocker):
"""Test deleting cluster with the cluster is beeing stopped. The code
should raise RuntimeError"""
def mocked_check_cluster(cfg):
"""Mocked check cluster status. STOPPING never changes to RUNNING."""
return 'STOPPING'
mocker.patch('elb.gcp.check_cluster', side_effect=mocked_check_cluster)
cfg = get_mocked_config()
with pytest.raises(UserReportError) as errinfo:
gcp.delete_cluster_with_cleanup(cfg)
# test return code and message in UserReportError
assert errinfo.value.returncode == CLUSTER_ERROR
assert GKE_CLUSTERS[0] in errinfo.value.message
assert 'already being deleted' in errinfo.value.message
def test_delete_cluster_with_cleanup_failed_kubectl(gke_mock, mocker):
"""Test that cluster deletion is called when we cannot communicate with
it with kubectl"""
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion"""
return GKE_CLUSTERS[0]
# any kubectl call fails
gke_mock.set_options(['kubectl-error'])
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
kubernetes.safe_exec.assert_called()
# test cluster deletion was called
gcp.delete_cluster.assert_called_with(cfg)
def test_delete_nonexistent_disk(mocker):
"""Test that deleting a GCP disk that does not exits raises util.SafeExecError"""
def fake_subprocess_run(cmd, check, stdout, stderr):
"""Fake subprocess.run function that raises exception and emulates
command line returning with a non-zero exit code"""
raise subprocess.CalledProcessError(returncode=1,
cmd=cmd,
output=b'',
stderr=b'')
mocker.patch('subprocess.run', side_effect=fake_subprocess_run)
cfg = get_mocked_config()
with pytest.raises(SafeExecError):
gcp.delete_disk('some-disk', cfg)
subprocess.run.assert_called()
def test_delete_cluster_with_cleanup_cluster_error(gke_mock, mocker):
"""Test deleting a cluster with ERROR status"""
def mocked_check_cluster(cfg):
"""Mocked checking cluster status"""
return 'ERROR'
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion only to verify that it was called"""
return cfg.cluster.name
mocker.patch('elb.gcp.check_cluster', side_effect=mocked_check_cluster)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.check_cluster.assert_called()
# cluster deletion must be called
gcp.delete_cluster.assert_called()
def test_delete_cluster_with_cleanup_failed_get_disks(gke_mock, mocker):
"""Test that cluster and disk deletion are called when getting a list of
GCP disks failed"""
def mocked_get_disks(cfg, dry_run):
"""Mocked listing of GCP disks"""
mocked_get_disks.invocation_counter += 1
if mocked_get_disks.invocation_counter == 1:
raise RuntimeError('Mocked GCP listing error')
pass
mocked_get_disks.invocation_counter = 0
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion"""
return GKE_CLUSTERS[0]
def mocked_delete_disk(name, cfg):
"""Mocked disk deletion"""
pass
def mocked_get_persistent_disks(dry_run):
"""Mocked listing of GKE cluster persistent disks"""
return [GCP_DISKS[0]]
mocker.patch('elb.gcp.get_disks', side_effect=mocked_get_disks)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
mocker.patch('elb.gcp.delete_disk', side_effect=mocked_delete_disk)
mocker.patch('elb.kubernetes.get_persistent_disks',
side_effect=mocked_get_persistent_disks)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
kubernetes.safe_exec.assert_called()
# test cluster deletion was called
gcp.delete_cluster.assert_called_with(cfg)
# test that disk deletion was called
gcp.delete_disk.assert_called_with(GCP_DISKS[0], cfg)
def test_delete_cluster_with_cleanup_disk_known(gke_mock, mocker):
"""Test deleting cluster and persistent disk when disk id is known"""
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion, to test that it was called"""
pass
def mocked_delete_disk(name, cfg):
"""Mocked disk deletion, to test that it was called"""
pass
def mocked_delete_all():
"""Mocked deletion of all kubernetes jobs and persistent disks"""
raise RuntimeError('It should not have been called')
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
mocker.patch('elb.gcp.delete_disk', side_effect=mocked_delete_disk)
mocker.patch('elb.kubernetes.delete_all', side_effect=mocked_delete_all)
cfg = get_mocked_config()
cfg.appstate.disk_id = GCP_DISKS[0]
gcp.delete_cluster_with_cleanup(cfg)
gcp.delete_cluster.assert_called()
gcp.delete_disk.assert_called_with(cfg.appstate.disk_id, cfg)
def test_get_disks(gke_mock):
"""Test getting a list of GCP persistent disks"""
cfg = get_mocked_config()
disks = gcp.get_disks(cfg)
assert sorted(disks) == sorted(GCP_DISKS)
gcp.safe_exec.assert_called()
def test_delete_cluster_with_cleanup(gke_mock):
"""Test deleting GKE cluster and its persistent disks"""
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
kubernetes.safe_exec.assert_called()
def test_get_gke_clusters(gke_mock):
"""Test listing GKE clusters"""
cfg = get_mocked_config()
clusters = gcp.get_gke_clusters(cfg)
assert sorted(clusters) == sorted(GKE_CLUSTERS)
gcp.safe_exec.assert_called()
def test_delete_disk_empty_name():
"""Test that deleting disk with and empty name results in ValueError"""
cfg = get_mocked_config()
with pytest.raises(ValueError):
gcp.delete_disk('', cfg)
def test_delete_disk(gke_mock):
"""Test deleting a GCP disk"""
cfg = get_mocked_config()
gcp.delete_disk(GCP_DISKS[0], cfg)
gcp.safe_exec.assert_called()
