def test_delete_cluster_with_cleanup_cluster_reconciling(gke_mock, mocker):
"""Test that cluster status RECONCILING is handled when deleting the
cluster. The code should wait until cluster status is RUNNING and delete
it then."""
class GKEStatusMock:
"""Class to mock changin GKE cluster status"""
def __init__(self):
self.status = 'RECONCILING'
def mocked_check_cluster(self, cfg):
"""Mocked check cluster status. Returns RECONCILING the first time
and RUNNING after that"""
if self.status == 'RECONCILING':
self.status = 'RUNNING'
return 'RECONCILING'
return self.status
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion"""
return cfg.cluster.name
mocked_cluster = GKEStatusMock()
mocker.patch('elb.gcp.check_cluster',
side_effect=mocked_cluster.mocked_check_cluster)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
# test that gcp.check_cluster was called more than once
assert gcp.check_cluster.call_count > 1
# test that cluster deletion was called
gcp.delete_cluster.assert_called()
def test_delete_cluster_with_cleanup_disk_left(gke_mock, mocker):
"""Test that disk is deleted even if k8s did not delete it"""
def mocked_get_disks(cfg, dry_run):
"""Mocked getting GCP disks"""
return GCP_DISKS
def mocked_delete_disk(name, cfg):
"""Mocked GCP disk deletion"""
pass
def mocked_delete_cluster(cfg):
"""Mocked deletion of GKE cluster"""
return GKE_CLUSTERS[0]
def mocked_get_persistent_disks(dry_run):
"""Mocked listing of kubernets persistent disks"""
# persistent disk to delete
return [GCP_DISKS[0]]
mocker.patch('elb.gcp.get_disks', side_effect=mocked_get_disks)
mocker.patch('elb.gcp.delete_disk', side_effect=mocked_delete_disk)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
mocker.patch('elb.kubernetes.get_persistent_disks',
side_effect=mocked_get_persistent_disks)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
kubernetes.get_persistent_disks.assert_called()
# test that GCP disk deletion was called for the appropriate disk
gcp.delete_disk.assert_called_with(GCP_DISKS[0], cfg)
# test that cluster deletion was called
gcp.delete_cluster.assert_called_with(cfg)
def test_delete_cluster_with_cleanup_no_cluster(gke_mock):
"""Test deleting GKE cluster with cleanup when no cluster is present"""
# no cluster found in GKE
gke_mock.set_options(['no-cluster'])
cfg = get_mocked_config()
with pytest.raises(SafeExecError):
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
def blastdb_not_found_fixture():
"""Cleanup cluster if it was created"""
# setup
args = Namespace(cfg=INI_NO_BLASTDB)
yield args
# teardown
cfg = configparser.ConfigParser()
cfg.read(args.cfg)
cfg = ElasticBlastConfig(cfg, task=ElbCommand.SUBMIT)
gcp.delete_cluster_with_cleanup(cfg)
def test_delete_cluster_with_cleanup_cluster_stopping(gke_mock, mocker):
"""Test deleting cluster with the cluster is beeing stopped. The code
should raise RuntimeError"""
def mocked_check_cluster(cfg):
"""Mocked check cluster status. STOPPING never changes to RUNNING."""
return 'STOPPING'
mocker.patch('elb.gcp.check_cluster', side_effect=mocked_check_cluster)
cfg = get_mocked_config()
with pytest.raises(UserReportError) as errinfo:
gcp.delete_cluster_with_cleanup(cfg)
# test return code and message in UserReportError
assert errinfo.value.returncode == CLUSTER_ERROR
assert GKE_CLUSTERS[0] in errinfo.value.message
assert 'already being deleted' in errinfo.value.message
def test_delete_cluster_with_cleanup_failed_kubectl(gke_mock, mocker):
"""Test that cluster deletion is called when we cannot communicate with
it with kubectl"""
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion"""
return GKE_CLUSTERS[0]
# any kubectl call fails
gke_mock.set_options(['kubectl-error'])
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
kubernetes.safe_exec.assert_called()
# test cluster deletion was called
gcp.delete_cluster.assert_called_with(cfg)
def test_delete_cluster_with_cleanup_cluster_error(gke_mock, mocker):
"""Test deleting a cluster with ERROR status"""
def mocked_check_cluster(cfg):
"""Mocked checking cluster status"""
return 'ERROR'
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion only to verify that it was called"""
return cfg.cluster.name
mocker.patch('elb.gcp.check_cluster', side_effect=mocked_check_cluster)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.check_cluster.assert_called()
# cluster deletion must be called
gcp.delete_cluster.assert_called()
def test_delete_cluster_with_cleanup_failed_get_disks(gke_mock, mocker):
"""Test that cluster and disk deletion are called when getting a list of
GCP disks failed"""
def mocked_get_disks(cfg, dry_run):
"""Mocked listing of GCP disks"""
mocked_get_disks.invocation_counter += 1
if mocked_get_disks.invocation_counter == 1:
raise RuntimeError('Mocked GCP listing error')
pass
mocked_get_disks.invocation_counter = 0
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion"""
return GKE_CLUSTERS[0]
def mocked_delete_disk(name, cfg):
"""Mocked disk deletion"""
pass
def mocked_get_persistent_disks(dry_run):
"""Mocked listing of GKE cluster persistent disks"""
return [GCP_DISKS[0]]
mocker.patch('elb.gcp.get_disks', side_effect=mocked_get_disks)
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
mocker.patch('elb.gcp.delete_disk', side_effect=mocked_delete_disk)
mocker.patch('elb.kubernetes.get_persistent_disks',
side_effect=mocked_get_persistent_disks)
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
kubernetes.safe_exec.assert_called()
# test cluster deletion was called
gcp.delete_cluster.assert_called_with(cfg)
# test that disk deletion was called
gcp.delete_disk.assert_called_with(GCP_DISKS[0], cfg)
def test_delete_cluster_with_cleanup_disk_known(gke_mock, mocker):
"""Test deleting cluster and persistent disk when disk id is known"""
def mocked_delete_cluster(cfg):
"""Mocked cluster deletion, to test that it was called"""
pass
def mocked_delete_disk(name, cfg):
"""Mocked disk deletion, to test that it was called"""
pass
def mocked_delete_all():
"""Mocked deletion of all kubernetes jobs and persistent disks"""
raise RuntimeError('It should not have been called')
mocker.patch('elb.gcp.delete_cluster', side_effect=mocked_delete_cluster)
mocker.patch('elb.gcp.delete_disk', side_effect=mocked_delete_disk)
mocker.patch('elb.kubernetes.delete_all', side_effect=mocked_delete_all)
cfg = get_mocked_config()
cfg.appstate.disk_id = GCP_DISKS[0]
gcp.delete_cluster_with_cleanup(cfg)
gcp.delete_cluster.assert_called()
gcp.delete_disk.assert_called_with(cfg.appstate.disk_id, cfg)
def initialize_cluster(cfg: ElasticBlastConfig, db: str, db_path: str,
clean_up_stack):
""" Creates a k8s cluster, connects to it and initializes the persistent disk"""
logging.info('Starting cluster')
clean_up_stack.append(lambda: logging.debug('Before creating cluster'))
clean_up_stack.append(lambda: delete_cluster_with_cleanup(cfg))
clean_up_stack.append(lambda: collect_k8s_logs(cfg))
start_cluster(cfg)
clean_up_stack.append(lambda: logging.debug('After creating cluster'))
get_gke_credentials(cfg)
logging.info('Initializing storage')
clean_up_stack.append(lambda: logging.debug('Before initializing storage'))
initialize_storage(cfg, db, db_path)
clean_up_stack.append(lambda: logging.debug('After initializing storage'))
def test_delete_cluster_with_cleanup(gke_mock):
"""Test deleting GKE cluster and its persistent disks"""
cfg = get_mocked_config()
gcp.delete_cluster_with_cleanup(cfg)
gcp.safe_exec.assert_called()
kubernetes.safe_exec.assert_called()