def backup_test(clients, volume_name, size, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
lht_hostId = get_self_host_id()
volume = volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, lht_hostId, volume_name, size)
cleanup_volume(client, volume)
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pod_name = 'csi-backup-test'
create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make,
base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
i = 1
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting.value == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential.value == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting.value == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential.value == ""
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i)
i += 1
def test_restore_inc(clients, core_api, volume_name, pod): # NOQA
for _, client in clients.iteritems():
break
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
restore_inc_test(client, core_api, volume_name, pod)
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pod_name = 'csi-backup-test'
create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make,
base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
i = 1
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i)
i += 1
def backupstore_get_volume_cfg_file_path(client, volume_name):
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
return minio_get_volume_cfg_file_path(volume_name)
elif is_backupTarget_nfs(backupstore):
return nfs_get_volume_cfg_file_path(client, volume_name)
def backupstore_count_backup_block_files(client, core_api, volume_name):
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
return minio_count_backup_block_files(client, core_api, volume_name)
elif is_backupTarget_nfs(backupstore):
return nfs_count_backup_block_files(client, volume_name)
def backupstore_delete_random_backup_block(client, core_api, volume_name):
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
minio_delete_random_backup_block(client, core_api, volume_name)
elif is_backupTarget_nfs(backupstore):
nfs_delete_random_backup_block(client, volume_name)
def backupstore_get_backup_blocks_dir(client, volume_name):
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
return minio_get_backup_blocks_dir(volume_name)
elif is_backupTarget_nfs(backupstore):
return nfs_get_backup_blocks_dir(client, volume_name)
def set_backupstore_s3(client):
backupstores = get_backupstore_url()
poll_interval = get_backupstore_poll_interval()
for backupstore in backupstores:
if is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
set_backupstore_url(client, backupsettings[0])
set_backupstore_credential_secret(client, backupsettings[1])
set_backupstore_poll_interval(client, poll_interval)
break
def backupstore_delete_volume_cfg_file(client, core_api, volume_name): # NOQA
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
minio_delete_volume_cfg_file(client,
core_api,
volume_name)
elif is_backupTarget_nfs(backupstore):
nfs_delete_volume_cfg_file(client, volume_name)
def backupstore_get_backup_cfg_file_path(client, volume_name, backup_name):
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
return minio_get_backup_cfg_file_path(volume_name, backup_name)
elif is_backupTarget_nfs(backupstore):
return nfs_get_backup_cfg_file_path(client, volume_name, backup_name)
else:
raise NotImplementedError
def backupstore_delete_file(client, core_api, file_path):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstore = backup_target_setting.value
if is_backupTarget_s3(backupstore):
return mino_delete_file_in_backupstore(client,
core_api,
file_path)
else:
raise NotImplementedError
def backupstore_create_file(client, core_api, file_path, data={}):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstore = backup_target_setting.value
if is_backupTarget_s3(backupstore):
return mino_create_file_in_backupstore(client, core_api, file_path,
data)
elif is_backupTarget_nfs(backupstore):
return nfs_create_file_in_backupstore(file_path, data={})
else:
raise NotImplementedError
def backupstore_delete_backup_cfg_file(client, core_api, volume_name,
backup_name): # NOQA
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
minio_delete_backup_cfg_file(client, core_api, volume_name,
backup_name)
elif is_backupTarget_nfs(backupstore):
nfs_delete_backup_cfg_file(client, volume_name, backup_name)
else:
raise NotImplementedError
def backup_test(clients, volume_name, size, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
volume = client.create_volume(name=volume_name,
size=size,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["baseImage"] == base_image
lht_hostId = get_self_host_id()
volume = volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, lht_hostId, volume_name, size)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
volume = wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def backupstore_write_backup_cfg_file(client, core_api, volume_name, backup_name, data): # NOQA
backupstore = backupstore_get_backup_target(client)
if is_backupTarget_s3(backupstore):
minio_write_backup_cfg_file(client,
core_api,
volume_name,
backup_name,
data)
elif is_backupTarget_nfs(backupstore):
nfs_write_backup_cfg_file(client,
volume_name,
backup_name,
data)
def backup_labels_test(clients,
random_labels,
volume_name,
size=SIZE,
base_image=""): # NOQA
for _, client in clients.iteritems():
break
host_id = get_self_host_id()
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
bv, b, _, _ = create_backup(client, volume_name, labels=random_labels)
# If we're running the test with a BaseImage, check that this Label is
# set properly.
backup = bv.backupGet(name=b["name"])
if base_image:
assert backup["labels"].get(common.BASE_IMAGE_LABEL) == base_image
# One extra Label from the BaseImage being set.
assert len(backup["labels"]) == len(random_labels) + 1
else:
assert len(backup["labels"]) == len(random_labels)
cleanup_volume(client, volume)
def set_backupstore_s3(client):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstores = get_backupstore_url()
for backupstore in backupstores:
if is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
backup_target_setting = client.update(backup_target_setting,
value=backupsettings[0])
assert backup_target_setting.value == backupsettings[0]
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
backup_target_credential_setting = \
client.update(backup_target_credential_setting,
value=backupsettings[1])
assert backup_target_credential_setting.value == backupsettings[1]
break
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=size,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name, size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def minio_get_backupstore_bucket_name(client):
backupstore = backupstore_get_backup_target(client)
assert is_backupTarget_s3(backupstore)
bucket_name = urlparse(backupstore).netloc.split('@')[0]
return bucket_name
def ha_backup_deletion_recovery_test(client,
volume_name,
size,
base_image=""): # NOQA
volume = client.create_volume(name=volume_name,
size=size,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name,
size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_restoration_completed(
client, res_name)
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
cleanup_volume(client, volume)
def minio_get_backupstore_path(client):
backupstore = backupstore_get_backup_target(client)
assert is_backupTarget_s3(backupstore)
backupstore_path = urlparse(backupstore).path.split('$')[0].strip("/")
return backupstore_path