def backup_test(client): # NOQA volume = client.by_id_volume(VOLUME_NAME) setting = client.by_id_setting("backupTarget") setting = client.update(setting, value=common.get_backupstore_url()) assert setting["value"] == common.get_backupstore_url() volume.snapshotCreate() snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) found = False for i in range(100): bvs = client.list_backupVolume() for bv in bvs: if bv["name"] == VOLUME_NAME: found = True break if found: break time.sleep(1) assert found found = False for i in range(20): backups = bv.backupList() for b in backups: if b["snapshotName"] == snap2["name"]: found = True break if found: break time.sleep(1) assert found new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap2["name"]: found = True break assert not found
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA pod_name = 'csi-backup-test' create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make, base_image, "") test_data = generate_random_data(VOLUME_RWTEST_SIZE) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() i = 1 for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting.value == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential.value == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting.value == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential.value == "" backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i) i += 1
def test_bgtasks(clients): # NOQA for host_id, client in clients.iteritems(): break setting = client.by_id_setting("backupTarget") setting = client.update(setting, value=common.get_backupstore_url()) volume = client.create_volume(name=VOLUME_NAME, size=SIZE, numberOfReplicas=2) volume = volume.attach(hostId=host_id) num_tasks = 4 snaps = map(lambda _: volume.snapshotCreate(), range(num_tasks)) for snap in snaps: volume.snapshotBackup(name=snap["name"]) queue = volume.bgTaskQueue() q_len = len(queue) assert len(queue) == num_tasks assert queue[0]["num"] == 1 assert queue[0]["finished"] == "" # running: finished is not set while q_len > 1: queue = volume.bgTaskQueue() if len(queue) < q_len: q_len = len(queue) assert queue[0]["finished"] != "" # done: finished is set assert queue[0]["err"] is None assert queue[0]["num"] == 1 + num_tasks - q_len else: time.sleep(1)
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA pod_name = 'csi-backup-test' create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make, base_image, "") test_data = generate_random_data(VOLUME_RWTEST_SIZE) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() i = 1 for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i) i += 1
def test_restore_inc(clients, core_api, volume_name, pod): # NOQA for _, client in clients.iteritems(): break setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" restore_inc_test(client, core_api, volume_name, pod)
def backup_test(clients, volume_name, size, base_image=""): # NOQA for host_id, client in clients.iteritems(): break volume = create_and_check_volume(client, volume_name, 2, size, base_image) lht_hostId = get_self_host_id() volume = volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" backupstore_test(client, lht_hostId, volume_name, size) cleanup_volume(client, volume)
def set_backupstore_nfs(client): backupstores = get_backupstore_url() poll_interval = get_backupstore_poll_interval() for backupstore in backupstores: if is_backupTarget_nfs(backupstore): set_backupstore_url(client, backupstore) set_backupstore_credential_secret(client, "") set_backupstore_poll_interval(client, poll_interval) break
def set_backupstore_s3(client): backupstores = get_backupstore_url() poll_interval = get_backupstore_poll_interval() for backupstore in backupstores: if is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") set_backupstore_url(client, backupsettings[0]) set_backupstore_credential_secret(client, backupsettings[1]) set_backupstore_poll_interval(client, poll_interval) break
def backup_test(clients, volume_name, size, base_image=""): # NOQA for host_id, client in clients.iteritems(): break volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["baseImage"] == base_image lht_hostId = get_self_host_id() volume = volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" backupstore_test(client, lht_hostId, volume_name, size) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) volume = wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def set_backupstore_nfs(client): backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET) backupstores = get_backupstore_url() for backupstore in backupstores: if is_backupTarget_nfs(backupstore): backup_target_setting = client.update(backup_target_setting, value=backupstore) assert backup_target_setting.value == backupstore backup_target_credential_setting = client.by_id_setting( SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) backup_target_credential_setting = \ client.update(backup_target_credential_setting, value="") assert backup_target_credential_setting.value == "" break
def backup_labels_test(clients, random_labels, volume_name, size=SIZE, base_image=""): # NOQA for _, client in clients.iteritems(): break host_id = get_self_host_id() volume = create_and_check_volume(client, volume_name, 2, size, base_image) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" bv, b, _, _ = create_backup(client, volume_name, labels=random_labels) # If we're running the test with a BaseImage, check that this Label is # set properly. backup = bv.backupGet(name=b["name"]) if base_image: assert backup["labels"].get(common.BASE_IMAGE_LABEL) == base_image # One extra Label from the BaseImage being set. assert len(backup["labels"]) == len(random_labels) + 1 else: assert len(backup["labels"]) == len(random_labels) cleanup_volume(client, volume)
def set_backupstore_s3(client): backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET) backupstores = get_backupstore_url() for backupstore in backupstores: if is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") backup_target_setting = client.update(backup_target_setting, value=backupsettings[0]) assert backup_target_setting.value == backupsettings[0] backup_target_credential_setting = client.by_id_setting( SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) backup_target_credential_setting = \ client.update(backup_target_credential_setting, value=backupsettings[1]) assert backup_target_credential_setting.value == backupsettings[1] break
def backup_test(client, host_id): # NOQA volume = client.by_id_volume(VOLUME_NAME) setting = client.by_id_setting("backupTarget") setting = client.update(setting, value=common.get_backupstore_url()) assert setting["value"] == common.get_backupstore_url() volume.snapshotCreate() snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) jobs = volume.jobList() found = False for job in jobs: if job["state"] == "ongoing" and job["jobType"] == "snapshot-backup": found = True break assert found found = False for i in range(100): bvs = client.list_backupVolume() for bv in bvs: if bv["name"] == VOLUME_NAME: found = True break if found: break time.sleep(1) assert found found = False for i in range(20): backups = bv.backupList() for b in backups: if b["snapshotName"] == snap2["name"]: found = True break if found: break time.sleep(1) assert found new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] # test restore volume = client.create_volume(name=VOLUME_RESTORE_NAME, size=SIZE, numberOfReplicas=2, fromBackup=b["url"]) volume = wait_for_volume_state(client, VOLUME_RESTORE_NAME, "detached") assert volume["name"] == VOLUME_RESTORE_NAME assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" volume = volume.attach(hostId=host_id) volume = wait_for_volume_state(client, VOLUME_RESTORE_NAME, "healthy") volume = volume.detach() volume = wait_for_volume_state(client, VOLUME_RESTORE_NAME, "detached") client.delete(volume) volume = wait_for_volume_delete(client, VOLUME_RESTORE_NAME) bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap2["name"]: found = True break assert not found
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_restoration_completed( client, res_name) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) cleanup_volume(client, volume)
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_listing_backup_volume(clients, base_image=""): # NOQA for host_id, client in clients.iteritems(): break lht_hostId = get_self_host_id() # create 3 volumes. volume1_name = generate_volume_name() volume2_name = generate_volume_name() volume3_name = generate_volume_name() volume1 = create_and_check_volume(client, volume1_name) volume2 = create_and_check_volume(client, volume2_name) volume3 = create_and_check_volume(client, volume3_name) volume1.attach(hostId=lht_hostId) volume1 = common.wait_for_volume_healthy(client, volume1_name) volume2.attach(hostId=lht_hostId) volume2 = common.wait_for_volume_healthy(client, volume2_name) volume3.attach(hostId=lht_hostId) volume3 = common.wait_for_volume_healthy(client, volume3_name) # we only test NFS here. # Since it is difficult to directly remove volume.cfg from s3 buckets setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_nfs(backupstore): updated = False for i in range(RETRY_COMMAND_COUNT): nfs_url = backupstore.strip("nfs://") setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) if "nfs" in setting["value"]: updated = True break assert updated _, _, snap1, _ = create_backup(client, volume1_name) _, _, snap2, _ = create_backup(client, volume2_name) _, _, snap3, _ = create_backup(client, volume3_name) # invalidate backup volume 1 by renaming volume.cfg to volume.cfg.tmp cmd = ["mkdir", "-p", "/mnt/nfs"] subprocess.check_output(cmd) cmd = ["mount", "-t", "nfs4", nfs_url, "/mnt/nfs"] subprocess.check_output(cmd) cmd = ["find", "/mnt/nfs", "-type", "d", "-name", volume1_name] volume1_backup_volume_path = subprocess.check_output(cmd).strip() cmd = ["find", volume1_backup_volume_path, "-name", "volume.cfg"] volume1_backup_volume_cfg_path = subprocess.check_output(cmd).strip() cmd = [ "mv", volume1_backup_volume_cfg_path, volume1_backup_volume_cfg_path + ".tmp" ] subprocess.check_output(cmd) subprocess.check_output(["sync"]) found1 = found2 = found3 = False for i in range(RETRY_COUNTS): bvs = client.list_backupVolume() for bv in bvs: if bv["name"] == volume1_name: if "error" in bv.messages: assert "volume.cfg" in bv.messages["error"].lower() found1 = True elif bv["name"] == volume2_name: assert not bv.messages found2 = True elif bv["name"] == volume3_name: assert not bv.messages found3 = True if found1 & found2 & found3: break time.sleep(RETRY_INTERVAL) assert found1 & found2 & found3 cmd = [ "mv", volume1_backup_volume_cfg_path + ".tmp", volume1_backup_volume_cfg_path ] subprocess.check_output(cmd) subprocess.check_output(["sync"]) found = False for i in range(RETRY_COMMAND_COUNT): try: bv1, b1 = common.find_backup(client, volume1_name, snap1["name"]) found = True break except Exception: time.sleep(1) assert found bv1.backupDelete(name=b1["name"]) for i in range(RETRY_COMMAND_COUNT): found = False backups1 = bv1.backupList() for b in backups1: if b["snapshotName"] == snap1["name"]: found = True break assert not found bv2, b2 = common.find_backup(client, volume2_name, snap2["name"]) bv2.backupDelete(name=b2["name"]) for i in range(RETRY_COMMAND_COUNT): found = False backups2 = bv2.backupList() for b in backups2: if b["snapshotName"] == snap2["name"]: found = True break assert not found bv3, b3 = common.find_backup(client, volume3_name, snap3["name"]) bv3.backupDelete(name=b3["name"]) for i in range(RETRY_COMMAND_COUNT): found = False backups3 = bv3.backupList() for b in backups3: if b["snapshotName"] == snap3["name"]: found = True break assert not found volume1.detach() volume1 = common.wait_for_volume_detached(client, volume1_name) client.delete(volume1) wait_for_volume_delete(client, volume1_name) volume2.detach() volume2 = common.wait_for_volume_detached(client, volume2_name) client.delete(volume2) wait_for_volume_delete(client, volume2_name) volume3.detach() volume3 = common.wait_for_volume_detached(client, volume3_name) client.delete(volume3) wait_for_volume_delete(client, volume3_name) volumes = client.list_volume() assert len(volumes) == 0