def test_backup_corrupt_deletion( grpc_replica1, grpc_replica2, # NOQA grpc_controller, backup_targets): # NOQA address = grpc_controller.address length = 128 for backup_target in backup_targets: dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) # write two backup blocks verify_data(dev, 0, random_string(length)) verify_data(dev, BLOCK_SIZE, random_string(length)) snap = cmd.snapshot_create(address) backup1 = create_backup(address, snap, backup_target) # overwrite second backup block verify_data(dev, BLOCK_SIZE, random_string(length)) snap = cmd.snapshot_create(address) backup2 = create_backup(address, snap, backup_target) # check that the volume now has 3 blocks # backup1 and backup2 share the first block # and have different second blocks check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3) # corrupt backup1 config cfg = findfile(BACKUP_DIR, "backup_" + backup1["Name"] + ".cfg") corrupt_backup = open(cfg, "w") assert corrupt_backup assert corrupt_backup.write("{corrupt: definitely") > 0 corrupt_backup.close() cmd.backup_rm(address, backup1["URL"]) # check that the volume now has 2 blocks # backup2 still relies on the backup1 first block check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2) # remove backup 2 and check that all blocks are deleted cmd.backup_rm(address, backup2["URL"]) check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0) # remove volume.cfg then delete the backup volume cfg = findfile(finddir(BACKUP_DIR, VOLUME_NAME), "volume.cfg") os.remove(cfg) cmd.backup_volume_rm(address, VOLUME_NAME, backup_target) info = cmd.backup_volume_list(address, VOLUME_NAME, backup_target)[VOLUME_NAME] assert "cannot find" in info["Messages"]["error"] cmd.sync_agent_server_reset(address) cleanup_controller(grpc_controller) cleanup_replica(grpc_replica1) cleanup_replica(grpc_replica2)
def backup_core(bin, engine_manager_client, # NOQA grpc_controller_client, # NOQA grpc_replica_client, # NOQA grpc_replica_client2, # NOQA backup_target): open_replica(grpc_replica_client) open_replica(grpc_replica_client2) r1_url = grpc_replica_client.url r2_url = grpc_replica_client2.url v = grpc_controller_client.volume_start(replicas=[ r1_url, r2_url, ]) assert v.replicaCount == 2 env = dict(os.environ) backup_type = urlparse(backup_target).scheme # create & process backup1 snapshot1 = snapshot_create(grpc_controller_client.address) output = grpc_replica_client.replica_get().chain[1] assert output == 'volume-snap-{}.img'.format(snapshot1) backup1 = backup_create(grpc_controller_client.address, snapshot1, backup_target, {'name': 'backup1', 'type': backup_type}) backup1_info = backup_inspect(grpc_controller_client.address, backup1) assert backup1_info["URL"] == backup1 assert backup1_info["IsIncremental"] is False assert backup1_info["VolumeName"] == VOLUME_NAME assert backup1_info["VolumeSize"] == SIZE_STR assert backup1_info["SnapshotName"] == snapshot1 assert len(backup1_info["Labels"]) == 2 assert backup1_info["Labels"]["name"] == "backup1" assert backup1_info["Labels"]["type"] == backup_type # create & process backup2 snapshot2 = snapshot_create(grpc_controller_client.address) output = grpc_replica_client.replica_get().chain[1] assert output == 'volume-snap-{}.img'.format(snapshot2) backup2 = backup_create(grpc_controller_client.address, snapshot2, backup_target) backup2_info = backup_inspect(grpc_controller_client.address, backup2) assert backup2_info["URL"] == backup2 assert backup2_info["IsIncremental"] is True assert backup2_info["VolumeName"] == VOLUME_NAME assert backup2_info["VolumeSize"] == SIZE_STR assert backup2_info["SnapshotName"] == snapshot2 if backup2_info["Labels"] is not None: assert len(backup2_info["Labels"]) == 0 # list all known backups for volume volume_info = backup_volume_list(grpc_controller_client.address, VOLUME_NAME, backup_target, include_backup_details=True)[VOLUME_NAME] assert volume_info["Name"] == VOLUME_NAME assert volume_info["Size"] == SIZE_STR backup_list = volume_info["Backups"] assert backup_list[backup1]["URL"] == backup1_info["URL"] assert backup_list[backup1]["SnapshotName"] == backup1_info["SnapshotName"] assert backup_list[backup1]["Size"] == backup1_info["Size"] assert backup_list[backup1]["Created"] == backup1_info["Created"] assert backup_list[backup1]["Messages"] is None assert backup_list[backup2]["URL"] == backup2_info["URL"] assert backup_list[backup2]["SnapshotName"] == backup2_info["SnapshotName"] assert backup_list[backup2]["Size"] == backup2_info["Size"] assert backup_list[backup2]["Created"] == backup2_info["Created"] assert backup_list[backup2]["Messages"] is None # test that corrupt backups are signaled during a list operation # https://github.com/longhorn/longhorn/issues/1212 volume_dir = finddir(BACKUP_DIR, VOLUME_NAME) assert volume_dir assert os.path.exists(volume_dir) backup_dir = os.path.join(volume_dir, "backups") assert os.path.exists(backup_dir) backup_cfg_name = "backup_" + backup2_info["Name"] + ".cfg" assert backup_cfg_name backup_cfg_path = findfile(backup_dir, backup_cfg_name) assert os.path.exists(backup_cfg_path) backup_tmp_cfg_path = os.path.join(volume_dir, backup_cfg_name) os.rename(backup_cfg_path, backup_tmp_cfg_path) assert os.path.exists(backup_tmp_cfg_path) corrupt_backup = open(backup_cfg_path, "w") assert corrupt_backup assert corrupt_backup.write("{corrupt: definitely") > 0 corrupt_backup.close() # request the new backup list volume_info = backup_volume_list(grpc_controller_client.address, VOLUME_NAME, backup_target, include_backup_details=True)[VOLUME_NAME] assert volume_info["Name"] == VOLUME_NAME backup_list = volume_info["Backups"] assert backup_list[backup1]["URL"] == backup1_info["URL"] assert backup_list[backup1]["Messages"] is None assert backup_list[backup2]["URL"] == backup2_info["URL"] assert MESSAGE_TYPE_ERROR in backup_list[backup2]["Messages"] # we still want to fail inspects, since they operate on urls # with no guarantee of backup existence with pytest.raises(subprocess.CalledProcessError): backup_inspect(grpc_controller_client.address, backup2) # switch back to valid cfg os.rename(backup_tmp_cfg_path, backup_cfg_path) assert backup_inspect(grpc_controller_client.address, backup2) # test that list returns a volume_info with an error message # for a missing volume.cfg instead of failing with an error # https://github.com/rancher/longhorn/issues/399 volume_cfg_path = findfile(volume_dir, VOLUME_CONFIG_FILE) assert os.path.exists(volume_cfg_path) volume_tmp_cfg_path = volume_cfg_path.replace( VOLUME_CONFIG_FILE, VOLUME_TMP_CONFIG_FILE) os.rename(volume_cfg_path, volume_tmp_cfg_path) assert os.path.exists(volume_tmp_cfg_path) volume_info = backup_volume_list(grpc_controller_client.address, "", backup_target) assert MESSAGE_TYPE_ERROR in volume_info[VOLUME_NAME]["Messages"] os.rename(volume_tmp_cfg_path, volume_cfg_path) assert os.path.exists(volume_cfg_path) volume_info = backup_volume_list(grpc_controller_client.address, "", backup_target) assert volume_info[VOLUME_NAME]["Messages"] is not None assert MESSAGE_TYPE_ERROR not in volume_info[VOLUME_NAME]["Messages"] # backup doesn't exists so it should error with pytest.raises(subprocess.CalledProcessError): url = backup_target + "?backup=backup-unk" + "&volume=" + VOLUME_NAME backup_inspect(grpc_controller_client.address, url) # this returns unsupported driver since `bad` is not a known scheme with pytest.raises(subprocess.CalledProcessError): backup_inspect(grpc_controller_client.address, "bad://xxx") restore_backup(engine_manager_client, bin, grpc_controller_client.address, backup1, env, grpc_controller_client) restore_backup(engine_manager_client, bin, grpc_controller_client.address, backup2, env, grpc_controller_client) # remove backups + volume backup_rm(grpc_controller_client.address, backup1) backup_rm(grpc_controller_client.address, backup2) backup_volume_rm(grpc_controller_client.address, VOLUME_NAME, backup_target) assert os.path.exists(BACKUP_DIR) assert not os.path.exists(volume_cfg_path) grpc_controller_client.volume_frontend_start( frontend=FRONTEND_TGT_BLOCKDEV) v = grpc_controller_client.volume_get() assert v.frontendState == "up"
def test_backup_S3_latest_unavailable( grpc_replica1, grpc_replica2, # NOQA grpc_controller, backup_targets): # NOQA for backup_target in backup_targets: if "s3://" not in backup_target: continue dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) address = grpc_controller.address volume_name = VOLUME_NAME engine_name = ENGINE_NAME offset = 0 length = 128 # initial backup snap1_data = random_string(length) verify_data(dev, offset, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) backup1_info = create_backup(address, snap1, backup_target) # backup to be unavailable snap2_data = random_string(length) verify_data(dev, offset, snap2_data) snap2 = cmd.snapshot_create(address) backup2_info = create_backup(address, snap2, backup_target) # the gc after the restore will clean up the missing backup cfg = findfile(BACKUP_DIR, "backup_" + backup2_info["Name"] + ".cfg") os.remove(cfg) # final full backup after unavailable backup snap3_data = random_string(length) verify_data(dev, offset, snap3_data) snap3_checksum = checksum_dev(dev) snap3 = cmd.snapshot_create(address) backup3_info = create_backup(address, snap3, backup_target) assert backup3_info["VolumeName"] == volume_name assert backup3_info["Size"] == BLOCK_SIZE_STR # write some stuff on head head_data = random_string(length) verify_data(dev, offset, head_data) # test restore of the initial backup reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, engine_name, backup1_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap1_data c = checksum_dev(dev) assert c == snap1_checksum # test a restore for the final backup reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, engine_name, backup3_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap3_data c = checksum_dev(dev) assert c == snap3_checksum rm_backups(address, engine_name, [backup1_info["URL"], backup3_info["URL"]]) cmd.sync_agent_server_reset(address) cleanup_controller(grpc_controller) cleanup_replica(grpc_replica1) cleanup_replica(grpc_replica2)