def create_lock_file(name, data):
locks_dir = os.path.join(finddir(BACKUP_DIR, VOLUME_NAME), "locks")
tmp = os.path.join(locks_dir, name + ".lck" + ".tmp")
os.makedirs(locks_dir, exist_ok=True)
cfg = open(tmp, "w")
cfg.write(data)
cfg.close()
os.rename(tmp, os.path.join(locks_dir, name + ".lck"))
def test_backup_corrupt_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup blocks
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# corrupt backup1 config
cfg = findfile(BACKUP_DIR, "backup_" + backup1["Name"] + ".cfg")
corrupt_backup = open(cfg, "w")
assert corrupt_backup
assert corrupt_backup.write("{corrupt: definitely") > 0
corrupt_backup.close()
cmd.backup_rm(address, backup1["URL"])
# check that the volume now has 2 blocks
# backup2 still relies on the backup1 first block
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# remove backup 2 and check that all blocks are deleted
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# remove volume.cfg then delete the backup volume
cfg = findfile(finddir(BACKUP_DIR, VOLUME_NAME), "volume.cfg")
os.remove(cfg)
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def create_in_progress_backup_file(volume):
volume_dir = finddir(BACKUP_DIR, volume)
assert os.path.exists(volume_dir)
backup_cfg_dir = os.path.join(volume_dir, "backups")
name = "backup-" + random_string(16)
backup_cfg_path = os.path.join(backup_cfg_dir, "backup_" + name + ".cfg")
cfg = json.dumps({"Name": name, "VolumeName": volume, "CreatedTime": ""})
file = open(backup_cfg_path, "w")
file.write(cfg)
file.close()
return backup_cfg_path
def check_backup_volume_block_count(address, volume, backup_target, expected):
# check the volume block & size
info = cmd.backup_volume_list(address, volume, backup_target)[volume]
assert info["DataStored"] == str(BLOCK_SIZE * expected)
# check the blocks on disk
volume_dir = finddir(BACKUP_DIR, volume)
assert os.path.exists(volume_dir)
block_count = 0
block_dir = os.path.join(volume_dir, "blocks")
if os.path.exists(block_dir):
for _ in Path(block_dir).rglob("*.blk"):
block_count += 1
assert block_count == expected
def backup_core(bin, engine_manager_client, # NOQA
grpc_controller_client, # NOQA
grpc_replica_client, # NOQA
grpc_replica_client2, # NOQA
backup_target):
open_replica(grpc_replica_client)
open_replica(grpc_replica_client2)
r1_url = grpc_replica_client.url
r2_url = grpc_replica_client2.url
v = grpc_controller_client.volume_start(replicas=[
r1_url,
r2_url,
])
assert v.replicaCount == 2
env = dict(os.environ)
backup_type = urlparse(backup_target).scheme
# create & process backup1
snapshot1 = snapshot_create(grpc_controller_client.address)
output = grpc_replica_client.replica_get().chain[1]
assert output == 'volume-snap-{}.img'.format(snapshot1)
backup1 = backup_create(grpc_controller_client.address,
snapshot1, backup_target,
{'name': 'backup1', 'type': backup_type})
backup1_info = backup_inspect(grpc_controller_client.address, backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["IsIncremental"] is False
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == SIZE_STR
assert backup1_info["SnapshotName"] == snapshot1
assert len(backup1_info["Labels"]) == 2
assert backup1_info["Labels"]["name"] == "backup1"
assert backup1_info["Labels"]["type"] == backup_type
# create & process backup2
snapshot2 = snapshot_create(grpc_controller_client.address)
output = grpc_replica_client.replica_get().chain[1]
assert output == 'volume-snap-{}.img'.format(snapshot2)
backup2 = backup_create(grpc_controller_client.address,
snapshot2, backup_target)
backup2_info = backup_inspect(grpc_controller_client.address, backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["IsIncremental"] is True
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == SIZE_STR
assert backup2_info["SnapshotName"] == snapshot2
if backup2_info["Labels"] is not None:
assert len(backup2_info["Labels"]) == 0
# list all known backups for volume
volume_info = backup_volume_list(grpc_controller_client.address,
VOLUME_NAME, backup_target,
include_backup_details=True)[VOLUME_NAME]
assert volume_info["Name"] == VOLUME_NAME
assert volume_info["Size"] == SIZE_STR
backup_list = volume_info["Backups"]
assert backup_list[backup1]["URL"] == backup1_info["URL"]
assert backup_list[backup1]["SnapshotName"] == backup1_info["SnapshotName"]
assert backup_list[backup1]["Size"] == backup1_info["Size"]
assert backup_list[backup1]["Created"] == backup1_info["Created"]
assert backup_list[backup1]["Messages"] is None
assert backup_list[backup2]["URL"] == backup2_info["URL"]
assert backup_list[backup2]["SnapshotName"] == backup2_info["SnapshotName"]
assert backup_list[backup2]["Size"] == backup2_info["Size"]
assert backup_list[backup2]["Created"] == backup2_info["Created"]
assert backup_list[backup2]["Messages"] is None
# test that corrupt backups are signaled during a list operation
# https://github.com/longhorn/longhorn/issues/1212
volume_dir = finddir(BACKUP_DIR, VOLUME_NAME)
assert volume_dir
assert os.path.exists(volume_dir)
backup_dir = os.path.join(volume_dir, "backups")
assert os.path.exists(backup_dir)
backup_cfg_name = "backup_" + backup2_info["Name"] + ".cfg"
assert backup_cfg_name
backup_cfg_path = findfile(backup_dir, backup_cfg_name)
assert os.path.exists(backup_cfg_path)
backup_tmp_cfg_path = os.path.join(volume_dir, backup_cfg_name)
os.rename(backup_cfg_path, backup_tmp_cfg_path)
assert os.path.exists(backup_tmp_cfg_path)
corrupt_backup = open(backup_cfg_path, "w")
assert corrupt_backup
assert corrupt_backup.write("{corrupt: definitely") > 0
corrupt_backup.close()
# request the new backup list
volume_info = backup_volume_list(grpc_controller_client.address,
VOLUME_NAME, backup_target,
include_backup_details=True)[VOLUME_NAME]
assert volume_info["Name"] == VOLUME_NAME
backup_list = volume_info["Backups"]
assert backup_list[backup1]["URL"] == backup1_info["URL"]
assert backup_list[backup1]["Messages"] is None
assert backup_list[backup2]["URL"] == backup2_info["URL"]
assert MESSAGE_TYPE_ERROR in backup_list[backup2]["Messages"]
# we still want to fail inspects, since they operate on urls
# with no guarantee of backup existence
with pytest.raises(subprocess.CalledProcessError):
backup_inspect(grpc_controller_client.address, backup2)
# switch back to valid cfg
os.rename(backup_tmp_cfg_path, backup_cfg_path)
assert backup_inspect(grpc_controller_client.address, backup2)
# test that list returns a volume_info with an error message
# for a missing volume.cfg instead of failing with an error
# https://github.com/rancher/longhorn/issues/399
volume_cfg_path = findfile(volume_dir, VOLUME_CONFIG_FILE)
assert os.path.exists(volume_cfg_path)
volume_tmp_cfg_path = volume_cfg_path.replace(
VOLUME_CONFIG_FILE, VOLUME_TMP_CONFIG_FILE)
os.rename(volume_cfg_path, volume_tmp_cfg_path)
assert os.path.exists(volume_tmp_cfg_path)
volume_info = backup_volume_list(grpc_controller_client.address,
"", backup_target)
assert MESSAGE_TYPE_ERROR in volume_info[VOLUME_NAME]["Messages"]
os.rename(volume_tmp_cfg_path, volume_cfg_path)
assert os.path.exists(volume_cfg_path)
volume_info = backup_volume_list(grpc_controller_client.address,
"", backup_target)
assert volume_info[VOLUME_NAME]["Messages"] is not None
assert MESSAGE_TYPE_ERROR not in volume_info[VOLUME_NAME]["Messages"]
# backup doesn't exists so it should error
with pytest.raises(subprocess.CalledProcessError):
url = backup_target + "?backup=backup-unk" + "&volume=" + VOLUME_NAME
backup_inspect(grpc_controller_client.address, url)
# this returns unsupported driver since `bad` is not a known scheme
with pytest.raises(subprocess.CalledProcessError):
backup_inspect(grpc_controller_client.address, "bad://xxx")
restore_backup(engine_manager_client,
bin, grpc_controller_client.address,
backup1, env, grpc_controller_client)
restore_backup(engine_manager_client,
bin, grpc_controller_client.address,
backup2, env, grpc_controller_client)
# remove backups + volume
backup_rm(grpc_controller_client.address, backup1)
backup_rm(grpc_controller_client.address, backup2)
backup_volume_rm(grpc_controller_client.address,
VOLUME_NAME, backup_target)
assert os.path.exists(BACKUP_DIR)
assert not os.path.exists(volume_cfg_path)
grpc_controller_client.volume_frontend_start(
frontend=FRONTEND_TGT_BLOCKDEV)
v = grpc_controller_client.volume_get()
assert v.frontendState == "up"
def remove_lock_file(name):
locks_dir = os.path.join(finddir(BACKUP_DIR, VOLUME_NAME), "locks")
lock = os.path.join(locks_dir, name + ".lck")
os.remove(lock)
def test_backup_volume_list(
grpc_replica_client,
grpc_controller_client, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
"""
Test backup volume list
Context:
We want to make sure that an error when listing a single backup volume
does not stop us from listing all the other backup volumes. Otherwise a
single faulty backup can block the retrieval of all known backup volumes.
Steps:
1. Create a volume(1,2) and attach to the current node
2. write some data to volume(1,2)
3. Create a backup(1) of volume(1,2)
4. request a backup list
5. verify backup list contains no error messages for volume(1,2)
6. verify backup list contains backup(1) for volume(1,2)
7. place a file named "*****@*****.**"
into the backups folder of volume(1)
8. request a backup list
9. verify backup list contains no error messages for volume(1,2)
10. verify backup list contains backup(1) for volume(1,2)
11. delete backup volumes(1 & 2)
12. cleanup
"""
# create a second volume
grpc2_replica1 = grpc_replica_client(REPLICA_2_NAME + "-1")
grpc2_replica2 = grpc_replica_client(REPLICA_2_NAME + "-2")
grpc2_controller = grpc_controller_client(ENGINE2_NAME, VOLUME2_NAME)
offset = 0
length = 128
address = grpc_controller.address
address2 = grpc2_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
dev2 = get_dev(grpc2_replica1, grpc2_replica2, grpc2_controller)
# create a regular backup
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
# create a regular backup on volume 2
verify_data(dev2, offset, random_string(length))
snap = cmd.snapshot_create(address2)
backup_info = create_backup(address2, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME2_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
# request a volume list
info = cmd.backup_volume_list(address,
"",
backup_target,
include_backup_details=True)
assert info[VOLUME_NAME]["Name"] == VOLUME_NAME
assert len(info[VOLUME_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME_NAME]["Messages"]
assert info[VOLUME2_NAME]["Name"] == VOLUME2_NAME
assert len(info[VOLUME2_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME2_NAME]["Messages"]
# place badly named backup.cfg file
# we want the list call to return all valid files correctly
backup_dir = os.path.join(finddir(BACKUP_DIR, VOLUME_NAME), "backups")
cfg = open(os.path.join(backup_dir, "*****@*****.**"), "w")
cfg.close()
info = cmd.backup_volume_list(address,
"",
backup_target,
include_backup_details=True)
assert info[VOLUME_NAME]["Name"] == VOLUME_NAME
assert len(info[VOLUME_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME_NAME]["Messages"]
assert info[VOLUME2_NAME]["Name"] == VOLUME2_NAME
assert len(info[VOLUME2_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME2_NAME]["Messages"]
# remove the volume with the badly named backup.cfg
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address,
VOLUME_NAME,
backup_target,
include_backup_details=True)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
# remove volume 2 backups
cmd.backup_volume_rm(address, VOLUME2_NAME, backup_target)
info = cmd.backup_volume_list(address,
VOLUME2_NAME,
backup_target,
include_backup_details=True)
assert "cannot find" in info[VOLUME2_NAME]["Messages"]["error"]
# cleanup volume 1
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
# cleanup volume 2
cmd.sync_agent_server_reset(address2)
cleanup_controller(grpc2_controller)
cleanup_replica(grpc2_replica1)
cleanup_replica(grpc2_replica2)
