def test_orphan_creation_and_deletion_in_multiple_disks(
client, volume_name, request): # NOQA
"""
Test orphan creation and deletion in multiple disks
1. Create multiple new-disks for holding orphaned replica directories
2. Create a volume and attach to the current node
3. Create multiple orphaned replica directories by copying the active
replica directory
4. Clean up volume
5. Verify orphan list contains the orphan CRs for replica
directories
6. Delete all orphaned CRs
7. Verify orphan list is empty
8. Verify orphaned replica directories are deleted
"""
disk_names = [
"vol-disk-" + generate_random_id(4),
"vol-disk-" + generate_random_id(4)
]
# Step 1
lht_hostId = get_self_host_id()
cleanup_node_disks(client, lht_hostId)
disk_paths = crate_disks_on_host(client, disk_names, request)
# Step 2
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 3
num_orphans = 5
create_orphaned_directories_on_host(volume, disk_paths, num_orphans)
# Step 4
cleanup_volume_by_name(client, volume_name)
# Step 5
count = wait_for_orphan_count(client, num_orphans * len(disk_paths), 180)
assert count == num_orphans * len(disk_paths)
# Step 6
delete_orphans(client)
# Step 7
assert wait_for_orphan_count(client, 0, 180) == 0
# Step 8
assert wait_for_file_count(os.path.join(disk_paths[0], "replicas"), 0,
180) == 0
assert wait_for_file_count(os.path.join(disk_paths[1], "replicas"), 0,
180) == 0
def test_delete_orphan_after_orphaned_dir_is_deleted(client, volume_name,
request): # NOQA
"""
Test the immediate deletion of orphan CRs after the orphaned replica
directory is deleted
1. Create a new disk for holding valid and invalid orphaned replica
directories
2. Create a volume and attach to the current node
3. Create a valid orphaned replica directories by copying the active
replica directory
4. Clean up volume
5. Verify orphan list contains the orphan CR for the orphaned replica
directories
6. Delete the on-disk orphaned replica directories
7. Delete the orphan CRs immediately
8. Verify orphan list is empty
"""
disk_names = ["vol-disk-" + generate_random_id(4)]
# Step 1
lht_hostId = get_self_host_id()
cleanup_node_disks(client, lht_hostId)
disk_paths = crate_disks_on_host(client, disk_names, request)
# Step 2
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 3
orphaned_directories = create_orphaned_directories_on_host(
volume, disk_paths, 1)
# Step 4
cleanup_volume_by_name(client, volume_name)
# Step 5
assert wait_for_orphan_count(client, 1, 180) == 1
# Step 6
delete_orphaned_directory_on_host(orphaned_directories)
# Step 7
delete_orphans(client)
# Step 8
assert wait_for_orphan_count(client, 0, 180) == 0
def test_orphan_auto_deletion(client, volume_name, request): # NOQA
"""
Test orphaned dirs creation and background deletion in multiple disks
1. Create a new disks for holding orphaned replica directories
2. Create a volume and attach to the current node
3. Create orphaned replica directories by copying the active
replica directory
4. Clean up volume
5. Verify orphan list contains the orphan CRs for replica
directories
6. Enable the orphan-auto-deletion setting
7. Verify orphan list is empty and the orphaned directory is
deleted in background
8. Clean up disk
"""
disk_names = ["vol-disk-" + generate_random_id(4)]
# Step 1
lht_hostId = get_self_host_id()
cleanup_node_disks(client, lht_hostId)
disk_paths = crate_disks_on_host(client, disk_names, request)
# Step 2
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 3
create_orphaned_directories_on_host(volume, disk_paths, 1)
# Step 4
cleanup_volume_by_name(client, volume_name)
# Step 5
assert wait_for_orphan_count(client, 1, 180) == 1
# Step 6: enable orphan auto deletion
setting = client.by_id_setting(SETTING_ORPHAN_AUTO_DELETION)
client.update(setting, value="true")
# Step 7
assert wait_for_orphan_count(client, 0, 180) == 0
assert wait_for_file_count(os.path.join(disk_paths[0], "replicas"), 0,
180) == 0
def test_orphaned_replica_dir_missing(client, volume_name, request): # NOQA
"""
Test orphan CRs are deleted in background if the orphaned replica
directories are missing
1. Create a new disk for holding valid and invalid orphaned replica
directories
2. Create a volume and attach to the current node
3. Create a orphaned replica directories by copying the active
replica directory
4. Clean up volume
5. Verify orphan list contains the orphan CRs for the orphaned replica
directories
6. Delete the on-disk orphaned replica directories
7. Verify the orphan CR is deleted in background
8. Clean up disk
"""
disk_names = ["vol-disk-" + generate_random_id(4)]
# Step 1
lht_hostId = get_self_host_id()
cleanup_node_disks(client, lht_hostId)
disk_paths = crate_disks_on_host(client, disk_names, request)
# Step 2
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 3
orphaned_directories = create_orphaned_directories_on_host(
volume, disk_paths, 1)
# Step 4
cleanup_volume_by_name(client, volume_name)
# Step 5
assert wait_for_orphan_count(client, 1, 180) == 1
# Step 6
delete_orphaned_directory_on_host(orphaned_directories)
# Step 7
assert wait_for_orphan_count(client, 0, 180) == 0
def test_orphan_with_same_orphaned_dir_name_in_another_disk(
client, volume_name, request): # NOQA
"""
Test orphan creation and deletion with same orphaned dir name in
another disk
1. Create a volume and attach to the current node's default disk
2. Create a new disks for holding orphaned replica directories
3. Create orphaned replica directories by copying the active
replica directory
4. Clean up volume
5. Verify orphan list contains the orphan CRs for replica directories
6. Delete the orphaned replica directories
7. Verify orphan list is empty
"""
disk_names = ["vol-disk-" + generate_random_id(4)]
# Step 1
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 2
disk_paths = crate_disks_on_host(client, disk_names, request)
# Step 3
create_orphaned_directories_on_host(volume, disk_paths, 1)
# Step 4
cleanup_volume_by_name(client, volume_name)
# Step 5
assert wait_for_orphan_count(client, 1, 180) == 1
# Step 6
delete_orphans(client)
# Step 7
assert wait_for_orphan_count(client, 0, 180) == 0
def test_orphaned_dirs_in_duplicated_disks(client, volume_name,
request): # NOQA
"""
Test orphaned dirs in duplicated disks. LH should not create a orphan CR
for the orphaned dir in the deduplicate and unscheduled disk.
1. Create a new disk for holding orphaned replica directories
2. Create a folder under the new disk. This folder will be the duplicated
disk. Add it to the node.
3. Create a volume and attach to the current node
4. Create multiple orphaned replica directories in the two disks by
copying the active replica directory
5. Clean up volume
6. Verify orphan list only contains the orphan CRs for replica directories
in the ready disk
7. Delete all orphan CRs
8. Verify orphan list is empty
9. Verify orphaned directories in the new disk are deleted
10. Verify orphaned directories in the duplicated disk are no deleted
"""
disk_names = [
"vol-disk-" + generate_random_id(4),
"vol-disk-" + generate_random_id(4)
]
# Step 1
disk_paths = []
lht_hostId = get_self_host_id()
cleanup_node_disks(client, lht_hostId)
disk_paths = crate_disks_on_host(client, [disk_names[0]], request)
# Step 2: create duplicated disks for node
node = client.by_id_node(lht_hostId)
disks = node.disks
disk_path = os.path.join(disk_paths[0], disk_names[1])
disk_paths.append(disk_path)
exec_nsenter("mkdir -p {}".format(disk_path))
disk2 = {"path": disk_path, "allowScheduling": True}
update_disk = get_update_disks(disks)
update_disk[disk_names[1]] = disk2
node = node.diskUpdate(disks=update_disk)
node = wait_for_disk_update(client, lht_hostId, len(update_disk))
# Step 3
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 4
num_orphans = 5
create_orphaned_directories_on_host(volume, disk_paths, num_orphans)
# Step 5
cleanup_volume_by_name(client, volume_name)
# Step 6
assert wait_for_orphan_count(client, num_orphans, 180) == num_orphans
# Step 7
delete_orphans(client)
# Step 8
count = wait_for_orphan_count(client, 0, 180)
assert count == 0
# Step 9: The orphaned directories in the ready disk should be deleted
assert wait_for_file_count(os.path.join(disk_paths[0], "replicas"), 0,
180) == 0
# Step 10: The orphaned directories in the duplicated disk should not be
# deleted
assert wait_for_file_count(os.path.join(disk_paths[1], "replicas"),
num_orphans, 180) == num_orphans
def test_disk_evicted(client, volume_name, request): # NOQA
"""
Test the orphan CR is deleted in background but on-disk data still exists
if the disk is evicted
1. Create a new disk for holding valid and invalid orphaned
replica directories
2. Create a volume and attach to the current node
3. Create a valid orphaned replica directories by copying the active
replica directory
4. Clean up volume
5. Verify orphan list contains the valid orphaned replica directory
6. Evict the disk containing the orphaned replica directory
7. Verify the orphan CR is deleted in background, but the on-disk orphaned
replica directory still exists
8. Set the disk scheduleable again
9. Verify the orphan CR is created again and the on-disk orphaned replica
directory still exists
"""
disk_names = ["vol-disk-" + generate_random_id(4)]
lht_hostId = get_self_host_id()
# Step 1
cleanup_node_disks(client, lht_hostId)
disk_paths = crate_disks_on_host(client, disk_names, request)
# Step 2
volume = create_volume_with_replica_on_host(client, volume_name)
# Step 3
create_orphaned_directories_on_host(volume, disk_paths, 1)
# Step 4
cleanup_volume_by_name(client, volume_name)
# Step 5
assert wait_for_orphan_count(client, 1, 180) == 1
# Step 6: Request disk eviction evictionRequested
node = client.by_id_node(lht_hostId)
update_disks = get_update_disks(node.disks)
update_disks[disk_names[0]].allowScheduling = False
update_disks[disk_names[0]].evictionRequested = True
node = node.diskUpdate(disks=update_disks)
node = wait_for_disk_update(client, node.name, len(update_disks))
# Step 7
assert wait_for_orphan_count(client, 0, 180) == 0
assert wait_for_file_count(os.path.join(disk_paths[0], "replicas"), 1,
180) == 1
# Step 8: Set disk allowScheduling to true and evictionRequested to false
node = client.by_id_node(lht_hostId)
update_disks = get_update_disks(node.disks)
update_disks[disk_names[0]].allowScheduling = True
update_disks[disk_names[0]].evictionRequested = False
node = node.diskUpdate(disks=update_disks)
node = wait_for_disk_update(client, node.name, len(update_disks))
# Step 9
assert wait_for_orphan_count(client, 1, 180) == 1
assert wait_for_file_count(os.path.join(disk_paths[0], "replicas"), 1,
180) == 1