def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2, recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
job_backup2 = {"name": "backup2", "cron": "* * * * *",
"task": "backup", "retain": 2}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def snapshot_test(clients, volume_name, base_image): # NOQA
for host_id, client in clients.iteritems():
break
volume = create_and_check_volume(client,
volume_name,
base_image=base_image)
lht_hostId = get_self_host_id()
volume = volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
positions = {}
snap1 = volume.snapshotCreate()
snap2_data = write_volume_random_data(volume, positions)
snap2 = volume.snapshotCreate()
snap3_data = write_volume_random_data(volume, positions)
snap3 = volume.snapshotCreate()
snapshots = volume.snapshotList()
snapMap = {}
for snap in snapshots:
snapMap[snap["name"]] = snap
assert snapMap[snap1["name"]]["name"] == snap1["name"]
assert snapMap[snap1["name"]]["removed"] is False
assert snapMap[snap2["name"]]["name"] == snap2["name"]
assert snapMap[snap2["name"]]["parent"] == snap1["name"]
assert snapMap[snap2["name"]]["removed"] is False
assert snapMap[snap3["name"]]["name"] == snap3["name"]
assert snapMap[snap3["name"]]["parent"] == snap2["name"]
assert snapMap[snap3["name"]]["removed"] is False
volume.snapshotDelete(name=snap3["name"])
check_volume_data(volume, snap3_data)
snapshots = volume.snapshotList(volume=volume_name)
snapMap = {}
for snap in snapshots:
snapMap[snap["name"]] = snap
assert snapMap[snap1["name"]]["name"] == snap1["name"]
assert snapMap[snap1["name"]]["removed"] is False
assert snapMap[snap2["name"]]["name"] == snap2["name"]
assert snapMap[snap2["name"]]["parent"] == snap1["name"]
assert snapMap[snap2["name"]]["removed"] is False
assert snapMap[snap3["name"]]["name"] == snap3["name"]
assert snapMap[snap3["name"]]["parent"] == snap2["name"]
assert len(snapMap[snap3["name"]]["children"]) == 1
assert "volume-head" in snapMap[snap3["name"]]["children"]
assert snapMap[snap3["name"]]["removed"] is True
snap = volume.snapshotGet(name=snap3["name"])
assert snap["name"] == snap3["name"]
assert snap["parent"] == snap3["parent"]
assert len(snap3["children"]) == 1
assert len(snap["children"]) == 1
assert "volume-head" in snap3["children"]
assert "volume-head" in snap["children"]
assert snap["removed"] is True
volume.snapshotRevert(name=snap2["name"])
check_volume_data(volume, snap2_data)
snapshots = volume.snapshotList(volume=volume_name)
snapMap = {}
for snap in snapshots:
snapMap[snap["name"]] = snap
assert snapMap[snap1["name"]]["name"] == snap1["name"]
assert snapMap[snap1["name"]]["removed"] is False
assert snapMap[snap2["name"]]["name"] == snap2["name"]
assert snapMap[snap2["name"]]["parent"] == snap1["name"]
assert "volume-head" in snapMap[snap2["name"]]["children"]
assert snap3["name"] in snapMap[snap2["name"]]["children"]
assert snapMap[snap2["name"]]["removed"] is False
assert snapMap[snap3["name"]]["name"] == snap3["name"]
assert snapMap[snap3["name"]]["parent"] == snap2["name"]
assert len(snapMap[snap3["name"]]["children"]) == 0
assert snapMap[snap3["name"]]["removed"] is True
volume.snapshotDelete(name=snap1["name"])
volume.snapshotDelete(name=snap2["name"])
volume.snapshotPurge()
wait_for_snapshot_purge(volume, snap1["name"], snap3["name"])
snapshots = volume.snapshotList(volume=volume_name)
snapMap = {}
for snap in snapshots:
snapMap[snap["name"]] = snap
assert snap1["name"] not in snapMap
assert snap3["name"] not in snapMap
# it's the parent of volume-head, so it cannot be purged at this time
assert snapMap[snap2["name"]]["name"] == snap2["name"]
assert snapMap[snap2["name"]]["parent"] == ""
assert "volume-head" in snapMap[snap2["name"]]["children"]
assert snapMap[snap2["name"]]["removed"] is True
check_volume_data(volume, snap2_data)
cleanup_volume(client, volume)
def volume_basic_test(clients, volume_name, base_image=""): # NOQA
num_hosts = len(clients)
num_replicas = 3
# get a random client
for host_id, client in clients.iteritems():
break
with pytest.raises(Exception):
volume = client.create_volume(name="wrong_volume-name-1.0",
size=SIZE,
numberOfReplicas=2)
volume = client.create_volume(name="wrong_volume-name",
size=SIZE,
numberOfReplicas=2)
volume = client.create_volume(name="wrong_volume-name",
size=SIZE,
numberOfReplicas=2,
frontend="invalid_frontend")
volume = create_and_check_volume(client, volume_name, num_replicas, SIZE,
base_image)
def validate_volume_basic(expected, actual):
assert actual["name"] == expected["name"]
assert actual["size"] == expected["size"]
assert actual["numberOfReplicas"] == expected["numberOfReplicas"]
assert actual["frontend"] == "blockdev"
assert actual["baseImage"] == base_image
assert actual["state"] == expected["state"]
assert actual["created"] == expected["created"]
volumes = client.list_volume()
assert len(volumes) == 1
validate_volume_basic(volume, volumes[0])
volumeByName = client.by_id_volume(volume_name)
validate_volume_basic(volume, volumeByName)
lht_hostId = get_self_host_id()
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
volumeByName = client.by_id_volume(volume_name)
validate_volume_basic(volume, volumeByName)
assert get_volume_endpoint(volumeByName) == DEV_PATH + volume_name
# validate soft anti-affinity
hosts = {}
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
hosts[id] = True
if num_hosts >= num_replicas:
assert len(hosts) == num_replicas
else:
assert len(hosts) == num_hosts
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == volume["name"]
assert volumes[0]["size"] == volume["size"]
assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumes[0]["state"] == volume["state"]
assert volumes[0]["created"] == volume["created"]
assert get_volume_endpoint(volumes[0]) == DEV_PATH + volume_name
volume = client.by_id_volume(volume_name)
assert get_volume_endpoint(volume) == DEV_PATH + volume_name
volume_rw_test(get_volume_endpoint(volume))
cleanup_volume(client, volume)
def test_recurring_job(clients, volume_name): # NOQA
for host_id, client in clients.iteritems():
break
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
# snapshot every one minute
job_snap = {
"name": "snap",
"cron": "* * * * *",
"task": "snapshot",
"retain": 2
}
# backup every two minutes
job_backup = {
"name": "backup",
"cron": "*/2 * * * *",
"task": "backup",
"retain": 1
}
volume.recurringUpdate(jobs=[job_snap, job_backup])
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 snapshots, 1 backup, 1 volume-head
assert count == 4
job_backup2 = {
"name": "backup2",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
volume.recurringUpdate(jobs=[job_snap, job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_volume_basic(clients): # NOQA
# get a random client
for host_id, client in clients.iteritems():
break
with pytest.raises(Exception):
volume = client.create_volume(name="wrong_volume-name-1.0",
size=SIZE,
numberOfReplicas=2)
volume = client.create_volume(name="wrong_volume-name",
size=SIZE,
numberOfReplicas=2)
volume = client.create_volume(name=VOLUME_NAME,
size=SIZE,
numberOfReplicas=2)
assert volume["name"] == VOLUME_NAME
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
volume = wait_for_volume_state(client, VOLUME_NAME, "detached")
# soft anti-affinity should work, and we have 3 nodes
assert len(volume["replicas"]) == 2
hosts = {}
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert id not in hosts
hosts[id] = True
assert len(hosts) == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == volume["name"]
assert volumes[0]["size"] == volume["size"]
assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumes[0]["state"] == volume["state"]
assert volumes[0]["created"] == volume["created"]
volumeByName = client.by_id_volume(VOLUME_NAME)
assert volumeByName["name"] == volume["name"]
assert volumeByName["size"] == volume["size"]
assert volumeByName["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumeByName["state"] == volume["state"]
assert volumeByName["created"] == volume["created"]
volume.attach(hostId=host_id)
volume = wait_for_volume_state(client, VOLUME_NAME, "healthy")
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == volume["name"]
assert volumes[0]["size"] == volume["size"]
assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumes[0]["state"] == volume["state"]
assert volumes[0]["created"] == volume["created"]
# assert volumes[0]["endpoint"] == DEV_PATH + VOLUME_NAME
# volume = client.by_id_volume(VOLUME_NAME)
# assert volume["endpoint"] == DEV_PATH + VOLUME_NAME
volume = volume.detach()
wait_for_volume_state(client, VOLUME_NAME, "detached")
client.delete(volume)
wait_for_volume_delete(client, VOLUME_NAME)
volumes = client.list_volume()
assert len(volumes) == 0
def get_random_client(clients): # NOQA
for host_id, client in clients.iteritems():
break
return client
def test_recurring_snapshot(clients): # NOQA
for host_id, client in clients.iteritems():
break
volume = client.create_volume(name=VOLUME_NAME,
size=SIZE,
numberOfReplicas=2)
volume = wait_for_volume_state(client, VOLUME_NAME, "detached")
snap2s = {
"name": "snap2s",
"cron": "@every 2s",
"task": "snapshot",
"retain": 3
}
snap3s = {
"name": "snap3s",
"cron": "@every 3s",
"task": "snapshot",
"retain": 2
}
volume.recurringUpdate(jobs=[snap2s, snap3s])
time.sleep(0.1)
volume = volume.attach(hostId=host_id)
volume = wait_for_volume_state(client, VOLUME_NAME, "healthy")
time.sleep(10)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
assert count == 5
snap4s = {
"name": "snap4s",
"cron": "@every 4s",
"task": "snapshot",
"retain": 2
}
volume.recurringUpdate(jobs=[snap2s, snap4s])
time.sleep(10)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
assert count == 7
volume = volume.detach()
wait_for_volume_state(client, VOLUME_NAME, "detached")
client.delete(volume)
wait_for_volume_delete(client, VOLUME_NAME)
volumes = client.list_volume()
assert len(volumes) == 0
def test_listing_backup_volume(clients, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
lht_hostId = get_self_host_id()
# create 3 volumes.
volume1_name = generate_volume_name()
volume2_name = generate_volume_name()
volume3_name = generate_volume_name()
volume1 = create_volume(client, volume1_name)
volume2 = create_volume(client, volume2_name)
volume3 = create_volume(client, volume3_name)
volume1.attach(hostId=lht_hostId)
volume1 = common.wait_for_volume_healthy(client, volume1_name)
volume2.attach(hostId=lht_hostId)
volume2 = common.wait_for_volume_healthy(client, volume2_name)
volume3.attach(hostId=lht_hostId)
volume3 = common.wait_for_volume_healthy(client, volume3_name)
# we only test NFS here.
# Since it is difficult to directly remove volume.cfg from s3 buckets
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_nfs(backupstore):
updated = False
for i in range(RETRY_COMMAND_COUNT):
nfs_url = backupstore.strip("nfs://")
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
if "nfs" in setting["value"]:
updated = True
break
assert updated
_, _, snap1, _ = create_backup(client, volume1_name)
_, _, snap2, _ = create_backup(client, volume2_name)
_, _, snap3, _ = create_backup(client, volume3_name)
# invalidate backup volume 1 by renaming volume.cfg to volume.cfg.tmp
cmd = ["mkdir", "-p", "/mnt/nfs"]
subprocess.check_output(cmd)
cmd = ["mount", "-t", "nfs4", nfs_url, "/mnt/nfs"]
subprocess.check_output(cmd)
cmd = ["find", "/mnt/nfs", "-type", "d", "-name", volume1_name]
volume1_backup_volume_path = subprocess.check_output(cmd).strip()
cmd = ["find", volume1_backup_volume_path, "-name", "volume.cfg"]
volume1_backup_volume_cfg_path = subprocess.check_output(cmd).strip()
cmd = ["mv", volume1_backup_volume_cfg_path,
volume1_backup_volume_cfg_path + ".tmp"]
subprocess.check_output(cmd)
subprocess.check_output(["sync"])
found1 = found2 = found3 = False
for i in range(RETRY_COUNTS):
bvs = client.list_backupVolume()
for bv in bvs:
if bv["name"] == volume1_name:
if "error" in bv.messages:
assert "volume.cfg" in bv.messages["error"].lower()
found1 = True
elif bv["name"] == volume2_name:
assert not bv.messages
found2 = True
elif bv["name"] == volume3_name:
assert not bv.messages
found3 = True
if found1 & found2 & found3:
break
time.sleep(RETRY_INTERVAL)
assert found1 & found2 & found3
cmd = ["mv", volume1_backup_volume_cfg_path + ".tmp",
volume1_backup_volume_cfg_path]
subprocess.check_output(cmd)
subprocess.check_output(["sync"])
found = False
for i in range(RETRY_COMMAND_COUNT):
try:
bv1, b1 = common.find_backup(client, volume1_name, snap1["name"])
found = True
break
except Exception:
time.sleep(1)
assert found
bv1.backupDelete(name=b1["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups1 = bv1.backupList()
for b in backups1:
if b["snapshotName"] == snap1["name"]:
found = True
break
assert not found
bv2, b2 = common.find_backup(client, volume2_name, snap2["name"])
bv2.backupDelete(name=b2["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups2 = bv2.backupList()
for b in backups2:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
bv3, b3 = common.find_backup(client, volume3_name, snap3["name"])
bv3.backupDelete(name=b3["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups3 = bv3.backupList()
for b in backups3:
if b["snapshotName"] == snap3["name"]:
found = True
break
assert not found
volume1.detach()
volume1 = common.wait_for_volume_detached(client, volume1_name)
client.delete(volume1)
wait_for_volume_delete(client, volume1_name)
volume2.detach()
volume2 = common.wait_for_volume_detached(client, volume2_name)
client.delete(volume2)
wait_for_volume_delete(client, volume2_name)
volume3.detach()
volume3 = common.wait_for_volume_detached(client, volume3_name)
client.delete(volume3)
wait_for_volume_delete(client, volume3_name)
volumes = client.list_volume()
assert len(volumes) == 0
