def test_transfered_ongoing_overlapping_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
# Start 2 ongoing operations.
# ongoing: 0-0, 80-80
# completed:
op1 = Operation(0, 120)
op2 = Operation(80, 120)
ticket._add_operation(op1)
ticket._add_operation(op2)
assert ticket.transferred() == 0
assert ticket.active()
# Consume op1 data:
# ongoing: 0-120, 80-80
# completed:
op1.run()
ticket._remove_operation(op1)
assert ticket.transferred() == 120
assert ticket.active()
# Consume op2 data:
# ongoing: 0-120, 80-200
# completed:
op2.run()
ticket._remove_operation(op2)
assert ticket.transferred() == 200
assert not ticket.active()
def test_run_operation_benchmark():
# Run 1000000 operations with 4 concurrent threads.
ticket = Ticket(testutils.create_ticket(ops=["read"]))
operations = 10**6
workers = 4
chunk = 10**9
step = chunk * workers // operations
def worker(offset, size):
while offset < size:
ticket.run(Operation(offset, step))
offset += step
start = time.time()
threads = []
try:
for i in range(workers):
t = util.start_thread(worker, args=(i * chunk, chunk))
threads.append(t)
finally:
for t in threads:
t.join()
elapsed = time.time() - start
print("%d operations, %d concurrent threads in %.2f seconds (%d nsec/op)"
% (operations, workers, elapsed, elapsed * 10**9 // operations))
def test_transfered_ongoing_non_continues_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
# Start 2 ongoing operations.
# ongoing: 0-0, 200-200
# completed:
op1 = Operation(0, 100)
op2 = Operation(200, 100)
ticket._add_operation(op1)
ticket._add_operation(op2)
assert ticket.transferred() == 0
assert ticket.active()
# Consume op1 data:
# ongoing: 0-100, 200-200
# completed:
op1.run()
ticket._remove_operation(op1)
assert ticket.transferred() == 100
# Consume op2 data:
# ongoing: 0-100, 200-300
# completed:
op2.run()
ticket._remove_operation(op2)
assert ticket.transferred() == 200
def test_transfered_inactive_empty_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
ticket.run(Operation(0, 0))
assert ticket.transferred() == 0
ticket.run(Operation(1000, 0))
assert ticket.transferred() == 0
def test_images_upload_too_big(tmpdir):
image_size = 100
image = create_tempfile(tmpdir, "image", "")
ticket = testutils.create_ticket(url="file://" + str(image),
size=image_size)
add_ticket(ticket)
res = upload(ticket["uuid"], "b" * (image_size + 1))
assert res.status == 403
assert image.read() == ""
def test_tickets_delete_all():
# Example usage: move host to maintenance
for i in range(5):
ticket = testutils.create_ticket(
url="file:///var/run/vdsm/storage/foo%s" % i)
add_ticket(ticket)
res = unix_request("DELETE", "/tickets/")
assert res.status == 204
pytest.raises(KeyError, tickets.get, ticket["uuid"])
def test_tickets_extend_invalid_timeout(fake_time):
ticket = testutils.create_ticket()
add_ticket(ticket)
prev_ticket = get_ticket(ticket["uuid"])
body = json.dumps({"timeout": "invalid"})
res = unix_request("PATCH", "/tickets/%(uuid)s" % ticket, body)
cur_ticket = get_ticket(ticket["uuid"])
assert res.status == 400
assert cur_ticket == prev_ticket
def test_tickets_extend_no_ticket_id(fake_time):
ticket = testutils.create_ticket()
add_ticket(ticket)
prev_ticket = get_ticket(ticket["uuid"])
body = json.dumps({"timeout": 300})
res = unix_request("PATCH", "/tickets/", body)
cur_ticket = get_ticket(ticket["uuid"])
assert res.status == 400
assert cur_ticket == prev_ticket
def test_tickets_get_expired_ticket(fake_time):
ticket = testutils.create_ticket()
add_ticket(ticket)
# Make the ticket expire.
fake_time.now += 500
res = unix_request("GET", "/tickets/%(uuid)s" % ticket)
assert res.status == 200
server_ticket = json.loads(res.read())
assert server_ticket["timeout"] == -200
def test_images_upload_max_size(tmpdir):
image_size = 100
content = "b" * image_size
image = create_tempfile(tmpdir, "image", "")
ticket = testutils.create_ticket(url="file://" + str(image),
size=image_size)
add_ticket(ticket)
res = upload(ticket["uuid"], content)
assert res.status == 200
assert image.read() == content
def test_images_download_partial_no_range_empty(tmpdir):
# Image is empty, no range, should return an empty file - we return invalid
# http response that fail on the client side with BadStatusLine: ''.
# See https://bugzilla.redhat.com/1512312
image = create_tempfile(tmpdir, "image") # Empty image
ticket = testutils.create_ticket(url="file://" + str(image), size=1024)
add_ticket(ticket)
res = download(ticket["uuid"])
assert res.status == http_client.OK
assert res.length == 0
def test_transfered_inactive_unordered_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
ticket.run(Operation(100, 100))
assert ticket.transferred() == 100
ticket.run(Operation(0, 100))
assert ticket.transferred() == 200
ticket.run(Operation(200, 100))
assert ticket.transferred() == 300
def test_transfered_inactive_non_continuous_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
# Run 2 non-continutes operations
ticket.run(Operation(0, 100))
ticket.run(Operation(200, 100))
assert ticket.transferred() == 200
# Run last operation filling the hole - with some overlap.
ticket.run(Operation(80, 120))
assert ticket.transferred() == 300
def test_repr():
ticket = Ticket(testutils.create_ticket(ops=["read"], filename="tmp_file"))
ticket_repr = repr(ticket)
info = ticket.info()
del info["timeout"]
for key, value in info.items():
pair = "%s=%r" % (key, value)
assert pair in ticket_repr
def test_transfered_inactive_overlapping_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
ticket.run(Operation(0, 120))
assert ticket.transferred() == 120
ticket.run(Operation(100, 100))
assert ticket.transferred() == 200
ticket.run(Operation(180, 120))
assert ticket.transferred() == 300
def test_images_download_out_of_range(tmpdir, rng, end):
data = "a" * 512 + "b" * 512
image = create_tempfile(tmpdir, "image", data)
ticket = testutils.create_ticket(url="file://" + str(image), size=end)
add_ticket(ticket)
res = download(ticket["uuid"], rng)
assert res.status == 403
error = json.loads(res.read())
assert error["code"] == 403
assert error["title"] == "Forbidden"
def test_transfered_inactive_non_continuous_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
# Run 2 non-continutes operations
ticket.run(Operation(0, 100))
ticket.run(Operation(200, 100))
assert ticket.transferred() == 200
# Run last operation filling the hole - with some overlap.
ticket.run(Operation(80, 120))
assert ticket.transferred() == 300
def test_ticket_run():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
op = Operation(0, 100)
assert ticket.transferred() == op.done
assert op.done == 0
ticket.run(op)
assert ticket.transferred() == op.done
assert op.done == 100
def test_transfered_inactive_unordered_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
ticket.run(Operation(100, 100))
assert ticket.transferred() == 100
ticket.run(Operation(0, 100))
assert ticket.transferred() == 200
ticket.run(Operation(200, 100))
assert ticket.transferred() == 300
def test_transfered_inactive_overlapping_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
ticket.run(Operation(0, 120))
assert ticket.transferred() == 120
ticket.run(Operation(100, 100))
assert ticket.transferred() == 200
ticket.run(Operation(180, 120))
assert ticket.transferred() == 300
def test_tickets_put(fake_time):
ticket = testutils.create_ticket()
body = json.dumps(ticket)
res = unix_request("PUT", "/tickets/%(uuid)s" % ticket, body)
# Server adds expires key
ticket["expires"] = int(util.monotonic_time()) + ticket["timeout"]
ticket["active"] = False
server_ticket = get_ticket(ticket["uuid"])
assert res.status == 200
assert server_ticket == ticket
def test_ticket_run():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
op = Operation(0, 100)
assert ticket.transferred() == op.done
assert op.done == 0
ticket.run(op)
assert ticket.transferred() == op.done
assert op.done == 100
def test_images_download_filename_in_ticket(tmpdir):
size = 1024
filename = u"\u05d0.raw" # hebrew aleph
image = create_tempfile(tmpdir, "image", size=size)
ticket = testutils.create_ticket(url="file://" + str(image),
size=size,
filename=filename)
add_ticket(ticket)
res = download(ticket["uuid"], "bytes=0-1023")
expected = "attachment; filename=\xd7\x90.raw"
assert res.getheader("Content-Disposition") == expected
def test_ticket_bind():
ticket = Ticket(testutils.create_ticket())
op = FakeOperation(data=["chunk 1", "chunk 2", "chunk 3"])
bop = ticket.bind(op)
assert op in ticket._operations
# Use as WebOB.Response.app_iter.
data = list(bop)
bop.close()
assert data == op.data
assert not op.active
def test_repr():
ticket = Ticket(testutils.create_ticket(
ops=["read"], filename="tmp_file"))
ticket_repr = repr(ticket)
info = ticket.info()
del info["timeout"]
for key, value in info.items():
pair = "%s=%r" % (key, value)
assert pair in ticket_repr
def test_images_download_partial_not_satistieble(tmpdir):
# Image is smaller than ticket size - may happen if engine failed to detect
# actual image size reported by vdsm - one byte difference is enough to
# cause a failure.
# See https://bugzilla.redhat.com/1512315.
size = 1024
image = create_tempfile(tmpdir, "image", size=size)
ticket = testutils.create_ticket(url="file://" + str(image), size=size + 1)
add_ticket(ticket)
unsatisfiable_range = "bytes=0-%d" % size # Max is size - 1
res = download(ticket["uuid"], unsatisfiable_range)
assert res.status == http_client.REQUESTED_RANGE_NOT_SATISFIABLE
def test_tickets_extend(fake_time):
ticket = testutils.create_ticket()
add_ticket(ticket)
patch = {"timeout": 300}
body = json.dumps(patch)
fake_time.now += 240
res = unix_request("PATCH", "/tickets/%(uuid)s" % ticket, body)
ticket["expires"] = int(fake_time.now + ticket["timeout"])
ticket["active"] = False
server_ticket = get_ticket(ticket["uuid"])
assert res.status == 200
assert server_ticket == ticket
def test_benchmark_transferred(self, transferred_gb):
ticket = Ticket(testutils.create_ticket(ops=["read"]))
operations = transferred_gb * 1024**3 // CHUNK_SIZE
ticket._operations = [
FakeOperation(offset=i * CHUNK_SIZE, done=CHUNK_SIZE)
for i in range(operations)
]
start = time.time()
assert ticket.transferred() == transferred_gb * 1024**3
end = time.time()
print("%dG file (%d operations) in %.6f seconds" %
(transferred_gb, operations, end - start))
def test_tickets_get(fake_time):
ticket = testutils.create_ticket(ops=["read"])
add_ticket(ticket)
fake_time.now += 200
res = unix_request("GET", "/tickets/%(uuid)s" % ticket)
assert res.status == 200
server_ticket = json.loads(res.read())
# The server adds an expires key
del server_ticket["expires"]
ticket["active"] = False
ticket["transferred"] = 0
ticket["timeout"] = 100
assert server_ticket == ticket
def test_images_download_partial_no_range(tmpdir):
# The image is smaller than the tiket size, but we don't request a range,
# so we should get the existing length of the image, since the ticket size
# is only an upper limit. Or maybe we should treat the ticket size as the
# expected size?
# This is another variant of https://bugzilla.redhat.com/1512315.
size = 1024
image = create_tempfile(tmpdir, "image", size=size)
ticket = testutils.create_ticket(url="file://" + str(image), size=size + 1)
add_ticket(ticket)
res = download(ticket["uuid"])
assert res.status == http_client.OK
# Should return the available image data, not the ticket size. Reading this
# response will fail with IncompleteRead.
assert res.length == 1024
def test_keep_connection_on_success(tmpdir):
image = create_tempfile(tmpdir, "image")
ticket = testutils.create_ticket(url="file://" + str(image))
add_ticket(ticket)
uri = "/images/%(uuid)s" % ticket
body = "data"
with https_connection() as con:
# Send the first request - it should succeed...
con.request("PUT", uri, body=body)
with closing(response(con)) as r1:
assert r1.status == 200
# The connection should be open for the next request.
con.request("PUT", uri, body=body)
with closing(response(con)) as r2:
assert r2.status == 200
def test_tickets_extend_expired_ticket(fake_time):
ticket = testutils.create_ticket()
add_ticket(ticket)
# Make the ticket expire.
fake_time.now += 500
server_ticket = get_ticket(ticket["uuid"])
assert server_ticket["timeout"] == -200
# Extend the expired ticket.
body = json.dumps({"timeout": 300})
res = unix_request("PATCH", "/tickets/%(uuid)s" % ticket, body)
assert res.status == 200
server_ticket = get_ticket(ticket["uuid"])
assert server_ticket["timeout"] == 300
fake_time.now += 100
server_ticket = get_ticket(ticket["uuid"])
# Timeout is still ticking.
assert server_ticket["timeout"] == 200
def test_ticket_bind():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
op = Operation(0, 100)
bop = ticket.bind(op)
assert ticket.active()
assert ticket.transferred() == 0
assert op.done == 0
# Use as WebOB.Response.app_iter.
data = list(bop)
assert op.done == 100
assert not op.closed
assert data == [b"x" * op.done]
bop.close()
assert not ticket.active()
assert ticket.transferred() == op.done
assert op.closed
def test_transfered_ongoing_concurrent_ops():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
# Start 2 ongoing operations:
# ongoing: 0-0, 100-100
# completed:
b1 = ticket.bind(Operation(0, 100))
b2 = ticket.bind(Operation(100, 100))
assert ticket.transferred() == 0
assert ticket.active()
# Consume b1 data:
# ongoing: 0-100, 100-100
# completed:
list(b1)
assert ticket.transferred() == 100
assert ticket.active()
# Consume b2 data:
# ongoing: 0-100, 100-200
# completed:
list(b2)
assert ticket.transferred() == 200
assert ticket.active()
# Close first operation:
# ongoing: 100-200
# completed: 0-100
b1.close()
assert ticket.transferred() == 200
assert ticket.active()
# Close last operation:
# ongoing:
# completed: 0-200
b2.close()
assert ticket.transferred() == 200
assert not ticket.active()
def test_transferred_benchmark(concurrent):
# Time trransferred call with multiple ongoing and completed operations.
ticket = Ticket(testutils.create_ticket(ops=["read"]))
calls = 10000
# Add some completed ranges - assume worst case when ranges are not
# continues.
for i in xrange(concurrent):
ticket.run(Operation(i * 1000, 100))
# Add some ongoing operations - assume worst case when ranges are not
# continues.
for i in xrange(concurrent):
ticket._add_operation(Operation(i * 1000 + 200, 100))
# Time transferred call - merging ongoing and completed ranges.
start = time.time()
for i in xrange(calls):
ticket.transferred()
elapsed = time.time() - start
print("%d concurrent operations, %d calls in %.2f seconds (%d nsec/op)"
% (concurrent, calls, elapsed, elapsed * 10**9 // calls))
def test_transferred_benchmark(concurrent):
# Time trransferred call with multiple ongoing and completed operations.
ticket = Ticket(testutils.create_ticket(ops=["read"]))
calls = 10000
# Add some completed ranges - assume worst case when ranges are not
# continues.
for i in xrange(concurrent):
ticket.run(Operation(i * 1000, 100))
# Add some ongoing operations - assume worst case when ranges are not
# continues.
for i in xrange(concurrent):
list(ticket.bind(Operation(i * 1000 + 200, 100)))
# Time transferred call - merging ongoing and completed ranges.
start = time.time()
for i in xrange(calls):
ticket.transferred()
elapsed = time.time() - start
print("%d concurrent operations, %d calls in %.2f seconds (%d nsec/op)" %
(concurrent, calls, elapsed, elapsed * 10**9 // calls))
def test_sparse():
ticket = Ticket(testutils.create_ticket(sparse=True))
assert ticket.sparse
def test_transfered_nothing():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
assert ticket.transferred() == 0
def test_ticket_run():
ticket = Ticket(testutils.create_ticket())
op = FakeOperation()
ticket.run(op)
assert op.was_run
assert op in ticket._operations
def test_transfer_id():
ticket = Ticket(testutils.create_ticket(transfer_id="123"))
assert ticket.transfer_id == "123"
def test_transfer_id_unset():
ticket = Ticket(testutils.create_ticket())
assert ticket.transfer_id is None
def test_sparse():
ticket = Ticket(testutils.create_ticket(sparse=True))
assert ticket.sparse
def test_sparse_unset():
ticket = Ticket(testutils.create_ticket())
assert not ticket.sparse
def test_active(self, operations, active):
ticket = Ticket(testutils.create_ticket())
ticket._operations = operations
assert ticket.active() == active
def test_invalid_parameter(kw):
with pytest.raises(errors.InvalidTicketParameter):
Ticket(testutils.create_ticket(**kw))
def test_transfered_nothing():
ticket = Ticket(testutils.create_ticket(ops=["read"]))
assert ticket.transferred() == 0
