def test_transfered_ongoing_overlapping_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) # Start 2 ongoing operations. # ongoing: 0-0, 80-80 # completed: op1 = Operation(0, 120) op2 = Operation(80, 120) ticket._add_operation(op1) ticket._add_operation(op2) assert ticket.transferred() == 0 assert ticket.active() # Consume op1 data: # ongoing: 0-120, 80-80 # completed: op1.run() ticket._remove_operation(op1) assert ticket.transferred() == 120 assert ticket.active() # Consume op2 data: # ongoing: 0-120, 80-200 # completed: op2.run() ticket._remove_operation(op2) assert ticket.transferred() == 200 assert not ticket.active()
def test_run_operation_benchmark(): # Run 1000000 operations with 4 concurrent threads. ticket = Ticket(testutils.create_ticket(ops=["read"])) operations = 10**6 workers = 4 chunk = 10**9 step = chunk * workers // operations def worker(offset, size): while offset < size: ticket.run(Operation(offset, step)) offset += step start = time.time() threads = [] try: for i in range(workers): t = util.start_thread(worker, args=(i * chunk, chunk)) threads.append(t) finally: for t in threads: t.join() elapsed = time.time() - start print("%d operations, %d concurrent threads in %.2f seconds (%d nsec/op)" % (operations, workers, elapsed, elapsed * 10**9 // operations))
def test_transfered_ongoing_non_continues_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) # Start 2 ongoing operations. # ongoing: 0-0, 200-200 # completed: op1 = Operation(0, 100) op2 = Operation(200, 100) ticket._add_operation(op1) ticket._add_operation(op2) assert ticket.transferred() == 0 assert ticket.active() # Consume op1 data: # ongoing: 0-100, 200-200 # completed: op1.run() ticket._remove_operation(op1) assert ticket.transferred() == 100 # Consume op2 data: # ongoing: 0-100, 200-300 # completed: op2.run() ticket._remove_operation(op2) assert ticket.transferred() == 200
def test_transfered_inactive_empty_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) ticket.run(Operation(0, 0)) assert ticket.transferred() == 0 ticket.run(Operation(1000, 0)) assert ticket.transferred() == 0
def test_images_upload_too_big(tmpdir): image_size = 100 image = create_tempfile(tmpdir, "image", "") ticket = testutils.create_ticket(url="file://" + str(image), size=image_size) add_ticket(ticket) res = upload(ticket["uuid"], "b" * (image_size + 1)) assert res.status == 403 assert image.read() == ""
def test_tickets_delete_all(): # Example usage: move host to maintenance for i in range(5): ticket = testutils.create_ticket( url="file:///var/run/vdsm/storage/foo%s" % i) add_ticket(ticket) res = unix_request("DELETE", "/tickets/") assert res.status == 204 pytest.raises(KeyError, tickets.get, ticket["uuid"])
def test_tickets_extend_invalid_timeout(fake_time): ticket = testutils.create_ticket() add_ticket(ticket) prev_ticket = get_ticket(ticket["uuid"]) body = json.dumps({"timeout": "invalid"}) res = unix_request("PATCH", "/tickets/%(uuid)s" % ticket, body) cur_ticket = get_ticket(ticket["uuid"]) assert res.status == 400 assert cur_ticket == prev_ticket
def test_tickets_extend_no_ticket_id(fake_time): ticket = testutils.create_ticket() add_ticket(ticket) prev_ticket = get_ticket(ticket["uuid"]) body = json.dumps({"timeout": 300}) res = unix_request("PATCH", "/tickets/", body) cur_ticket = get_ticket(ticket["uuid"]) assert res.status == 400 assert cur_ticket == prev_ticket
def test_tickets_get_expired_ticket(fake_time): ticket = testutils.create_ticket() add_ticket(ticket) # Make the ticket expire. fake_time.now += 500 res = unix_request("GET", "/tickets/%(uuid)s" % ticket) assert res.status == 200 server_ticket = json.loads(res.read()) assert server_ticket["timeout"] == -200
def test_images_upload_max_size(tmpdir): image_size = 100 content = "b" * image_size image = create_tempfile(tmpdir, "image", "") ticket = testutils.create_ticket(url="file://" + str(image), size=image_size) add_ticket(ticket) res = upload(ticket["uuid"], content) assert res.status == 200 assert image.read() == content
def test_images_download_partial_no_range_empty(tmpdir): # Image is empty, no range, should return an empty file - we return invalid # http response that fail on the client side with BadStatusLine: ''. # See https://bugzilla.redhat.com/1512312 image = create_tempfile(tmpdir, "image") # Empty image ticket = testutils.create_ticket(url="file://" + str(image), size=1024) add_ticket(ticket) res = download(ticket["uuid"]) assert res.status == http_client.OK assert res.length == 0
def test_transfered_inactive_unordered_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) ticket.run(Operation(100, 100)) assert ticket.transferred() == 100 ticket.run(Operation(0, 100)) assert ticket.transferred() == 200 ticket.run(Operation(200, 100)) assert ticket.transferred() == 300
def test_transfered_inactive_non_continuous_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) # Run 2 non-continutes operations ticket.run(Operation(0, 100)) ticket.run(Operation(200, 100)) assert ticket.transferred() == 200 # Run last operation filling the hole - with some overlap. ticket.run(Operation(80, 120)) assert ticket.transferred() == 300
def test_repr(): ticket = Ticket(testutils.create_ticket(ops=["read"], filename="tmp_file")) ticket_repr = repr(ticket) info = ticket.info() del info["timeout"] for key, value in info.items(): pair = "%s=%r" % (key, value) assert pair in ticket_repr
def test_transfered_inactive_overlapping_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) ticket.run(Operation(0, 120)) assert ticket.transferred() == 120 ticket.run(Operation(100, 100)) assert ticket.transferred() == 200 ticket.run(Operation(180, 120)) assert ticket.transferred() == 300
def test_images_download_out_of_range(tmpdir, rng, end): data = "a" * 512 + "b" * 512 image = create_tempfile(tmpdir, "image", data) ticket = testutils.create_ticket(url="file://" + str(image), size=end) add_ticket(ticket) res = download(ticket["uuid"], rng) assert res.status == 403 error = json.loads(res.read()) assert error["code"] == 403 assert error["title"] == "Forbidden"
def test_ticket_run(): ticket = Ticket(testutils.create_ticket(ops=["read"])) op = Operation(0, 100) assert ticket.transferred() == op.done assert op.done == 0 ticket.run(op) assert ticket.transferred() == op.done assert op.done == 100
def test_tickets_put(fake_time): ticket = testutils.create_ticket() body = json.dumps(ticket) res = unix_request("PUT", "/tickets/%(uuid)s" % ticket, body) # Server adds expires key ticket["expires"] = int(util.monotonic_time()) + ticket["timeout"] ticket["active"] = False server_ticket = get_ticket(ticket["uuid"]) assert res.status == 200 assert server_ticket == ticket
def test_images_download_filename_in_ticket(tmpdir): size = 1024 filename = u"\u05d0.raw" # hebrew aleph image = create_tempfile(tmpdir, "image", size=size) ticket = testutils.create_ticket(url="file://" + str(image), size=size, filename=filename) add_ticket(ticket) res = download(ticket["uuid"], "bytes=0-1023") expected = "attachment; filename=\xd7\x90.raw" assert res.getheader("Content-Disposition") == expected
def test_ticket_bind(): ticket = Ticket(testutils.create_ticket()) op = FakeOperation(data=["chunk 1", "chunk 2", "chunk 3"]) bop = ticket.bind(op) assert op in ticket._operations # Use as WebOB.Response.app_iter. data = list(bop) bop.close() assert data == op.data assert not op.active
def test_repr(): ticket = Ticket(testutils.create_ticket( ops=["read"], filename="tmp_file")) ticket_repr = repr(ticket) info = ticket.info() del info["timeout"] for key, value in info.items(): pair = "%s=%r" % (key, value) assert pair in ticket_repr
def test_images_download_partial_not_satistieble(tmpdir): # Image is smaller than ticket size - may happen if engine failed to detect # actual image size reported by vdsm - one byte difference is enough to # cause a failure. # See https://bugzilla.redhat.com/1512315. size = 1024 image = create_tempfile(tmpdir, "image", size=size) ticket = testutils.create_ticket(url="file://" + str(image), size=size + 1) add_ticket(ticket) unsatisfiable_range = "bytes=0-%d" % size # Max is size - 1 res = download(ticket["uuid"], unsatisfiable_range) assert res.status == http_client.REQUESTED_RANGE_NOT_SATISFIABLE
def test_tickets_extend(fake_time): ticket = testutils.create_ticket() add_ticket(ticket) patch = {"timeout": 300} body = json.dumps(patch) fake_time.now += 240 res = unix_request("PATCH", "/tickets/%(uuid)s" % ticket, body) ticket["expires"] = int(fake_time.now + ticket["timeout"]) ticket["active"] = False server_ticket = get_ticket(ticket["uuid"]) assert res.status == 200 assert server_ticket == ticket
def test_benchmark_transferred(self, transferred_gb): ticket = Ticket(testutils.create_ticket(ops=["read"])) operations = transferred_gb * 1024**3 // CHUNK_SIZE ticket._operations = [ FakeOperation(offset=i * CHUNK_SIZE, done=CHUNK_SIZE) for i in range(operations) ] start = time.time() assert ticket.transferred() == transferred_gb * 1024**3 end = time.time() print("%dG file (%d operations) in %.6f seconds" % (transferred_gb, operations, end - start))
def test_tickets_get(fake_time): ticket = testutils.create_ticket(ops=["read"]) add_ticket(ticket) fake_time.now += 200 res = unix_request("GET", "/tickets/%(uuid)s" % ticket) assert res.status == 200 server_ticket = json.loads(res.read()) # The server adds an expires key del server_ticket["expires"] ticket["active"] = False ticket["transferred"] = 0 ticket["timeout"] = 100 assert server_ticket == ticket
def test_images_download_partial_no_range(tmpdir): # The image is smaller than the tiket size, but we don't request a range, # so we should get the existing length of the image, since the ticket size # is only an upper limit. Or maybe we should treat the ticket size as the # expected size? # This is another variant of https://bugzilla.redhat.com/1512315. size = 1024 image = create_tempfile(tmpdir, "image", size=size) ticket = testutils.create_ticket(url="file://" + str(image), size=size + 1) add_ticket(ticket) res = download(ticket["uuid"]) assert res.status == http_client.OK # Should return the available image data, not the ticket size. Reading this # response will fail with IncompleteRead. assert res.length == 1024
def test_keep_connection_on_success(tmpdir): image = create_tempfile(tmpdir, "image") ticket = testutils.create_ticket(url="file://" + str(image)) add_ticket(ticket) uri = "/images/%(uuid)s" % ticket body = "data" with https_connection() as con: # Send the first request - it should succeed... con.request("PUT", uri, body=body) with closing(response(con)) as r1: assert r1.status == 200 # The connection should be open for the next request. con.request("PUT", uri, body=body) with closing(response(con)) as r2: assert r2.status == 200
def test_tickets_extend_expired_ticket(fake_time): ticket = testutils.create_ticket() add_ticket(ticket) # Make the ticket expire. fake_time.now += 500 server_ticket = get_ticket(ticket["uuid"]) assert server_ticket["timeout"] == -200 # Extend the expired ticket. body = json.dumps({"timeout": 300}) res = unix_request("PATCH", "/tickets/%(uuid)s" % ticket, body) assert res.status == 200 server_ticket = get_ticket(ticket["uuid"]) assert server_ticket["timeout"] == 300 fake_time.now += 100 server_ticket = get_ticket(ticket["uuid"]) # Timeout is still ticking. assert server_ticket["timeout"] == 200
def test_ticket_bind(): ticket = Ticket(testutils.create_ticket(ops=["read"])) op = Operation(0, 100) bop = ticket.bind(op) assert ticket.active() assert ticket.transferred() == 0 assert op.done == 0 # Use as WebOB.Response.app_iter. data = list(bop) assert op.done == 100 assert not op.closed assert data == [b"x" * op.done] bop.close() assert not ticket.active() assert ticket.transferred() == op.done assert op.closed
def test_transfered_ongoing_concurrent_ops(): ticket = Ticket(testutils.create_ticket(ops=["read"])) # Start 2 ongoing operations: # ongoing: 0-0, 100-100 # completed: b1 = ticket.bind(Operation(0, 100)) b2 = ticket.bind(Operation(100, 100)) assert ticket.transferred() == 0 assert ticket.active() # Consume b1 data: # ongoing: 0-100, 100-100 # completed: list(b1) assert ticket.transferred() == 100 assert ticket.active() # Consume b2 data: # ongoing: 0-100, 100-200 # completed: list(b2) assert ticket.transferred() == 200 assert ticket.active() # Close first operation: # ongoing: 100-200 # completed: 0-100 b1.close() assert ticket.transferred() == 200 assert ticket.active() # Close last operation: # ongoing: # completed: 0-200 b2.close() assert ticket.transferred() == 200 assert not ticket.active()
def test_transferred_benchmark(concurrent): # Time trransferred call with multiple ongoing and completed operations. ticket = Ticket(testutils.create_ticket(ops=["read"])) calls = 10000 # Add some completed ranges - assume worst case when ranges are not # continues. for i in xrange(concurrent): ticket.run(Operation(i * 1000, 100)) # Add some ongoing operations - assume worst case when ranges are not # continues. for i in xrange(concurrent): ticket._add_operation(Operation(i * 1000 + 200, 100)) # Time transferred call - merging ongoing and completed ranges. start = time.time() for i in xrange(calls): ticket.transferred() elapsed = time.time() - start print("%d concurrent operations, %d calls in %.2f seconds (%d nsec/op)" % (concurrent, calls, elapsed, elapsed * 10**9 // calls))
def test_transferred_benchmark(concurrent): # Time trransferred call with multiple ongoing and completed operations. ticket = Ticket(testutils.create_ticket(ops=["read"])) calls = 10000 # Add some completed ranges - assume worst case when ranges are not # continues. for i in xrange(concurrent): ticket.run(Operation(i * 1000, 100)) # Add some ongoing operations - assume worst case when ranges are not # continues. for i in xrange(concurrent): list(ticket.bind(Operation(i * 1000 + 200, 100))) # Time transferred call - merging ongoing and completed ranges. start = time.time() for i in xrange(calls): ticket.transferred() elapsed = time.time() - start print("%d concurrent operations, %d calls in %.2f seconds (%d nsec/op)" % (concurrent, calls, elapsed, elapsed * 10**9 // calls))
def test_sparse(): ticket = Ticket(testutils.create_ticket(sparse=True)) assert ticket.sparse
def test_transfered_nothing(): ticket = Ticket(testutils.create_ticket(ops=["read"])) assert ticket.transferred() == 0
def test_ticket_run(): ticket = Ticket(testutils.create_ticket()) op = FakeOperation() ticket.run(op) assert op.was_run assert op in ticket._operations
def test_transfer_id(): ticket = Ticket(testutils.create_ticket(transfer_id="123")) assert ticket.transfer_id == "123"
def test_transfer_id_unset(): ticket = Ticket(testutils.create_ticket()) assert ticket.transfer_id is None
def test_sparse_unset(): ticket = Ticket(testutils.create_ticket()) assert not ticket.sparse
def test_active(self, operations, active): ticket = Ticket(testutils.create_ticket()) ticket._operations = operations assert ticket.active() == active
def test_invalid_parameter(kw): with pytest.raises(errors.InvalidTicketParameter): Ticket(testutils.create_ticket(**kw))