def test_remove_context_error(cfg):
class FailingContext:
def __init__(self):
self.count = 1
self.closed = False
def close(self):
if self.count > 0:
self.count -= 1
raise RuntimeError("Cannot close yet")
self.closed = True
ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg)
ctx = FailingContext()
ticket.add_context(1, ctx)
# If closing a context fails, fail. The ticket cannot be removed
# until this context is closed successfully.
with pytest.raises(RuntimeError):
ticket.cancel(timeout=0)
assert not ctx.closed
# Calling again will close context successfully, and the ticket can
# be removed.
ticket.cancel(timeout=0)
assert ctx.closed
def test_authorizer_add(cfg):
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
assert ticket.uuid == ticket_info["uuid"]
def test_cancel_timeout(cfg):
ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg)
# Add conection - having connections does not block cancelation, but we
# cannot have ongoing operations without a connection.
ctx = Context()
ticket.add_context(1, ctx)
# Ongoing operation blocks cancel.
ticket._add_operation(Operation(0, 100))
# Canceling will time out.
with pytest.raises(errors.TransferCancelTimeout):
ticket.cancel(timeout=0.001)
# Ticket is marked as canceled, but the context was not closed.
assert ticket.canceled
assert not ctx.closed
# Caller can poll ticket "active" property and remove the ticket when the
# ticket is inactive.
info = ticket.info()
assert info["canceled"]
assert info["active"]
assert info["connections"] == 1
def test_remove_context_missing():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.add_context(1, Context())
assert ticket.info()["connections"] == 1
ticket.remove_context(2)
assert ticket.info()["connections"] == 1
def test_transfered_ongoing_concurrent_ops(cfg):
ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg)
# Start 2 ongoing operations:
# ongoing: 0-0, 100-100
# completed:
op1 = Operation(0, 100)
ticket._add_operation(op1)
assert ticket.transferred() == 0
assert ticket.active()
op2 = Operation(100, 100)
ticket._add_operation(op2)
assert ticket.transferred() == 0
assert ticket.active()
# Consume op1 data:
# ongoing: 0-100, 100-100
# completed:
op1.run()
ticket._remove_operation(op1)
assert ticket.transferred() == 100
assert ticket.active()
# Consume op2 data:
# ongoing: 0-100, 100-200
# completed:
op2.run()
ticket._remove_operation(op2)
assert ticket.transferred() == 200
assert not ticket.active()
def test_remove_context_error():
class FailingContext:
def __init__(self):
self.count = 1
self.closed = False
def close(self):
if self.count > 0:
self.count -= 1
raise RuntimeError("Cannot close yet")
self.closed = True
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.add_context(1, FailingContext())
ticket.cancel(timeout=0)
# If closing a context fails, keep it. The ticket cannot be removed until
# this context is removed successfully.
with pytest.raises(RuntimeError):
ticket.remove_context(1)
info = ticket.info()
assert info["connections"] == 1
# Calling again will close and remove the context successfully.
ticket.remove_context(1)
info = ticket.info()
assert info["connections"] == 0
def test_cancel_wait():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
# Add connections using this ticket.
connections = []
for i in range(4):
ctx = Context()
ticket.add_context(i, ctx)
connections.append(ctx)
def close_connections():
time.sleep(0.1)
for i in range(4):
ticket.remove_context(i)
info = ticket.info()
assert not info["canceled"]
assert info["connections"] == 4
t = util.start_thread(close_connections)
try:
ticket.cancel(timeout=10)
# After the ticket was canceled, number of connections must be zero.
info = ticket.info()
assert info["connections"] == 0
# And all contexts must be closed.
assert all(ctx.closed for ctx in connections)
finally:
t.join()
def test_authorizer_remove_timeout():
cfg = config.load(["test/conf.d/daemon.conf"])
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
ticket.add_context(1, Context())
assert ticket.info()["connections"] == 1
# Use short timeout to keep the tests fast.
cfg.control.remove_timeout = 0.001
# Ticket cannot be removed since it is used by connection 1.
with pytest.raises(errors.TicketCancelTimeout):
auth.remove(ticket.uuid)
# Ticket was not removed.
assert auth.get(ticket.uuid) is ticket
# The connection was closed, the ticket can be removed now.
ticket.remove_context(1)
assert ticket.info()["connections"] == 0
auth.remove(ticket.uuid)
# Ticket was removed.
with pytest.raises(KeyError):
auth.get(ticket.uuid)
def test_authorizer_remove_async():
cfg = config.load(["test/conf.d/daemon.conf"])
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
ticket.add_context(1, Context())
assert ticket.info()["connections"] == 1
# Disable the timeout, so removing a ticket cancel the ticket without
# waiting, and requiring polling the ticket status.
cfg.control.remove_timeout = 0
# Ticket is canceled, but not removed.
auth.remove(ticket.uuid)
assert ticket.canceled
assert ticket.info()["connections"] == 1
# Ticket was not removed.
assert auth.get(ticket.uuid) is ticket
# The connection was closed, the ticket can be removed now.
ticket.remove_context(1)
assert ticket.info()["connections"] == 0
auth.remove(ticket.uuid)
# Ticket was removed.
with pytest.raises(KeyError):
auth.get(ticket.uuid)
def test_transfered_ongoing_non_continues_ops():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
# Start 2 ongoing operations.
# ongoing: 0-0, 200-200
# completed:
op1 = Operation(0, 100)
op2 = Operation(200, 100)
ticket._add_operation(op1)
ticket._add_operation(op2)
assert ticket.transferred() == 0
assert ticket.active()
# Consume op1 data:
# ongoing: 0-100, 200-200
# completed:
op1.run()
ticket._remove_operation(op1)
assert ticket.transferred() == 100
# Consume op2 data:
# ongoing: 0-100, 200-300
# completed:
op2.run()
ticket._remove_operation(op2)
assert ticket.transferred() == 200
def test_transfered_ongoing_overlapping_ops():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
# Start 2 ongoing operations.
# ongoing: 0-0, 80-80
# completed:
op1 = Operation(0, 120)
op2 = Operation(80, 120)
ticket._add_operation(op1)
ticket._add_operation(op2)
assert ticket.transferred() == 0
assert ticket.active()
# Consume op1 data:
# ongoing: 0-120, 80-80
# completed:
op1.run()
ticket._remove_operation(op1)
assert ticket.transferred() == 120
assert ticket.active()
# Consume op2 data:
# ongoing: 0-120, 80-200
# completed:
op2.run()
ticket._remove_operation(op2)
assert ticket.transferred() == 200
assert not ticket.active()
def test_run_operation_benchmark():
# Run 1000000 operations with 4 concurrent threads.
ticket = Ticket(testutil.create_ticket(ops=["read"]))
operations = 10**6
workers = 4
chunk = 10**9
step = chunk * workers // operations
def worker(offset, size):
while offset < size:
ticket.run(Operation(offset, step))
offset += step
start = time.time()
threads = []
try:
for i in range(workers):
t = util.start_thread(worker, args=(i * chunk, chunk))
threads.append(t)
finally:
for t in threads:
t.join()
elapsed = time.time() - start
print("%d operations, %d concurrent threads in %.2f seconds (%d nsec/op)" %
(operations, workers, elapsed, elapsed * 10**9 // operations))
def test_transfered_inactive_empty_ops():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.run(Operation(0, 0))
assert ticket.transferred() == 0
ticket.run(Operation(1000, 0))
assert ticket.transferred() == 0
def test_authorizer_add():
cfg = config.load(["test/conf.d/daemon.conf"])
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
assert ticket.uuid == ticket_info["uuid"]
def test_canceled_fail_add_context():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.cancel()
ctx = Context()
# Adding new context must fail.
with pytest.raises(errors.AuthorizationError):
ticket.add_context(2, ctx)
def test_cancel_unused():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.cancel()
# Ticket is canceled and can be removed immediately.
assert ticket.canceled
info = ticket.info()
assert info["canceled"]
assert info["connections"] == 0
def test_authorizer_remove_async(cfg):
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
ctx = Context()
ticket.add_context(1, ctx)
idle_ctx = Context()
ticket.add_context(2, idle_ctx)
assert not ticket.info()["active"]
op = Operation(0, 100)
ticket._add_operation(op)
assert ticket.info()["active"]
# Disable the timeout, so removing a ticket cancel the ticket
# without waiting, and requiring polling the ticket status.
cfg.control.remove_timeout = 0
auth.remove(ticket.uuid)
# Ticket is canceled, but not removed.
assert ticket.canceled
assert auth.get(ticket.uuid) is ticket
info = ticket.info()
assert info["active"]
assert info["connections"] == 2
assert not ctx.closed
assert not idle_ctx.closed
# Ending the operation makes the ticket inactive. The call raise and
# error handller close the connection, which remove the contenxt
# from the ticket.
try:
ticket._remove_operation(op)
except errors.AuthorizationError:
ticket.remove_context(1)
info = ticket.info()
assert info["canceled"]
assert not info["active"]
assert info["connections"] == 1
assert ctx.closed
# Idle context not closed yet.
assert not idle_ctx.closed
# Removing the ticket again close the idle context.
auth.remove(ticket.uuid)
assert idle_ctx.closed
# Ticket was removed.
with pytest.raises(KeyError):
auth.get(ticket.uuid)
def test_authorizer_remove_unused(cfg):
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
# Ticket is unused so it will be removed.
auth.remove(ticket_info["uuid"])
with pytest.raises(KeyError):
auth.get(ticket_info["uuid"])
def test_transfered_inactive_non_continuous_ops():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
# Run 2 non-continutes operations
ticket.run(Operation(0, 100))
ticket.run(Operation(200, 100))
assert ticket.transferred() == 200
# Run last operation filling the hole - with some overlap.
ticket.run(Operation(80, 120))
assert ticket.transferred() == 300
def test_transfered_inactive_unordered_ops():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.run(Operation(100, 100))
assert ticket.transferred() == 100
ticket.run(Operation(0, 100))
assert ticket.transferred() == 200
ticket.run(Operation(200, 100))
assert ticket.transferred() == 300
def test_transfered_inactive_overlapping_ops():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.run(Operation(0, 120))
assert ticket.transferred() == 120
ticket.run(Operation(100, 100))
assert ticket.transferred() == 200
ticket.run(Operation(180, 120))
assert ticket.transferred() == 300
def test_ticket_run():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
op = Operation(0, 100)
assert ticket.transferred() == op.done
assert op.done == 0
ticket.run(op)
assert ticket.transferred() == op.done
assert op.done == 100
def test_authorize_read(cfg):
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["read"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
assert auth.authorize(ticket.uuid, "read") == ticket
with pytest.raises(errors.AuthorizationError):
auth.authorize(ticket.uuid, "write")
def test_authorize_write(cfg):
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["write"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
assert auth.authorize(ticket.uuid, "write") == ticket
# "write" implies also "read".
assert auth.authorize(ticket.uuid, "read") == ticket
def test_repr():
ticket = Ticket(testutil.create_ticket(ops=["read"], filename="tmp_file"))
ticket_repr = repr(ticket)
info = ticket.info()
del info["timeout"]
for key, value in info.items():
pair = "%s=%r" % (key, value)
assert pair in ticket_repr
def test_cancel_async():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.add_context(1, None)
ticket.cancel(timeout=0)
# Ticket is canceled, but it cannot be removed.
assert ticket.canceled
info = ticket.info()
assert info["canceled"]
assert info["connections"] == 1
def test_cancel_no_connection(cfg):
ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg)
ticket.cancel()
# Ticket is canceled and can be removed immediately.
assert ticket.canceled
info = ticket.info()
assert info["canceled"]
assert not info["active"]
assert info["connections"] == 0
def test_authorize_write():
cfg = config.load(["test/conf.d/daemon.conf"])
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["write"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
assert auth.authorize(ticket.uuid, "write") == ticket
# "write" implies also "read".
assert auth.authorize(ticket.uuid, "read") == ticket
def test_authorizer_expired(cfg):
auth = Authorizer(cfg)
ticket_info = testutil.create_ticket(ops=["write"])
auth.add(ticket_info)
ticket = auth.get(ticket_info["uuid"])
# Extending with zero timeout expire the ticket.
ticket.extend(0)
for op in ("read", "write"):
with pytest.raises(errors.AuthorizationError):
auth.authorize(ticket.uuid, op)
def test_canceled_fail_run_before():
ticket = Ticket(testutil.create_ticket(ops=["read"]))
ticket.cancel()
op = Operation()
# Running operations must fail.
with pytest.raises(errors.AuthorizationError):
ticket.run(op)
# Operation was not run.
assert op.done == 0
