def test_remove_context_error(cfg): class FailingContext: def __init__(self): self.count = 1 self.closed = False def close(self): if self.count > 0: self.count -= 1 raise RuntimeError("Cannot close yet") self.closed = True ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg) ctx = FailingContext() ticket.add_context(1, ctx) # If closing a context fails, fail. The ticket cannot be removed # until this context is closed successfully. with pytest.raises(RuntimeError): ticket.cancel(timeout=0) assert not ctx.closed # Calling again will close context successfully, and the ticket can # be removed. ticket.cancel(timeout=0) assert ctx.closed
def test_authorizer_add(cfg): auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) assert ticket.uuid == ticket_info["uuid"]
def test_cancel_timeout(cfg): ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg) # Add conection - having connections does not block cancelation, but we # cannot have ongoing operations without a connection. ctx = Context() ticket.add_context(1, ctx) # Ongoing operation blocks cancel. ticket._add_operation(Operation(0, 100)) # Canceling will time out. with pytest.raises(errors.TransferCancelTimeout): ticket.cancel(timeout=0.001) # Ticket is marked as canceled, but the context was not closed. assert ticket.canceled assert not ctx.closed # Caller can poll ticket "active" property and remove the ticket when the # ticket is inactive. info = ticket.info() assert info["canceled"] assert info["active"] assert info["connections"] == 1
def test_remove_context_missing(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.add_context(1, Context()) assert ticket.info()["connections"] == 1 ticket.remove_context(2) assert ticket.info()["connections"] == 1
def test_transfered_ongoing_concurrent_ops(cfg): ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg) # Start 2 ongoing operations: # ongoing: 0-0, 100-100 # completed: op1 = Operation(0, 100) ticket._add_operation(op1) assert ticket.transferred() == 0 assert ticket.active() op2 = Operation(100, 100) ticket._add_operation(op2) assert ticket.transferred() == 0 assert ticket.active() # Consume op1 data: # ongoing: 0-100, 100-100 # completed: op1.run() ticket._remove_operation(op1) assert ticket.transferred() == 100 assert ticket.active() # Consume op2 data: # ongoing: 0-100, 100-200 # completed: op2.run() ticket._remove_operation(op2) assert ticket.transferred() == 200 assert not ticket.active()
def test_remove_context_error(): class FailingContext: def __init__(self): self.count = 1 self.closed = False def close(self): if self.count > 0: self.count -= 1 raise RuntimeError("Cannot close yet") self.closed = True ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.add_context(1, FailingContext()) ticket.cancel(timeout=0) # If closing a context fails, keep it. The ticket cannot be removed until # this context is removed successfully. with pytest.raises(RuntimeError): ticket.remove_context(1) info = ticket.info() assert info["connections"] == 1 # Calling again will close and remove the context successfully. ticket.remove_context(1) info = ticket.info() assert info["connections"] == 0
def test_cancel_wait(): ticket = Ticket(testutil.create_ticket(ops=["read"])) # Add connections using this ticket. connections = [] for i in range(4): ctx = Context() ticket.add_context(i, ctx) connections.append(ctx) def close_connections(): time.sleep(0.1) for i in range(4): ticket.remove_context(i) info = ticket.info() assert not info["canceled"] assert info["connections"] == 4 t = util.start_thread(close_connections) try: ticket.cancel(timeout=10) # After the ticket was canceled, number of connections must be zero. info = ticket.info() assert info["connections"] == 0 # And all contexts must be closed. assert all(ctx.closed for ctx in connections) finally: t.join()
def test_authorizer_remove_timeout(): cfg = config.load(["test/conf.d/daemon.conf"]) auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) ticket.add_context(1, Context()) assert ticket.info()["connections"] == 1 # Use short timeout to keep the tests fast. cfg.control.remove_timeout = 0.001 # Ticket cannot be removed since it is used by connection 1. with pytest.raises(errors.TicketCancelTimeout): auth.remove(ticket.uuid) # Ticket was not removed. assert auth.get(ticket.uuid) is ticket # The connection was closed, the ticket can be removed now. ticket.remove_context(1) assert ticket.info()["connections"] == 0 auth.remove(ticket.uuid) # Ticket was removed. with pytest.raises(KeyError): auth.get(ticket.uuid)
def test_authorizer_remove_async(): cfg = config.load(["test/conf.d/daemon.conf"]) auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) ticket.add_context(1, Context()) assert ticket.info()["connections"] == 1 # Disable the timeout, so removing a ticket cancel the ticket without # waiting, and requiring polling the ticket status. cfg.control.remove_timeout = 0 # Ticket is canceled, but not removed. auth.remove(ticket.uuid) assert ticket.canceled assert ticket.info()["connections"] == 1 # Ticket was not removed. assert auth.get(ticket.uuid) is ticket # The connection was closed, the ticket can be removed now. ticket.remove_context(1) assert ticket.info()["connections"] == 0 auth.remove(ticket.uuid) # Ticket was removed. with pytest.raises(KeyError): auth.get(ticket.uuid)
def test_transfered_ongoing_non_continues_ops(): ticket = Ticket(testutil.create_ticket(ops=["read"])) # Start 2 ongoing operations. # ongoing: 0-0, 200-200 # completed: op1 = Operation(0, 100) op2 = Operation(200, 100) ticket._add_operation(op1) ticket._add_operation(op2) assert ticket.transferred() == 0 assert ticket.active() # Consume op1 data: # ongoing: 0-100, 200-200 # completed: op1.run() ticket._remove_operation(op1) assert ticket.transferred() == 100 # Consume op2 data: # ongoing: 0-100, 200-300 # completed: op2.run() ticket._remove_operation(op2) assert ticket.transferred() == 200
def test_transfered_ongoing_overlapping_ops(): ticket = Ticket(testutil.create_ticket(ops=["read"])) # Start 2 ongoing operations. # ongoing: 0-0, 80-80 # completed: op1 = Operation(0, 120) op2 = Operation(80, 120) ticket._add_operation(op1) ticket._add_operation(op2) assert ticket.transferred() == 0 assert ticket.active() # Consume op1 data: # ongoing: 0-120, 80-80 # completed: op1.run() ticket._remove_operation(op1) assert ticket.transferred() == 120 assert ticket.active() # Consume op2 data: # ongoing: 0-120, 80-200 # completed: op2.run() ticket._remove_operation(op2) assert ticket.transferred() == 200 assert not ticket.active()
def test_run_operation_benchmark(): # Run 1000000 operations with 4 concurrent threads. ticket = Ticket(testutil.create_ticket(ops=["read"])) operations = 10**6 workers = 4 chunk = 10**9 step = chunk * workers // operations def worker(offset, size): while offset < size: ticket.run(Operation(offset, step)) offset += step start = time.time() threads = [] try: for i in range(workers): t = util.start_thread(worker, args=(i * chunk, chunk)) threads.append(t) finally: for t in threads: t.join() elapsed = time.time() - start print("%d operations, %d concurrent threads in %.2f seconds (%d nsec/op)" % (operations, workers, elapsed, elapsed * 10**9 // operations))
def test_transfered_inactive_empty_ops(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.run(Operation(0, 0)) assert ticket.transferred() == 0 ticket.run(Operation(1000, 0)) assert ticket.transferred() == 0
def test_authorizer_add(): cfg = config.load(["test/conf.d/daemon.conf"]) auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) assert ticket.uuid == ticket_info["uuid"]
def test_canceled_fail_add_context(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.cancel() ctx = Context() # Adding new context must fail. with pytest.raises(errors.AuthorizationError): ticket.add_context(2, ctx)
def test_cancel_unused(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.cancel() # Ticket is canceled and can be removed immediately. assert ticket.canceled info = ticket.info() assert info["canceled"] assert info["connections"] == 0
def test_authorizer_remove_async(cfg): auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) ctx = Context() ticket.add_context(1, ctx) idle_ctx = Context() ticket.add_context(2, idle_ctx) assert not ticket.info()["active"] op = Operation(0, 100) ticket._add_operation(op) assert ticket.info()["active"] # Disable the timeout, so removing a ticket cancel the ticket # without waiting, and requiring polling the ticket status. cfg.control.remove_timeout = 0 auth.remove(ticket.uuid) # Ticket is canceled, but not removed. assert ticket.canceled assert auth.get(ticket.uuid) is ticket info = ticket.info() assert info["active"] assert info["connections"] == 2 assert not ctx.closed assert not idle_ctx.closed # Ending the operation makes the ticket inactive. The call raise and # error handller close the connection, which remove the contenxt # from the ticket. try: ticket._remove_operation(op) except errors.AuthorizationError: ticket.remove_context(1) info = ticket.info() assert info["canceled"] assert not info["active"] assert info["connections"] == 1 assert ctx.closed # Idle context not closed yet. assert not idle_ctx.closed # Removing the ticket again close the idle context. auth.remove(ticket.uuid) assert idle_ctx.closed # Ticket was removed. with pytest.raises(KeyError): auth.get(ticket.uuid)
def test_authorizer_remove_unused(cfg): auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) # Ticket is unused so it will be removed. auth.remove(ticket_info["uuid"]) with pytest.raises(KeyError): auth.get(ticket_info["uuid"])
def test_transfered_inactive_non_continuous_ops(): ticket = Ticket(testutil.create_ticket(ops=["read"])) # Run 2 non-continutes operations ticket.run(Operation(0, 100)) ticket.run(Operation(200, 100)) assert ticket.transferred() == 200 # Run last operation filling the hole - with some overlap. ticket.run(Operation(80, 120)) assert ticket.transferred() == 300
def test_transfered_inactive_unordered_ops(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.run(Operation(100, 100)) assert ticket.transferred() == 100 ticket.run(Operation(0, 100)) assert ticket.transferred() == 200 ticket.run(Operation(200, 100)) assert ticket.transferred() == 300
def test_transfered_inactive_overlapping_ops(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.run(Operation(0, 120)) assert ticket.transferred() == 120 ticket.run(Operation(100, 100)) assert ticket.transferred() == 200 ticket.run(Operation(180, 120)) assert ticket.transferred() == 300
def test_ticket_run(): ticket = Ticket(testutil.create_ticket(ops=["read"])) op = Operation(0, 100) assert ticket.transferred() == op.done assert op.done == 0 ticket.run(op) assert ticket.transferred() == op.done assert op.done == 100
def test_authorize_read(cfg): auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["read"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) assert auth.authorize(ticket.uuid, "read") == ticket with pytest.raises(errors.AuthorizationError): auth.authorize(ticket.uuid, "write")
def test_authorize_write(cfg): auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["write"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) assert auth.authorize(ticket.uuid, "write") == ticket # "write" implies also "read". assert auth.authorize(ticket.uuid, "read") == ticket
def test_repr(): ticket = Ticket(testutil.create_ticket(ops=["read"], filename="tmp_file")) ticket_repr = repr(ticket) info = ticket.info() del info["timeout"] for key, value in info.items(): pair = "%s=%r" % (key, value) assert pair in ticket_repr
def test_cancel_async(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.add_context(1, None) ticket.cancel(timeout=0) # Ticket is canceled, but it cannot be removed. assert ticket.canceled info = ticket.info() assert info["canceled"] assert info["connections"] == 1
def test_cancel_no_connection(cfg): ticket = Ticket(testutil.create_ticket(ops=["read"]), cfg) ticket.cancel() # Ticket is canceled and can be removed immediately. assert ticket.canceled info = ticket.info() assert info["canceled"] assert not info["active"] assert info["connections"] == 0
def test_authorize_write(): cfg = config.load(["test/conf.d/daemon.conf"]) auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["write"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) assert auth.authorize(ticket.uuid, "write") == ticket # "write" implies also "read". assert auth.authorize(ticket.uuid, "read") == ticket
def test_authorizer_expired(cfg): auth = Authorizer(cfg) ticket_info = testutil.create_ticket(ops=["write"]) auth.add(ticket_info) ticket = auth.get(ticket_info["uuid"]) # Extending with zero timeout expire the ticket. ticket.extend(0) for op in ("read", "write"): with pytest.raises(errors.AuthorizationError): auth.authorize(ticket.uuid, op)
def test_canceled_fail_run_before(): ticket = Ticket(testutil.create_ticket(ops=["read"])) ticket.cancel() op = Operation() # Running operations must fail. with pytest.raises(errors.AuthorizationError): ticket.run(op) # Operation was not run. assert op.done == 0