def test_re_blocks_ops_from_queue(self, stage, mocker):
first_connect = pipeline_ops_base.ConnectOperation(
callback=mocker.MagicMock())
first_fake_op = FakeOperation(callback=mocker.MagicMock())
second_connect = pipeline_ops_base.ReconnectOperation(
callback=mocker.MagicMock())
second_fake_op = FakeOperation(callback=mocker.MagicMock())
stage.run_op(first_connect)
stage.run_op(first_fake_op)
stage.run_op(second_connect)
stage.run_op(second_fake_op)
# at this point, ops are pended waiting for the first connect to complete. Verify this and complete the connect.
assert stage.next.run_op.call_count == 1
assert stage.next.run_op.call_args[0][0] == first_connect
stage.next._complete_op(first_connect)
# The connect is complete. This passes down first_fake_op and second_connect and second_fake_op gets pended waiting i
# for second_connect to complete.
# Note: this isn't ideal. In a perfect world, second_connect wouldn't start until first_fake_op is complete, but we
# dont have this logic in place yet.
assert stage.next.run_op.call_count == 3
assert stage.next.run_op.call_args_list[1][0][0] == first_fake_op
assert stage.next.run_op.call_args_list[2][0][0] == second_connect
# now, complete second_connect to give second_fake_op a chance to get passed down
stage.next._complete_op(second_connect)
assert stage.next.run_op.call_count == 4
assert stage.next.run_op.call_args_list[3][0][0] == second_fake_op
def test_fails_active_reconnect_op(self, stage, create_transport, callback, fake_exception):
op = pipeline_ops_base.ReconnectOperation(callback=callback)
callback.reset_mock()
stage.run_op(op)
assert callback.call_count == 0
stage.transport.on_mqtt_connection_failure_handler(fake_exception)
assert_callback_failed(op=op, error=fake_exception)
def test_completes_active_reconenct_op(self, stage, create_transport, callback):
op = pipeline_ops_base.ReconnectOperation(callback=callback)
callback.reset_mock()
stage.run_op(op)
assert callback.call_count == 0
stage.transport.on_mqtt_connected_handler()
assert_callback_succeeded(op=op)
def test_completes_pending_reconnect_op(self, mocker, stage, create_transport):
op = pipeline_ops_base.ReconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connected_handler()
assert_callback_succeeded(op=op)
assert stage._pending_connection_op is None
def test_fails_pending_reconnect_op(self, mocker, stage, create_transport, arbitrary_exception):
op = pipeline_ops_base.ReconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connection_failure_handler(arbitrary_exception)
assert_callback_failed(op=op, error=arbitrary_exception)
assert stage._pending_connection_op is None
def test_does_not_call_handler_with_active_reconnect_op(
self, stage, create_transport, callback, fake_exception
):
op = pipeline_ops_base.ReconnectOperation(callback=callback)
stage.run_op(op)
assert stage.previous.on_connected.call_count == 0
stage.transport.on_mqtt_connection_failure_handler(fake_exception)
assert stage.previous.on_connected.call_count == 0
class TestMQTTTransportStageOnConnected(MQTTTransportStageTestBase):
@pytest.mark.it("Calls self.on_connected when the transport connected event fires")
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(None, id="No pending operation"),
pytest.param(pipeline_ops_base.ConnectOperation(1), id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1), id="Pending ReconnectOperation"),
pytest.param(
pipeline_ops_base.DisconnectOperation(1), id="Pending DisconnectOperation"
),
],
)
def test_connected_handler(self, stage, create_transport, pending_connection_op):
stage._pending_connection_op = pending_connection_op
assert stage.previous.on_connected.call_count == 0
stage.transport.on_mqtt_connected_handler()
assert stage.previous.on_connected.call_count == 1
@pytest.mark.it(
"Completes a pending ConnectOperation with success when the transport connected event fires"
)
def test_completes_pending_connect_op(self, mocker, stage, create_transport):
op = pipeline_ops_base.ConnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connected_handler()
assert_callback_succeeded(op=op)
assert stage._pending_connection_op is None
@pytest.mark.it(
"Completes a pending ReconnectOperation with success when the transport connected event fires"
)
def test_completes_pending_reconnect_op(self, mocker, stage, create_transport):
op = pipeline_ops_base.ReconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connected_handler()
assert_callback_succeeded(op=op)
assert stage._pending_connection_op is None
@pytest.mark.it(
"Ignores a pending DisconnectOperation when the transport connected event fires"
)
def test_ignores_pending_disconnect_op(self, mocker, stage, create_transport):
op = pipeline_ops_base.DisconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connected_handler()
# handler did NOT trigger a callback
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
class TestMQTTTransportStageExecuteOpWithReconnect(MQTTTransportStageTestBase,
RunOpTests):
@pytest.mark.it(
"Sets the ReconnectOperation as the pending connection operation")
def test_sets_pending_operation(self, stage, create_transport,
op_reconnect):
stage.run_op(op_reconnect)
assert stage._pending_connection_op is op_reconnect
@pytest.mark.it("Cancels any already pending connection operation")
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(pipeline_ops_base.ConnectOperation(1),
id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1),
id="Pending ReconnectOperation"),
pytest.param(pipeline_ops_base.DisconnectOperation(1),
id="Pending DisconnectOperation"),
],
)
def test_pending_operation_cancelled(self, mocker, stage, create_transport,
op_reconnect, pending_connection_op):
mocker.spy(pending_connection_op, "complete")
stage._pending_connection_op = pending_connection_op
stage.run_op(op_reconnect)
# Operation has been completed, with an OperationCancelled exception set indicating early cancellation
assert pending_connection_op.complete.call_count == 1
assert (type(pending_connection_op.complete.call_args[1]["error"]) is
pipeline_exceptions.OperationCancelled)
# New operation is now the pending operation
assert stage._pending_connection_op is op_reconnect
@pytest.mark.it("Does an MQTT reconnect via the MQTTTransport")
def test_mqtt_reconnect(self, mocker, stage, create_transport,
op_reconnect):
stage.run_op(op_reconnect)
assert stage.transport.reconnect.call_count == 1
assert stage.transport.reconnect.call_args == mocker.call(
password=stage.sas_token)
@pytest.mark.it(
"Fails the operation and resets the pending connection operation to None, if there is a failure reconnecting in the MQTTTransport"
)
def test_fails_operation(self, mocker, stage, create_transport,
op_reconnect, arbitrary_exception):
stage.transport.reconnect.side_effect = arbitrary_exception
stage.run_op(op_reconnect)
assert op_reconnect.complete.call_count == 1
assert op_reconnect.complete.call_args == mocker.call(
error=arbitrary_exception)
assert stage._pending_connection_op is None
class TestMQTTProviderExecuteOpWithReconnect(RunOpTests):
@pytest.mark.it(
"Sets the ReconnectOperation as the pending connection operation")
def test_sets_pending_operation(self, stage, create_transport,
op_reconnect):
stage.run_op(op_reconnect)
assert stage._pending_connection_op is op_reconnect
@pytest.mark.it("Cancels any already pending connection operation")
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(pipeline_ops_base.ConnectOperation(),
id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(),
id="Pending ReconnectOperation"),
pytest.param(pipeline_ops_base.DisconnectOperation(),
id="Pending DisconnectOperation"),
],
)
def test_pending_operation_cancelled(self, mocker, stage, create_transport,
op_reconnect, pending_connection_op):
pending_connection_op.callback = mocker.MagicMock()
stage._pending_connection_op = pending_connection_op
stage.run_op(op_reconnect)
# Callback has been completed, with a PipelineError set indicating early cancellation
assert_callback_failed(op=pending_connection_op,
error=errors.PipelineError)
# New operation is now the pending operation
assert stage._pending_connection_op is op_reconnect
@pytest.mark.it("Does an MQTT reconnect via the MQTTTransport")
def test_mqtt_reconnect(self, mocker, stage, create_transport,
op_reconnect):
stage.run_op(op_reconnect)
assert stage.transport.reconnect.call_count == 1
assert stage.transport.reconnect.call_args == mocker.call(
password=stage.sas_token)
@pytest.mark.it(
"Fails the operation and resets the pending connection operation to None, if there is a failure reconnecting in the MQTTTransport"
)
def test_fails_operation(self, mocker, stage, create_transport,
op_reconnect, fake_exception):
stage.transport.reconnect.side_effect = fake_exception
stage.run_op(op_reconnect)
assert_callback_failed(op=op_reconnect, error=fake_exception)
assert stage._pending_connection_op is None
class TestMQTTTransportStageExecuteOpWithDisconnect(MQTTTransportStageTestBase, RunOpTests):
@pytest.mark.it("Sets the DisconnectOperation as the pending connection operation")
def test_sets_pending_operation(self, stage, create_transport, op_disconnect):
stage.run_op(op_disconnect)
assert stage._pending_connection_op is op_disconnect
@pytest.mark.it("Cancels any already pending connection operation")
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(pipeline_ops_base.ConnectOperation(1), id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1), id="Pending ReconnectOperation"),
pytest.param(
pipeline_ops_base.DisconnectOperation(1), id="Pending DisconnectOperation"
),
],
)
def test_pending_operation_cancelled(
self, mocker, stage, create_transport, op_disconnect, pending_connection_op
):
pending_connection_op.callback = mocker.MagicMock()
stage._pending_connection_op = pending_connection_op
stage.run_op(op_disconnect)
# Callback has been completed, with an OperationCancelled exception set indicating early cancellation
assert_callback_failed(
op=pending_connection_op, error=pipeline_exceptions.OperationCancelled
)
# New operation is now the pending operation
assert stage._pending_connection_op is op_disconnect
@pytest.mark.it("Does an MQTT disconnect via the MQTTTransport")
def test_mqtt_disconnect(self, mocker, stage, create_transport, op_disconnect):
stage.run_op(op_disconnect)
assert stage.transport.disconnect.call_count == 1
assert stage.transport.disconnect.call_args == mocker.call()
@pytest.mark.it(
"Fails the operation and resets the pending connection operation to None, if there is a failure disconnecting in the MQTTTransport"
)
def test_fails_operation(
self, mocker, stage, create_transport, op_disconnect, arbitrary_exception
):
stage.transport.disconnect.side_effect = arbitrary_exception
stage.run_op(op_disconnect)
assert_callback_failed(op=op_disconnect, error=arbitrary_exception)
assert stage._pending_connection_op is None
def on_token_update_complete(op, error):
op.callback = old_callback
if error:
logger.error(
"{}({}) token update failed. returning failure {}".
format(self.name, op.name, error))
self.send_completed_op_up(op, error=error)
else:
logger.debug(
"{}({}) token update succeeded. reconnecting".format(
self.name, op.name))
self.send_op_down(
pipeline_ops_base.ReconnectOperation(
callback=on_reconnect_complete))
logger.debug(
"{}({}): passing to next stage with updated callback.".
format(self.name, op.name))
def on_token_update_complete(op):
op.callback = old_callback
if op.error:
logger.error(
"{}({}) token update failed. returning failure {}".
format(self.name, op.name, op.error))
operation_flow.complete_op(stage=self, op=op)
else:
logger.debug(
"{}({}) token update succeeded. reconnecting".format(
self.name, op.name))
operation_flow.pass_op_to_next_stage(
stage=self,
op=pipeline_ops_base.ReconnectOperation(
callback=on_reconnect_complete),
)
logger.debug(
"{}({}): passing to next stage with updated callback.".
format(self.name, op.name))
def op_reconnect(mocker):
op = pipeline_ops_base.ReconnectOperation(callback=mocker.MagicMock())
mocker.spy(op, "complete")
return op
class TestMQTTProviderOnDisconnected(object):
@pytest.mark.it(
"Calls self.on_disconnected when the transport disconnected event fires"
)
@pytest.mark.parametrize(
"cause",
[
pytest.param(None, id="No error cause"),
pytest.param(SomeException(), id="With error cause"),
],
)
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(None, id="No pending operation"),
pytest.param(pipeline_ops_base.ConnectOperation(1),
id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1),
id="Pending ReconnectOperation"),
pytest.param(pipeline_ops_base.DisconnectOperation(1),
id="Pending DisconnectOperation"),
],
)
def test_disconnected_handler(self, stage, create_transport,
pending_connection_op, cause):
stage._pending_connection_op = pending_connection_op
assert stage.previous.on_disconnected.call_count == 0
stage.transport.on_mqtt_disconnected_handler(cause)
assert stage.previous.on_disconnected.call_count == 1
@pytest.mark.it(
"Completes a pending DisconnectOperation with success when the transport disconnected event fires without an error cause"
)
def test_compltetes_pending_disconnect_op_when_no_error(
self, mocker, stage, create_transport):
op = pipeline_ops_base.DisconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_disconnected_handler(None)
assert_callback_succeeded(op=op)
assert stage._pending_connection_op is None
@pytest.mark.it(
"Completes a pending DisconnectOperation with success when the transport disconnected event fires with an error cause"
)
def test_completes_pending_disconnect_op_with_error(
self, mocker, stage, create_transport, arbitrary_exception):
op = pipeline_ops_base.DisconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_disconnected_handler(arbitrary_exception)
assert_callback_succeeded(op=op)
assert stage._pending_connection_op is None
@pytest.mark.it(
"Ignores an unrelated pending operation when the transport disconnected event fires"
)
@pytest.mark.parametrize(
"cause",
[
pytest.param(None, id="No error cause"),
pytest.param(SomeException(), id="With error cause"),
],
)
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(pipeline_ops_base.ConnectOperation(1),
id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1),
id="Pending ReconnectOperation"),
],
)
def test_ignores_unrelated_op(self, mocker, stage, create_transport,
pending_connection_op, cause):
stage._pending_connection_op = pending_connection_op
stage.transport.on_mqtt_disconnected_handler(cause)
# The unrelated pending operation is STILL the pending connection op
assert stage._pending_connection_op is pending_connection_op
@pytest.mark.it(
"Triggers the unhandled exception handler (with ConnectionDroppedError) when the disconnect is unexpected"
)
@pytest.mark.parametrize(
"cause",
[
pytest.param(None, id="No error cause"),
pytest.param(SomeException(), id="With error cause"),
],
)
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(None, id="No pending operation"),
pytest.param(pipeline_ops_base.ConnectOperation(1),
id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1),
id="Pending ReconnectOperation"),
],
)
def test_unexpected_disconnect(self, mocker, stage, create_transport,
pending_connection_op, cause):
# A disconnect is unexpected when there is no pending operation, or a pending, non-Disconnect operation
mock_handler = mocker.patch.object(handle_exceptions,
"handle_background_exception")
stage._pending_connection_op = pending_connection_op
stage.transport.on_mqtt_disconnected_handler(cause)
assert mock_handler.call_count == 1
assert isinstance(mock_handler.call_args[0][0],
transport_exceptions.ConnectionDroppedError)
assert mock_handler.call_args[0][0].__cause__ is cause
class TestMQTTProviderOnConnectionFailure(object):
@pytest.mark.it(
"Does not call self.on_connected when the transport connection failure event fires"
)
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(None, id="No pending operation"),
pytest.param(pipeline_ops_base.ConnectOperation(1),
id="Pending ConnectOperation"),
pytest.param(pipeline_ops_base.ReconnectOperation(1),
id="Pending ReconnectOperation"),
pytest.param(pipeline_ops_base.DisconnectOperation(1),
id="Pending DisconnectOperation"),
],
)
def test_does_not_call_connected_handler(self, stage, create_transport,
arbitrary_exception,
pending_connection_op):
# This test is testing negative space - something the function does NOT do - rather than something it does
stage._pending_connection_op = pending_connection_op
assert stage.previous.on_connected.call_count == 0
stage.transport.on_mqtt_connection_failure_handler(arbitrary_exception)
assert stage.previous.on_connected.call_count == 0
@pytest.mark.it(
"Fails a pending ConnectOperation if the connection failure event fires"
)
def test_fails_pending_connect_op(self, mocker, stage, create_transport,
arbitrary_exception):
op = pipeline_ops_base.ConnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connection_failure_handler(arbitrary_exception)
assert_callback_failed(op=op, error=arbitrary_exception)
assert stage._pending_connection_op is None
@pytest.mark.it(
"Fails a pending ReconnectOperation if the connection failure event fires"
)
def test_fails_pending_reconnect_op(self, mocker, stage, create_transport,
arbitrary_exception):
op = pipeline_ops_base.ReconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connection_failure_handler(arbitrary_exception)
assert_callback_failed(op=op, error=arbitrary_exception)
assert stage._pending_connection_op is None
@pytest.mark.it(
"Ignores a pending DisconnectOperation if the connection failure event fires"
)
def test_ignores_pending_disconnect_op(self, mocker, stage,
create_transport,
arbitrary_exception):
op = pipeline_ops_base.DisconnectOperation(callback=mocker.MagicMock())
stage.run_op(op)
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
stage.transport.on_mqtt_connection_failure_handler(arbitrary_exception)
# Assert nothing changed about the operation
assert op.callback.call_count == 0
assert stage._pending_connection_op is op
@pytest.mark.it(
"Triggers the unhandled exception handler (with error cause) when the connection failure is unexpected"
)
@pytest.mark.parametrize(
"pending_connection_op",
[
pytest.param(None, id="No pending operation"),
pytest.param(pipeline_ops_base.DisconnectOperation(1),
id="Pending DisconnectOperation"),
],
)
def test_unexpected_connection_failure(self, mocker, stage,
create_transport,
arbitrary_exception,
pending_connection_op):
# A connection failure is unexpected if there is not a pending Connect/Reconnect operation
# i.e. "Why did we get a connection failure? We weren't even trying to connect!"
mock_handler = mocker.patch.object(handle_exceptions,
"handle_background_exception")
stage._pending_connection_operation = pending_connection_op
stage.transport.on_mqtt_connection_failure_handler(arbitrary_exception)
assert mock_handler.call_count == 1
assert mock_handler.call_args[0][0] is arbitrary_exception
def op_reconnect(mocker):
return pipeline_ops_base.ReconnectOperation(callback=mocker.MagicMock())
def test_calls_handler_with_active_reconnect_op(self, stage, create_transport, callback):
op = pipeline_ops_base.ReconnectOperation(callback=callback)
stage.run_op(op)
assert stage.previous.on_connected.call_count == 0
stage.transport.on_mqtt_connected_handler()
assert stage.previous.on_connected.call_count == 1