def test_reconnect_on_failure(monkeypatch, mock_grpc_server,
buffer_empty_event, app):
status_code = "INTERNAL"
wait_event = threading.Event()
continue_event = threading.Event()
class WaitOnWait(CONDITION_CLS):
def wait(self, *args, **kwargs):
wait_event.set()
continue_event.wait()
return True
@staticmethod
def condition(*args, **kwargs):
return WaitOnWait(*args, **kwargs)
monkeypatch.setattr(StreamingRpc, 'condition', condition)
terminating_span = Span(
intrinsics={'status_code': AttributeValue(string_value=status_code)},
agent_attributes={},
user_attributes={})
span = Span(intrinsics={}, agent_attributes={}, user_attributes={})
@override_generic_settings(
settings, {
'distributed_tracing.enabled': True,
'span_events.enabled': True,
'infinite_tracing.trace_observer_host': 'localhost',
'infinite_tracing.trace_observer_port': mock_grpc_server,
'infinite_tracing.ssl': False,
})
def _test():
app.connect_to_data_collector(None)
# Send a span that will trigger a failure
app._stats_engine.span_stream.put(terminating_span)
assert wait_event.wait(timeout=5)
# Send a normal span afterwards
app._stats_engine.span_stream.put(span)
buffer_empty_event.clear()
# Trigger the event so that a reconnect will occur
continue_event.set()
# Wait for the stream buffer to empty meaning all spans have been sent.
assert buffer_empty_event.wait(10)
app.internal_agent_shutdown(restart=False)
_test()
def span_protos(self, settings):
for i_attrs, u_attrs, a_attrs in self.span_events(
settings, attr_class=SpanProtoAttrs):
yield Span(trace_id=self.trace_id,
intrinsics=i_attrs,
user_attributes=u_attrs,
agent_attributes=a_attrs)
def test_disconnect_on_UNIMPLEMENTED(mock_grpc_server, monkeypatch, app):
event = threading.Event()
class WaitOnNotify(CONDITION_CLS):
def notify_all(self, *args, **kwargs):
event.set()
return super(WaitOnNotify, self).notify_all(*args, **kwargs)
@staticmethod
def condition(*args, **kwargs):
return WaitOnNotify(*args, **kwargs)
monkeypatch.setattr(StreamingRpc, 'condition', condition)
terminating_span = Span(intrinsics={
'status_code':
AttributeValue(string_value='UNIMPLEMENTED')
},
agent_attributes={},
user_attributes={})
@override_generic_settings(
settings, {
'distributed_tracing.enabled': True,
'span_events.enabled': True,
'infinite_tracing.trace_observer_host': 'localhost',
'infinite_tracing.trace_observer_port': mock_grpc_server,
'infinite_tracing.ssl': False,
})
def _test():
app.connect_to_data_collector(None)
# Send a span that will trigger disconnect
app._stats_engine.span_stream.put(terminating_span)
# Wait for the notify event in close to be called
assert event.wait(timeout=5)
# Verify the rpc management thread is killed
rpc_thread = app._active_session._rpc.response_processing_thread
rpc_thread.join(timeout=5)
assert not rpc_thread.is_alive()
_test()
def test_close_while_connected(mock_grpc_server, buffer_empty_event):
channel = grpc.insecure_channel("localhost:%s" % mock_grpc_server)
stream_buffer = StreamBuffer(1)
rpc = StreamingRpc(channel, stream_buffer, DEFAULT_METADATA, record_metric)
rpc.connect()
# Check the processing thread is alive and spans are being sent
assert rpc.response_processing_thread.is_alive()
span = Span(intrinsics={}, agent_attributes={}, user_attributes={})
buffer_empty_event.clear()
stream_buffer.put(span)
assert buffer_empty_event.wait(5)
rpc.close()
assert not rpc.response_processing_thread.is_alive()
def test_infinite_tracing_span_streaming(mock_grpc_server, status_code,
metrics, monkeypatch, app):
event = threading.Event()
class TerminateOnWait(CONDITION_CLS):
def notify_all(self, *args, **kwargs):
event.set()
return super(TerminateOnWait, self).notify_all(*args, **kwargs)
def wait(self, *args, **kwargs):
event.set()
return super(TerminateOnWait, self).wait(*args, **kwargs)
@staticmethod
def condition(*args, **kwargs):
return TerminateOnWait(*args, **kwargs)
monkeypatch.setattr(StreamingRpc, 'condition', condition)
span = Span(
intrinsics={'status_code': AttributeValue(string_value=status_code)},
agent_attributes={},
user_attributes={})
@override_generic_settings(
settings, {
'distributed_tracing.enabled': True,
'span_events.enabled': True,
'infinite_tracing.trace_observer_host': 'localhost',
'infinite_tracing.trace_observer_port': mock_grpc_server,
'infinite_tracing.ssl': False,
})
@validate_metric_payload(metrics=metrics)
def _test():
app.connect_to_data_collector(None)
app._stats_engine.span_stream.put(span)
assert event.wait(timeout=5)
app.harvest(shutdown=True)
_test()
def test_close_while_awaiting_reconnect(mock_grpc_server, monkeypatch):
event = threading.Event()
class WaitOnWait(CONDITION_CLS):
def wait(self, *args, **kwargs):
event.set()
# Call super wait with no arguments to block until a notify
return super(WaitOnWait, self).wait()
@staticmethod
def condition(*args, **kwargs):
return WaitOnWait(*args, **kwargs)
monkeypatch.setattr(StreamingRpc, "condition", condition)
span = Span(
intrinsics={"status_code": AttributeValue(string_value="INTERNAL")},
agent_attributes={},
user_attributes={},
)
endpoint = "localhost:%s" % mock_grpc_server
stream_buffer = StreamBuffer(1)
rpc = StreamingRpc(
endpoint, stream_buffer, DEFAULT_METADATA, record_metric, ssl=False
)
rpc.connect()
# Send a span to trigger reconnect
stream_buffer.put(span)
# Wait until for StreamingRpc to pause before attempting reconnect
assert event.wait(timeout=5)
# Close the rpc
rpc.close()
# Make sure the processing_thread is closed
assert not rpc.response_processing_thread.is_alive()
def test_no_delay_on_ok(mock_grpc_server, monkeypatch, app):
wait_event = threading.Event()
connect_event = threading.Event()
metrics = [('Supportability/InfiniteTracing/Span/gRPC/OK', 1),
('Supportability/InfiniteTracing/Span/Response/Error', None)]
class SetFlagOnWait(CONDITION_CLS):
def wait(self, *args, **kwargs):
wait_event.set()
return super(SetFlagOnWait, self).wait(*args, **kwargs)
@staticmethod
def condition(*args, **kwargs):
return SetFlagOnWait(*args, **kwargs)
monkeypatch.setattr(StreamingRpc, 'condition', condition)
span = Span(
intrinsics={"status_code": AttributeValue(string_value="OK")},
agent_attributes={},
user_attributes={},
)
@override_generic_settings(
settings, {
'distributed_tracing.enabled': True,
'span_events.enabled': True,
'infinite_tracing.trace_observer_host': 'localhost',
'infinite_tracing.trace_observer_port': mock_grpc_server,
'infinite_tracing.ssl': False,
})
@validate_metric_payload(metrics=metrics)
def _test():
def connect_complete():
connect_event.set()
app.connect_to_data_collector(connect_complete)
assert connect_event.wait(timeout=5)
connect_event.clear()
# Send a span that will trigger disconnect
stream_buffer = app._stats_engine.span_stream
rpc = app._active_session._rpc
_rpc = rpc.rpc
def patched_rpc(*args, **kwargs):
connect_event.set()
return _rpc(*args, **kwargs)
rpc.rpc = patched_rpc
# Put a span that will trigger an OK status code and wait for an attempted
# reconnect.
stream_buffer.put(span)
assert connect_event.wait(timeout=5)
rpc.close()
assert not wait_event.is_set()
app.harvest()
_test()