class SidecarRecorder(Recorder):
"""Interface for recording spans to a POSIX message queue.
The SidecarRecorder serializes spans to a string representation before
adding them to the queue.
"""
def __init__(self, queue_name: str):
self.queue = MessageQueue(
"/traces-" + queue_name,
max_messages=MAX_SIDECAR_QUEUE_SIZE,
max_message_size=MAX_SIDECAR_MESSAGE_SIZE,
)
def send(self, span: TraceSpanObserver) -> None:
# Don't raise exceptions from here. This is called in the
# request/response path and should finish cleanly.
serialized_str = json.dumps(span._serialize()).encode("utf8")
if len(serialized_str) > MAX_SIDECAR_MESSAGE_SIZE:
logger.warning(
"Trace too big. Traces published to %s are not allowed to be larger "
"than %d bytes. Received trace is %d bytes. This can be caused by "
"an excess amount of tags or a large amount of child spans.",
self.queue.queue.name,
MAX_SIDECAR_MESSAGE_SIZE,
len(serialized_str),
)
try:
self.queue.put(serialized_str, timeout=0)
except TimedOutError:
logger.warning("Trace queue %s is full. Is trace sidecar healthy?",
self.queue.queue.name)
class EventQueue(ContextFactory, config.Parser, Generic[T]):
"""A queue to transfer events to the publisher.
:param name: The name of the event queue to send to. This specifies
which publisher should send the events which can be useful for routing
to different event pipelines (prod/test/v2 etc.).
:param event_serializer: A callable that takes an event object
and returns serialized bytes ready to send on the wire. See below for
options.
"""
def __init__(self, name: str, event_serializer: Callable[[T], bytes]):
self.queue = MessageQueue("/events-" + name,
max_messages=MAX_QUEUE_SIZE,
max_message_size=MAX_EVENT_SIZE)
self.serialize_event = event_serializer
def put(self, event: T) -> None:
"""Add an event to the queue.
The queue is local to the server this code is run on. The event
publisher on the server will take these events and send them to the
collector.
:param event: The event to send. The type of event object passed in
depends on the selected ``event_serializer``.
:raises: :py:exc:`EventTooLargeError` The serialized event is too large.
:raises: :py:exc:`EventQueueFullError` The queue is full. Events are
not being published fast enough.
"""
serialized = self.serialize_event(event)
if len(serialized) > MAX_EVENT_SIZE:
raise EventTooLargeError(len(serialized))
try:
self.queue.put(serialized, timeout=0)
except TimedOutError:
raise EventQueueFullError
def make_object_for_context(self, name: str,
span: Span) -> "EventQueue[T]":
return self
def parse(self, key_path: str,
raw_config: config.RawConfig) -> "EventQueue[T]":
return self
def test_create_queue(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
self.assertEqual(mq.queue.max_messages, 1)
self.assertEqual(mq.queue.max_message_size, 1)
def test_put_zero_timeout(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
mq.put(b"x", timeout=0)
message = mq.get()
self.assertEqual(message, b"x")
def test_put_full_zero_timeout(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
mq.put(b"1", timeout=0)
with self.assertRaises(TimedOutError):
mq.put(b"2", timeout=0)
def test_get_timeout(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
start = time.time()
with self.assertRaises(TimedOutError):
mq.get(timeout=0.1)
elapsed = time.time() - start
self.assertAlmostEqual(elapsed, 0.1, places=2)
def publish_traces() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of trace queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
arg_parser.add_argument(
"--app-name",
default="main",
metavar="NAME",
help="name of app to load from config_file (default: main)",
)
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
config_parser.read_file(args.config_file)
publisher_raw_cfg = dict(
config_parser.items("trace-publisher:" + args.queue_name))
publisher_cfg = config.parse_config(
publisher_raw_cfg,
{
"zipkin_api_url":
config.DefaultFromEnv(config.Endpoint, "BASEPLATE_ZIPKIN_API_URL"),
"post_timeout":
config.Optional(config.Integer, POST_TIMEOUT_DEFAULT),
"max_batch_size":
config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT),
"retry_limit":
config.Optional(config.Integer, RETRY_LIMIT_DEFAULT),
"max_queue_size":
config.Optional(config.Integer, MAX_QUEUE_SIZE),
},
)
trace_queue = MessageQueue(
"/traces-" + args.queue_name,
max_messages=publisher_cfg.max_queue_size,
max_message_size=MAX_SPAN_SIZE,
)
# pylint: disable=maybe-no-member
inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size)
batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE)
metrics_client = metrics_client_from_config(publisher_raw_cfg)
publisher = ZipkinPublisher(
publisher_cfg.zipkin_api_url.address,
metrics_client,
post_timeout=publisher_cfg.post_timeout,
)
while True:
message: Optional[bytes]
try:
message = trace_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
except BatchFull:
serialized = batcher.serialize()
publisher.publish(serialized)
batcher.reset()
batcher.add(message)
def publish_events() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of event queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
config_parser.read_file(args.config_file)
raw_config = dict(config_parser.items("event-publisher:" +
args.queue_name))
cfg = config.parse_config(
raw_config,
{
"collector": {
"hostname": config.String,
"version": config.Optional(config.String, default="2"),
"scheme": config.Optional(config.String, default="https"),
},
"key": {
"name": config.String,
"secret": config.Base64
},
"max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE),
},
)
metrics_client = metrics_client_from_config(raw_config)
event_queue = MessageQueue(
"/events-" + args.queue_name,
max_messages=cfg.max_queue_size,
max_message_size=MAX_EVENT_SIZE,
)
# pylint: disable=maybe-no-member
serializer = SERIALIZER_BY_VERSION[cfg.collector.version]()
batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE)
publisher = BatchPublisher(metrics_client, cfg)
while True:
message: Optional[bytes]
try:
message = event_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
continue
except BatchFull:
pass
serialized = batcher.serialize()
try:
publisher.publish(serialized)
except Exception:
logger.exception("Events publishing failed.")
batcher.reset()
batcher.add(message)
def __init__(self, queue_name: str):
self.queue = MessageQueue(
"/traces-" + queue_name,
max_messages=MAX_SIDECAR_QUEUE_SIZE,
max_message_size=MAX_SIDECAR_MESSAGE_SIZE,
)
def __init__(self, name: str, event_serializer: Callable[[T], bytes]):
self.queue = MessageQueue("/events-" + name,
max_messages=MAX_QUEUE_SIZE,
max_message_size=MAX_EVENT_SIZE)
self.serialize_event = event_serializer
