def test_create_queue(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
self.assertEqual(mq.queue.max_messages, 1)
self.assertEqual(mq.queue.max_message_size, 1)
def test_put_zero_timeout(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
mq.put(b"x", timeout=0)
message = mq.get()
self.assertEqual(message, b"x")
def test_put_full_zero_timeout(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
mq.put(b"1", timeout=0)
with self.assertRaises(TimedOutError):
mq.put(b"2", timeout=0)
def test_get_timeout(self):
message_queue = MessageQueue(self.qname,
max_messages=1,
max_message_size=1)
with contextlib.closing(message_queue) as mq:
start = time.time()
with self.assertRaises(TimedOutError):
mq.get(timeout=0.1)
elapsed = time.time() - start
self.assertAlmostEqual(elapsed, 0.1, places=2)
def publish_traces() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of trace queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
arg_parser.add_argument(
"--app-name",
default="main",
metavar="NAME",
help="name of app to load from config_file (default: main)",
)
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
config_parser.read_file(args.config_file)
publisher_raw_cfg = dict(
config_parser.items("trace-publisher:" + args.queue_name))
publisher_cfg = config.parse_config(
publisher_raw_cfg,
{
"zipkin_api_url":
config.DefaultFromEnv(config.Endpoint, "BASEPLATE_ZIPKIN_API_URL"),
"post_timeout":
config.Optional(config.Integer, POST_TIMEOUT_DEFAULT),
"max_batch_size":
config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT),
"retry_limit":
config.Optional(config.Integer, RETRY_LIMIT_DEFAULT),
"max_queue_size":
config.Optional(config.Integer, MAX_QUEUE_SIZE),
},
)
trace_queue = MessageQueue(
"/traces-" + args.queue_name,
max_messages=publisher_cfg.max_queue_size,
max_message_size=MAX_SPAN_SIZE,
)
# pylint: disable=maybe-no-member
inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size)
batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE)
metrics_client = metrics_client_from_config(publisher_raw_cfg)
publisher = ZipkinPublisher(
publisher_cfg.zipkin_api_url.address,
metrics_client,
post_timeout=publisher_cfg.post_timeout,
)
while True:
message: Optional[bytes]
try:
message = trace_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
except BatchFull:
serialized = batcher.serialize()
publisher.publish(serialized)
batcher.reset()
batcher.add(message)
def publish_events() -> None:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of event queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser(
interpolation=EnvironmentInterpolation())
config_parser.read_file(args.config_file)
raw_config = dict(config_parser.items("event-publisher:" +
args.queue_name))
cfg = config.parse_config(
raw_config,
{
"collector": {
"hostname": config.String,
"version": config.Optional(config.String, default="2"),
"scheme": config.Optional(config.String, default="https"),
},
"key": {
"name": config.String,
"secret": config.Base64
},
"max_queue_size": config.Optional(config.Integer, MAX_QUEUE_SIZE),
},
)
metrics_client = metrics_client_from_config(raw_config)
event_queue = MessageQueue(
"/events-" + args.queue_name,
max_messages=cfg.max_queue_size,
max_message_size=MAX_EVENT_SIZE,
)
# pylint: disable=maybe-no-member
serializer = SERIALIZER_BY_VERSION[cfg.collector.version]()
batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE)
publisher = BatchPublisher(metrics_client, cfg)
while True:
message: Optional[bytes]
try:
message = event_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
continue
except BatchFull:
pass
serialized = batcher.serialize()
try:
publisher.publish(serialized)
except Exception:
logger.exception("Events publishing failed.")
batcher.reset()
batcher.add(message)
def __init__(self, queue_name: str):
self.queue = MessageQueue(
"/traces-" + queue_name,
max_messages=MAX_SIDECAR_QUEUE_SIZE,
max_message_size=MAX_SIDECAR_MESSAGE_SIZE,
)
def __init__(self, name: str, event_serializer: Callable[[T], bytes]):
self.queue = MessageQueue("/events-" + name,
max_messages=MAX_QUEUE_SIZE,
max_message_size=MAX_EVENT_SIZE)
self.serialize_event = event_serializer