def publish_traces():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file", type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument("--queue-name", default="main",
help="name of trace queue / publisher config (default: main)")
arg_parser.add_argument("--debug", default=False, action="store_true",
help="enable debug logging")
arg_parser.add_argument("--app-name", default="main", metavar="NAME",
help="name of app to load from config_file (default: main)")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser()
config_parser.readfp(args.config_file)
publisher_raw_cfg = dict(config_parser.items("trace-publisher:" + args.queue_name))
publisher_cfg = config.parse_config(publisher_raw_cfg, {
"zipkin_api_url": config.Endpoint,
"post_timeout": config.Optional(config.Integer, POST_TIMEOUT_DEFAULT),
"max_batch_size": config.Optional(config.Integer, MAX_BATCH_SIZE_DEFAULT),
"retry_limit": config.Optional(config.Integer, RETRY_LIMIT_DEFAULT),
})
trace_queue = MessageQueue(
"/traces-" + args.queue_name,
max_messages=MAX_QUEUE_SIZE,
max_message_size=MAX_SPAN_SIZE,
)
# pylint: disable=maybe-no-member
inner_batch = TraceBatch(max_size=publisher_cfg.max_batch_size)
batcher = TimeLimitedBatch(inner_batch, MAX_BATCH_AGE)
metrics_client = metrics_client_from_config(publisher_raw_cfg)
publisher = ZipkinPublisher(
publisher_cfg.zipkin_api_url.address,
metrics_client,
post_timeout=publisher_cfg.post_timeout,
)
while True:
try:
message = trace_queue.get(timeout=.2)
except TimedOutError:
message = None
try:
batcher.add(message)
except BatchFull:
serialized = batcher.serialize()
publisher.publish(serialized)
batcher.reset()
batcher.add(message)
def main():
"""Run a consumer.
Two environment variables are expected:
* CONFIG_URI: A PasteDeploy URI pointing at the configuration for the
application.
* QUEUE: The name of the queue to consume (currently one of "events" or
"errors").
"""
config_uri = os.environ["CONFIG_URI"]
config = paste.deploy.loadwsgi.appconfig(config_uri)
logging.config.fileConfig(config["__file__"])
queue_name = os.environ["QUEUE"]
queue = MessageQueue("/" + queue_name,
max_messages=MAXIMUM_QUEUE_LENGTH,
max_message_size=MAXIMUM_EVENT_SIZE)
metrics_client = baseplate.make_metrics_client(config)
topic_name = config["topic." + queue_name]
producer_options = {
"codec": CODEC_GZIP,
"batch_send_every_n": 20,
"batch_send_every_t": 0.01, # 10 milliseconds
}
while True:
try:
kafka_client = KafkaClient(config["kafka_brokers"])
kafka_producer = SimpleProducer(kafka_client, **producer_options)
except KafkaError as exc:
_LOG.warning("could not connect: %s", exc)
metrics_client.counter("injector.connection_error").increment()
time.sleep(_RETRY_DELAY)
continue
while True:
message = queue.get()
for retry in itertools.count():
try:
kafka_producer.send_messages(topic_name, message)
except KafkaError as exc:
_LOG.warning("failed to send message: %s", exc)
metrics_client.counter("injector.error").increment()
time.sleep(_RETRY_DELAY)
else:
metrics_client.counter("collected.injector").increment()
break
kafka_producer.stop()
def main():
"""Run a consumer.
Two environment variables are expected:
* CONFIG_URI: A PasteDeploy URI pointing at the configuration for the
application.
* QUEUE: The name of the queue to consume (currently one of "events" or
"errors").
"""
config_uri = os.environ["CONFIG_URI"]
config = paste.deploy.loadwsgi.appconfig(config_uri)
logging.config.fileConfig(config["__file__"])
queue_name = os.environ["QUEUE"]
queue = MessageQueue("/" + queue_name,
max_messages=MAXIMUM_QUEUE_LENGTH, max_message_size=MAXIMUM_EVENT_SIZE)
metrics_client = baseplate.make_metrics_client(config)
topic_name = config["topic." + queue_name]
producer_options = {
"codec": CODEC_GZIP,
"batch_send_every_n": 20,
"batch_send_every_t": 0.01, # 10 milliseconds
}
while True:
try:
kafka_client = KafkaClient(config["kafka_brokers"])
kafka_producer = SimpleProducer(kafka_client, **producer_options)
except KafkaError as exc:
_LOG.warning("could not connect: %s", exc)
metrics_client.counter("injector.connection_error").increment()
time.sleep(_RETRY_DELAY)
continue
while True:
message = queue.get()
for retry in itertools.count():
try:
kafka_producer.send_messages(topic_name, message)
except KafkaError as exc:
_LOG.warning("failed to send message: %s", exc)
metrics_client.counter("injector.error").increment()
time.sleep(_RETRY_DELAY)
else:
metrics_client.counter("collected.injector").increment()
break
kafka_producer.stop()
# consumer.py
from baseplate.message_queue import MessageQueue
mq = MessageQueue("/baseplate-testing", max_messages=1, max_message_size=1)
# Unless a `timeout` kwarg is passed, this will block until
# we can pop a message from the queue.
message = mq.get()
print("Get Message: %s" % message)
class CollectorFunctionalTests(unittest.TestCase):
def setUp(self):
# we create the queues before the actual code can so that we can
# override the max sizes to use these numbers which are safe to use
# without extra privileges on linux
self.events_queue = MessageQueue(name="/events",
max_messages=10,
max_message_size=8192)
self.errors_queue = MessageQueue(name="/errors",
max_messages=10,
max_message_size=8192)
class MockDatetime(datetime.datetime):
@classmethod
def utcnow(cls):
return datetime.datetime(2015, 11, 17, 12, 34, 56)
datetime.datetime = MockDatetime
app = collector.make_app(global_config={},
**{
"key.TestKey1": "dGVzdA==",
"msgq.events": "0xcafe",
"msgq.errors": "0xdecaf",
"allowed_origins": "example.com",
"metrics.namespace": "eventcollector",
"metrics.endpoint": "",
})
self.test_app = webtest.TestApp(app)
def tearDown(self):
self.events_queue.queue.unlink()
self.events_queue.queue.close()
self.errors_queue.queue.unlink()
self.errors_queue.queue.close()
def test_batch(self):
self.test_app.post(
"/v1",
'[{"event1": "value"}, {"event2": "value"}]',
headers={
"Content-Type":
"application/json",
"User-Agent":
"TestApp/1.0",
"Date":
"Wed, 25 Nov 2015 06:25:24 GMT",
"X-Signature":
"key=TestKey1, mac=d7aab40b9db8ae0e0b40d98e9c50b2cfc80ca06127b42fbbbdf146752b47a5ed",
},
extra_environ={
"REMOTE_ADDR": "1.2.3.4",
},
)
event1 = self.events_queue.get(timeout=0)
self.assertEqual(
event1,
'{"ip": "1.2.3.4", "event": {"event1": "value"}, "time": "2015-11-17T12:34:56"}'
)
event2 = self.events_queue.get(timeout=0)
self.assertEqual(
event2,
'{"ip": "1.2.3.4", "event": {"event2": "value"}, "time": "2015-11-17T12:34:56"}'
)
with self.assertRaises(TimedOutError):
self.errors_queue.get(timeout=0)
def test_cors(self):
response = self.test_app.options("/v1",
headers={
"Origin":
"http://example.com",
"Access-Control-Request-Method":
"POST",
"Access-Control-Request-Headers":
"X-Signature",
})
self.assertEqual(response.status_code, 204)
def publish_events():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("config_file",
type=argparse.FileType("r"),
help="path to a configuration file")
arg_parser.add_argument(
"--queue-name",
default="main",
help="name of event queue / publisher config (default: main)",
)
arg_parser.add_argument("--debug",
default=False,
action="store_true",
help="enable debug logging")
args = arg_parser.parse_args()
if args.debug:
level = logging.DEBUG
else:
level = logging.WARNING
logging.basicConfig(level=level)
config_parser = configparser.RawConfigParser()
config_parser.readfp(args.config_file) # pylint: disable=deprecated-method
raw_config = dict(config_parser.items("event-publisher:" +
args.queue_name))
cfg = config.parse_config(
raw_config,
{
"collector": {
"hostname": config.String,
"version": config.Optional(config.Integer, default=1),
},
"key": {
"name": config.String,
"secret": config.Base64
},
},
)
metrics_client = metrics_client_from_config(raw_config)
event_queue = MessageQueue("/events-" + args.queue_name,
max_messages=MAX_QUEUE_SIZE,
max_message_size=MAX_EVENT_SIZE)
# pylint: disable=maybe-no-member
serializer = SERIALIZER_BY_VERSION[cfg.collector.version]()
batcher = TimeLimitedBatch(serializer, MAX_BATCH_AGE)
publisher = BatchPublisher(metrics_client, cfg)
while True:
try:
message = event_queue.get(timeout=0.2)
except TimedOutError:
message = None
try:
batcher.add(message)
except BatchFull:
serialized = batcher.serialize()
publisher.publish(serialized)
batcher.reset()
batcher.add(message)
class CollectorFunctionalTests(unittest.TestCase):
def setUp(self):
# we create the queues before the actual code can so that we can
# override the max sizes to use these numbers which are safe to use
# without extra privileges on linux
self.events_queue = MessageQueue(name="/events",
max_messages=10, max_message_size=8192)
self.errors_queue = MessageQueue(name="/errors",
max_messages=10, max_message_size=8192)
class MockDatetime(datetime.datetime):
@classmethod
def utcnow(cls):
return datetime.datetime(2015, 11, 17, 12, 34, 56)
datetime.datetime = MockDatetime
app = collector.make_app(global_config={}, **{
"key.TestKey1": "dGVzdA==",
"msgq.events": "0xcafe",
"msgq.errors": "0xdecaf",
"allowed_origins": "example.com",
"metrics.namespace": "eventcollector",
"metrics.endpoint": "",
})
self.test_app = webtest.TestApp(app)
def tearDown(self):
self.events_queue.queue.unlink()
self.events_queue.queue.close()
self.errors_queue.queue.unlink()
self.errors_queue.queue.close()
def test_batch(self):
self.test_app.post("/v1",
'[{"event1": "value"}, {"event2": "value"}]',
headers={
"Content-Type": "application/json",
"User-Agent": "TestApp/1.0",
"Date": "Wed, 25 Nov 2015 06:25:24 GMT",
"X-Signature": "key=TestKey1, mac=d7aab40b9db8ae0e0b40d98e9c50b2cfc80ca06127b42fbbbdf146752b47a5ed",
},
extra_environ={
"REMOTE_ADDR": "1.2.3.4",
},
)
event1 = self.events_queue.get(timeout=0)
self.assertEqual(event1, '{"ip": "1.2.3.4", "event": {"event1": "value"}, "time": "2015-11-17T12:34:56"}')
event2 = self.events_queue.get(timeout=0)
self.assertEqual(event2, '{"ip": "1.2.3.4", "event": {"event2": "value"}, "time": "2015-11-17T12:34:56"}')
with self.assertRaises(TimedOutError):
self.errors_queue.get(timeout=0)
def test_cors(self):
response = self.test_app.options("/v1", headers={
"Origin": "http://example.com",
"Access-Control-Request-Method": "POST",
"Access-Control-Request-Headers": "X-Signature",
})
self.assertEqual(response.status_code, 204)
