def test_can_use_stdlog(self):
self.mock_sender = StdLibLoggingSender('testlogger')
expected = '{"fields": [{"representation": "", "value_type": "DOUBLE", "name": "rate", "value_double": [1.0]}, {"representation": "", "value_type": "STRING", "name": "name", "value_string": ["foo"]}], "logger": "my_logger_name", "env_version": "0.8", "type": "counter", "payload": "1", "severity": 6}'
with patch.object(self.mock_sender.logger, 'log') as mock_log:
self.client = HekaClient(self.mock_sender, 'my_logger_name')
self.client.incr('foo')
ok_(mock_log.called)
ok_(mock_log.call_count == 1)
eq_(mock_log.call_args[0][0], logging.INFO)
data = mock_log.call_args[0][1]
jdata = json.loads(data)
assert 'uuid' in jdata
assert 'timestamp' in jdata
assert 'hostname' in jdata
assert 'pid' in jdata
del jdata['uuid']
del jdata['timestamp']
del jdata['hostname']
del jdata['pid']
expected_jdata = json.loads(expected)
eq_(jdata, expected_jdata)
class TestUnicode(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.mock_sender.send_message.side_effect = \
UnicodeError("UnicodeError encoding user data")
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
self.old_stderr = sys.stderr
sys.stderr = StringIO.StringIO()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
sys.stderr = self.old_stderr
def test_unicode_failure(self):
msg = "mock will raise unicode error here"
self.client.send_message(msg)
sys.stderr.seek(0)
err = sys.stderr.read()
ok_('Error sending' in err)
class TestUnicode(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.mock_sender.send_message.side_effect = \
UnicodeError("UnicodeError encoding user data")
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
self.old_stderr = sys.stderr
sys.stderr = StringIO.StringIO()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
sys.stderr = self.old_stderr
def test_unicode_failure(self):
msg = "mock will raise unicode error here"
self.client.send_message(msg)
sys.stderr.seek(0)
err = sys.stderr.read()
ok_('Error sending' in err)
def setUp(self):
self.mock_sender = Mock()
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def setUp(self):
self.stream = DebugCaptureStream()
self.client = HekaClient(self.stream, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def setUp(self):
msg = Message()
msg.timestamp = int(time.time() * 1000000)
msg.type = 'sentry'
msg.logger = ''
msg.severity = 3
msg.payload = 'some_data'
msg.env_version = '0.8'
msg.pid = 55
msg.hostname = 'localhost'
client = HekaClient(None, None)
client._flatten_fields(
msg, {
'foo': 'bar',
'blah': 42,
'cef_meta': {
'syslog_name': 'some-syslog-thing',
'syslog_level': 5
}
})
msg.uuid = str(uuid.uuid5(uuid.NAMESPACE_OID, str(msg)))
self.msg = msg
def setUp(self):
self.mock_sender = Mock()
self.mock_sender.send_message.side_effect = \
UnicodeError("UnicodeError encoding user data")
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
self.old_stderr = sys.stderr
sys.stderr = StringIO.StringIO()
def test_can_use_stdlog(self):
self.mock_stream = StdLibLoggingStream('testlogger')
with patch.object(self.mock_stream.logger, 'log') as mock_log:
self.client = HekaClient(
self.mock_stream,
'my_logger_name',
encoder='heka.encoders.StdlibPayloadEncoder')
self.client.heka('stdlog', payload='this is some text')
ok_(mock_log.called)
ok_(mock_log.call_count == 1)
log_level, call_data = mock_log.call_args[0]
eq_(call_data, 'stdlog: this is some text')
eq_(log_level, logging.INFO)
def get_client(self, name):
"""Return the specified HekaClient, creating it if it doesn't
exist.
.. note::
Auto-created HekaClient instances will *not* yet be usable,
it is the downstream developer's responsibility to provide
them with a working stream.
:param name: String token identifying the client, also used as the
client's `logger` value.
"""
if name is None:
return None
client = self._clients.get(name)
if client is None:
with self.lock:
# check again to make sure nobody else got the lock first
client = self._clients.get(name)
if client is None:
client = HekaClient(stream=None, logger=name)
if (not self._clients and not self._default_clientname):
# first one, set as default
self._default_clientname = name
self._clients[name] = client
return client
def setUp(self):
self.stream = DebugCaptureStream()
self.client = HekaClient(self.stream, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
class TestStdLogging(object):
def test_can_use_stdlog(self):
self.mock_stream = StdLibLoggingStream('testlogger')
with patch.object(self.mock_stream.logger, 'log') as mock_log:
self.client = HekaClient(self.mock_stream,
'my_logger_name',
encoder='heka.encoders.StdlibPayloadEncoder')
self.client.heka('stdlog', payload='this is some text')
ok_(mock_log.called)
ok_(mock_log.call_count == 1)
log_level, call_data = mock_log.call_args[0]
eq_(call_data, 'this is some text')
eq_(log_level, logging.INFO)
def setUp(self):
self.mock_stream = DebugCaptureStream()
class NoEncoder(object):
def __init__(self, hmc):
pass
def encode(self, msg):
return msg
self.client = HekaClient(self.mock_stream,
self.logger,
encoder=NoEncoder)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def setUp(self):
msg = Message()
msg.timestamp = int(time.time() * 1000000)
msg.type = 'sentry'
msg.logger = ''
msg.severity = 3
msg.payload = 'some_data'
msg.env_version = '0.8'
msg.pid = 55
msg.hostname = 'localhost'
client = HekaClient(None, None)
client._flatten_fields(msg, {'foo': 'bar',
'blah': 42,
'cef_meta': {'syslog_name': 'some-syslog-thing',
'syslog_level': 5}})
msg.uuid = str(uuid.uuid5(uuid.NAMESPACE_OID, str(msg)))
self.msg = msg
def setUp(self):
self.mock_sender = Mock()
self.mock_sender.send_message.side_effect = \
UnicodeError("UnicodeError encoding user data")
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
self.old_stderr = sys.stderr
sys.stderr = StringIO.StringIO()
def setUp(self):
self.mock_stream = DebugCaptureStream()
class NoEncoder(object):
def __init__(self, hmc):
pass
def encode(self, msg):
return msg
self.client = HekaClient(self.mock_stream, self.logger,
encoder=NoEncoder)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
class TestHekaClient(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
def _extract_full_msg(self):
msg = self.mock_sender.send_message.call_args[0][0]
return msg
def compute_timestamp(self):
return int(time.time() * 1000000)
@raises(ValueError)
def test_heka_flatten_nulls(self):
"""
None values in the fields dictionary should throw an error
"""
payload = 'this is a test'
self.client.heka('some_msg_type',
payload=payload,
fields={'foo': None})
def test_heka_bare(self):
payload = 'this is a test'
before = self.compute_timestamp()
msgtype = 'testtype'
self.client.heka(msgtype, payload=payload)
after = self.compute_timestamp()
full_msg = self._extract_full_msg()
# check the payload
eq_(full_msg.payload, payload)
# check the various default values
ok_(before < full_msg.timestamp < after)
eq_(full_msg.type, msgtype)
eq_(full_msg.severity, self.client.severity)
eq_(full_msg.logger, self.logger)
eq_(full_msg.pid, os.getpid())
eq_(full_msg.hostname, socket.gethostname())
eq_(full_msg.env_version, self.client.env_version)
def test_heka_full(self):
heka_args = dict(payload='this is another test',
logger='alternate',
severity=2,
fields={
'foo': 'bar',
'boo': 'far'
})
msgtype = 'bawlp'
self.client.heka(msgtype, **heka_args)
actual_msg = self._extract_full_msg()
heka_args.update({
'type': msgtype,
'env_version': self.client.env_version,
'heka_pid': os.getpid(),
'heka_hostname': socket.gethostname(),
'timestamp': actual_msg.timestamp
})
# Everything but the UUID should be identical
expected_msg = dict_to_msg(heka_args)
actual_dict = json.loads(JSONMessageEncoder().encode(actual_msg))
expected_dict = json.loads(JSONMessageEncoder().encode(expected_msg))
del expected_dict['uuid']
del actual_dict['uuid']
eq_(actual_dict, expected_dict)
def test_oldstyle(self):
payload = 'debug message'
self.client.debug(payload)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, payload)
eq_(full_msg.severity, SEVERITY.DEBUG)
def test_oldstyle_args(self):
payload = '1, 2: %s\n3, 4: %s'
args = ('buckle my shoe', 'shut the door')
self.client.warn(payload, *args)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, payload % args)
def test_oldstyle_mapping_arg(self):
payload = '1, 2: %(onetwo)s\n3, 4: %(threefour)s'
args = {'onetwo': 'buckle my shoe', 'threefour': 'shut the door'}
self.client.warn(payload, args)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, payload % args)
def test_oldstyle_exc_info(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.error(payload, exc_info=True)
full_msg = self._extract_full_msg()
ok_(full_msg.payload.startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg.payload)
ok_('test_client.py' in full_msg.payload)
def test_oldstyle_exc_info_auto(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.exception(payload)
full_msg = self._extract_full_msg()
ok_(full_msg.payload.startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg.payload)
ok_('test_client.py' in full_msg.payload)
def test_oldstyle_exc_info_passed(self):
def name_error():
try:
a = b # NOQA
except NameError:
return sys.exc_info()
ei = name_error()
payload = 'traceback ahead -->'
self.client.critical(payload, exc_info=ei)
full_msg = self._extract_full_msg()
ok_(full_msg.payload.startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg.payload)
ok_('test_client.py' in full_msg.payload)
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, str(timer.result))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), name)
eq_(first_value(full_msg, 'rate'), 1)
def test_timer_decorator(self):
@self.client.timer(self.timer_name)
def timed():
time.sleep(0.01)
ok_(not self.mock_sender.send_message.called)
timed()
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10)
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), self.timer_name)
eq_(first_value(full_msg, 'rate'), 1)
eq_(first_value(full_msg, 'rate'), 1)
def test_timer_with_rate(self):
name = self.timer_name
@self.client.timer(name, rate=0.01)
def timed():
time.sleep(0.001)
# leverage chance by using a large sample
# instead of just 10 samples
for i in range(1000):
timed()
# this is a weak test, but not quite sure how else to
# test explicitly random behaviour
ok_(self.mock_sender.send_message.call_count < 100)
def test_incr(self):
name = 'incr'
self.client.incr(name)
full_msg = self._extract_full_msg()
eq_(full_msg.type, 'counter')
eq_(full_msg.logger, self.logger)
eq_(first_value(full_msg, 'name'), name)
# You have to have a rate set here
eq_(first_value(full_msg, 'rate'), 1)
eq_(full_msg.payload, '1')
self.client.incr(name, 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, '10')
def setUp(self):
self.mock_stream = Mock()
self.client = HekaClient(self.mock_stream, self.logger)
def client_from_dict_config(config, client=None):
"""
Configure a heka client, fully configured w/ stream and plugins.
:param config: Configuration dictionary.
:param client: HekaClient instance to configure. If None, one will be
created.
The configuration dict supports the following values:
logger
Heka client default logger value.
severity
Heka client default severity value.
disabled_timers
Sequence of string tokens identifying timers that are to be deactivated.
filters
Sequence of 2-tuples `(filter_provider, config)`. Each `filter_provider`
is a dotted name referring to a function which, when called and passed
the associated `config` dict as kwargs, will return a usable HekaClient
filter function.
plugins
Nested dictionary containing plugin configuration. Keys are the plugin
names (i.e. the name the method will be given when attached to the
client). Values are 2-tuples `(plugin_provider, config)`. Each
`plugin_provider` is a dotted name referring to a function which, when
called and passed the associated `config`, will return the usable plugin
method.
stream
Nested dictionary containing stream configuration.
All of the configuration values are optional, but failure to include a
stream may result in a non-functional Heka client. Any unrecognized keys
will be ignored.
Note that any top level config values starting with `stream_` will be added
to the `stream` config dictionary, overwriting any values that may already
be set.
The stream configuration supports the following values:
class (required)
Dotted name identifying the stream class to instantiate.
args
Sequence of non-keyword args to pass to stream constructor.
<kwargs>
All remaining key-value pairs in the stream config dict will be passed as
keyword arguments to the stream constructor.
"""
# Make a deep copy of the configuration so that subsequent uses of
# the config won't blow up
config = nest_prefixes(copy.deepcopy(config))
config_copy = json.dumps(copy.deepcopy(config))
stream_config = config.get('stream', {})
logger = config.get('logger', '')
severity = config.get('severity', 6)
disabled_timers = config.get('disabled_timers', [])
filter_specs = config.get('filters', [])
plugins_data = config.pop('plugins', {})
encoder = config.get('encoder', 'heka.encoders.JSONEncoder')
hmc = config.get('hmac', {})
resolver = DottedNameResolver()
# instantiate stream
stream_clsname = stream_config.pop('class')
stream_cls = resolver.resolve(stream_clsname)
stream_args = stream_config.pop('args', tuple())
stream = stream_cls(*stream_args, **stream_config)
# initialize filters
filters = [resolver.resolve(dotted_name)(**cfg)
for (dotted_name, cfg) in filter_specs]
# instantiate and/or configure client
sender = build_sender(stream, encoder, hmc)
if client is None:
client = HekaClient(sender,
logger,
severity,
disabled_timers,
filters)
else:
client.setup(sender, logger, severity, disabled_timers, filters)
# initialize plugins and attach to client
for section_name, plugin_spec in plugins_data.items():
# each plugin spec is a 2-tuple: (dotted_name, cfg)
plugin_config = plugin_spec[1]
plugin_override = plugin_config.pop('override', False)
plugin_fn = resolver.resolve(plugin_spec[0])(plugin_config)
client.add_method(plugin_fn, plugin_override)
# We bind the configuration into the client itself to ease
# debugging
client._config = config_copy
return client
class TestDisabledTimer(object):
logger = 'tests'
timer_name = 'test'
def _extract_full_msg(self):
h, m = decode_message(self.mock_sender.msgs[0])
return m
def setUp(self):
self.mock_sender = DebugCaptureSender()
self.client = HekaClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
self.client.sender.msgs.clear()
del self.timer_ob.__dict__['_local']
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, str(timer.result))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), self.timer_name)
eq_(first_value(full_msg, 'rate'), 1)
# Now disable it
self.client._disabled_timers.add(name)
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result is None)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.client.sender.msgs.clear()
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, str(timer.result))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), name)
eq_(first_value(full_msg, 'rate'), 1.0)
def test_timer_decorator(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.client.sender.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10, "Got: %d" % int(full_msg.payload))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), 'test')
eq_(first_value(full_msg, 'rate'), 1)
# Now disable it
self.client._disabled_timers.add(name)
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.mock_sender.msgs), 0)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo3():
time.sleep(0.01)
foo3()
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10)
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), 'test')
eq_(first_value(full_msg, 'rate'), 1)
def test_disable_all_timers(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.client.sender.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10)
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), 'test')
eq_(first_value(full_msg, 'rate'), 1)
# Now disable everything
self.client._disabled_timers.add('*')
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.mock_sender.msgs), 0)
class TestHekaClientFilters(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.stream = DebugCaptureStream()
self.client = HekaClient(self.stream, self.logger)
def tearDown(self):
del self.stream
del self.client
def test_severity_max(self):
from heka.filters import severity_max_provider
self.client.filters = [severity_max_provider(severity=SEVERITY.ERROR)]
payload = 'foo'
self.client.debug(payload)
self.client.info(payload)
self.client.warn(payload)
self.client.error(payload)
self.client.exception(payload)
self.client.critical(payload)
# only half of the messages should have gone out
eq_(len(self.stream.msgs), 3)
# make sure it's the right half
for json_msg in self.stream.msgs:
msg = self._extract_msg(json_msg)
ok_(msg.severity <= SEVERITY.ERROR)
def test_type_blacklist(self):
from heka.filters import type_blacklist_provider
type_blacklist = type_blacklist_provider(types=set(['foo']))
self.client.filters = [type_blacklist]
choices = ['foo', 'bar']
notfoos = 0
for i in range(10):
choice = random.choice(choices)
if choice != 'foo':
notfoos += 1
self.client.heka(choice, payload='msg')
eq_(len(self.stream.msgs), notfoos)
def test_type_whitelist(self):
from heka.filters import type_whitelist_provider
type_whitelist = type_whitelist_provider(types=set(['foo']))
self.client.filters = [type_whitelist]
choices = ['foo', 'bar']
foos = 0
for i in range(10):
choice = random.choice(choices)
if choice == 'foo':
foos += 1
self.client.heka(choice, payload='msg')
eq_(len(self.stream.msgs), foos)
def test_type_severity_max(self):
from heka.filters import type_severity_max_provider
config = {'types': {'foo': {'severity': 3},
'bar': {'severity': 5},
},
}
type_severity_max = type_severity_max_provider(**config)
self.client.filters = [type_severity_max]
for msgtype in ['foo', 'bar']:
for sev in range(8):
self.client.heka(msgtype, severity=sev, payload='msg')
eq_(len(self.stream.msgs), 10)
msgs = [self._extract_msg(msg) for msg in self.stream.msgs]
foos = [msg for msg in msgs if msg.type == 'foo']
eq_(len(foos), 4)
bars = [msg for msg in msgs if msg.type == 'bar']
eq_(len(bars), 6)
def _extract_msg(self, bytes):
h, m = decode_message(bytes)
return m
def setUp(self):
self.sender = DebugCaptureSender()
self.client = HekaClient(self.sender, self.logger)
def setUp(self):
self.mock_sender = Mock()
self.client = HekaClient(self.mock_sender, self.logger)
def setUp(self):
self.stream = DebugCaptureStream()
self.client = HekaClient(self.stream, self.logger)
def client_from_dict_config(config, client=None):
"""
Configure a heka client, fully configured w/ stream and plugins.
:param config: Configuration dictionary.
:param client: HekaClient instance to configure. If None, one will be
created.
The configuration dict supports the following values:
logger
Heka client default logger value.
severity
Heka client default severity value.
disabled_timers
Sequence of string tokens identifying timers that are to be deactivated.
filters
Sequence of 2-tuples `(filter_provider, config)`. Each `filter_provider`
is a dotted name referring to a function which, when called and passed
the associated `config` dict as kwargs, will return a usable HekaClient
filter function.
plugins
Nested dictionary containing plugin configuration. Keys are the plugin
names (i.e. the name the method will be given when attached to the
client). Values are 2-tuples `(plugin_provider, config)`. Each
`plugin_provider` is a dotted name referring to a function which, when
called and passed the associated `config`, will return the usable plugin
method.
stream
Nested dictionary containing stream configuration.
All of the configuration values are optional, but failure to include a
stream may result in a non-functional Heka client. Any unrecognized keys
will be ignored.
Note that any top level config values starting with `stream_` will be added
to the `stream` config dictionary, overwriting any values that may already
be set.
The stream configuration supports the following values:
class (required)
Dotted name identifying the stream class to instantiate.
args
Sequence of non-keyword args to pass to stream constructor.
<kwargs>
All remaining key-value pairs in the stream config dict will be passed as
keyword arguments to the stream constructor.
"""
# Make a deep copy of the configuration so that subsequent uses of
# the config won't blow up
config = nest_prefixes(copy.deepcopy(config))
config_copy = json.dumps(copy.deepcopy(config))
stream_config = config.get('stream', {})
logger = config.get('logger', '')
severity = config.get('severity', 6)
disabled_timers = config.get('disabled_timers', [])
filter_specs = config.get('filters', [])
plugins_data = config.pop('plugins', {})
encoder = config.get('encoder', 'heka.encoders.ProtobufEncoder')
hmc = config.get('hmac', {})
resolver = DottedNameResolver()
# instantiate stream
stream_clsname = stream_config.pop('class')
stream_cls = resolver.resolve(stream_clsname)
stream_args = stream_config.pop('args', tuple())
stream = stream_cls(*stream_args, **stream_config)
# initialize filters
filters = [resolver.resolve(dotted_name)(**cfg)
for (dotted_name, cfg) in filter_specs]
if client is None:
client = HekaClient(stream,
logger,
severity,
disabled_timers,
filters,
encoder=encoder,
hmc=hmc)
else:
client.setup(stream, encoder, hmc, logger, severity, disabled_timers, filters)
# initialize plugins and attach to client
for section_name, plugin_spec in plugins_data.items():
# each plugin spec is a 2-tuple: (dotted_name, cfg)
plugin_config = plugin_spec[1]
plugin_override = plugin_config.pop('override', False)
plugin_fn = resolver.resolve(plugin_spec[0])(plugin_config)
client.add_method(plugin_fn, plugin_override)
# We bind the configuration into the client itself to ease
# debugging
client._config = config_copy
return client
class TestDisabledTimer(object):
logger = 'tests'
timer_name = 'test'
def _extract_full_msg(self):
h, m = decode_message(self.stream.msgs[0])
return m
def setUp(self):
self.stream = DebugCaptureStream()
self.client = HekaClient(self.stream, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
self.stream.msgs.clear()
del self.timer_ob.__dict__['_local']
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, str(timer.result))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), self.timer_name)
eq_(first_value(full_msg, 'rate'), 1)
# Now disable it
self.client._disabled_timers.add(name)
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result is None)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.stream.msgs.clear()
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, str(timer.result))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), name)
eq_(first_value(full_msg, 'rate'), 1.0)
def test_timer_decorator(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.stream.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10,
"Got: %d" % int(full_msg.payload))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), 'test')
eq_(first_value(full_msg, 'rate'), 1)
# Now disable it
self.client._disabled_timers.add(name)
self.stream.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.stream.msgs), 0)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.stream.msgs.clear()
@self.client.timer(name)
def foo3():
time.sleep(0.01)
foo3()
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10)
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), 'test')
eq_(first_value(full_msg, 'rate'), 1)
def test_disable_all_timers(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.stream.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10)
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), 'test')
eq_(first_value(full_msg, 'rate'), 1)
# Now disable everything
self.client._disabled_timers.add('*')
self.stream.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.stream.msgs), 0)
class TestHekaClient(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_stream = DebugCaptureStream()
class NoEncoder(object):
def __init__(self, hmc):
pass
def encode(self, msg):
return msg
self.client = HekaClient(self.mock_stream, self.logger,
encoder=NoEncoder)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_stream
def _extract_full_msg(self):
msg = self.mock_stream.msgs.pop()
return msg
def compute_timestamp(self):
"""
These should be nanoseconds
"""
return int(time.time() * 1000000000)
@raises(ValueError)
def test_heka_flatten_nulls(self):
"""
None values in the fields dictionary should throw an error
"""
payload = 'this is a test'
self.client.heka('some_msg_type', payload=payload, fields={'foo': None})
def test_heka_bare(self):
payload = 'this is a test'
before = self.compute_timestamp()
msgtype = 'testtype'
self.client.heka(msgtype, payload=payload)
after = self.compute_timestamp()
full_msg = self._extract_full_msg()
# check the payload
eq_(full_msg.payload, payload)
# check the various default values
ok_(before < full_msg.timestamp < after)
eq_(full_msg.type, msgtype)
eq_(full_msg.severity, self.client.severity)
eq_(full_msg.logger, self.logger)
eq_(full_msg.pid, os.getpid())
eq_(full_msg.hostname, socket.gethostname())
eq_(full_msg.env_version, self.client.env_version)
def test_heka_full(self):
heka_args = dict(payload='this is another test',
logger='alternate',
severity=2,
fields={'foo': 'bar',
'boo': 'far'})
msgtype = 'bawlp'
self.client.heka(msgtype, **heka_args)
actual_msg = self._extract_full_msg()
heka_args.update({'type': msgtype,
'env_version': self.client.env_version,
'heka_pid': os.getpid(),
'heka_hostname': socket.gethostname(),
'timestamp': actual_msg.timestamp})
# Everything but the UUID should be identical
expected_msg = dict_to_msg(heka_args)
pbencoder = ProtobufEncoder()
h, actual_msg = decode_message(pbencoder.encode(actual_msg))
h, expected_msg = decode_message(pbencoder.encode(expected_msg))
expected_msg.uuid = ''
actual_msg.uuid = ''
eq_(actual_msg, expected_msg)
def test_heka_timestamp(self):
payload = 'this is a timestamp test'
timestamp = time.time()
msgtype = 'testtype'
self.client.heka(msgtype, payload=payload, timestamp=timestamp)
full_msg = self._extract_full_msg()
eq_(full_msg.timestamp, timestamp * 1000000000)
timestamp = datetime.datetime.now()
self.client.heka(msgtype, payload=payload, timestamp=timestamp)
full_msg = self._extract_full_msg()
eq_(full_msg.timestamp, time.mktime(timestamp.timetuple()) * 1000000000)
def test_oldstyle(self):
payload = 'debug message'
self.client.debug(payload)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, payload)
eq_(full_msg.severity, SEVERITY.DEBUG)
def test_oldstyle_args(self):
payload = '1, 2: %s\n3, 4: %s'
args = ('buckle my shoe', 'shut the door')
self.client.warn(payload, *args)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, payload % args)
def test_oldstyle_mapping_arg(self):
payload = '1, 2: %(onetwo)s\n3, 4: %(threefour)s'
args = {'onetwo': 'buckle my shoe',
'threefour': 'shut the door'}
self.client.warn(payload, args)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, payload % args)
def test_oldstyle_exc_info(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.error(payload, exc_info=True)
full_msg = self._extract_full_msg()
ok_(full_msg.payload.startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg.payload)
ok_('test_client.py' in full_msg.payload)
def test_oldstyle_exc_info_auto(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.exception(payload)
full_msg = self._extract_full_msg()
ok_(full_msg.payload.startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg.payload)
ok_('test_client.py' in full_msg.payload)
def test_oldstyle_exc_info_passed(self):
def name_error():
try:
a = b # NOQA
except NameError:
return sys.exc_info()
ei = name_error()
payload = 'traceback ahead -->'
self.client.critical(payload, exc_info=ei)
full_msg = self._extract_full_msg()
ok_(full_msg.payload.startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg.payload)
ok_('test_client.py' in full_msg.payload)
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, str(timer.result))
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), name)
eq_(first_value(full_msg, 'rate'), 1)
def test_timer_decorator(self):
@self.client.timer(self.timer_name)
def timed():
time.sleep(0.01)
ok_(not len(self.mock_stream.msgs))
timed()
full_msg = self._extract_full_msg()
ok_(int(full_msg.payload) >= 10)
eq_(full_msg.type, 'timer')
eq_(first_value(full_msg, 'name'), self.timer_name)
eq_(first_value(full_msg, 'rate'), 1)
eq_(first_value(full_msg, 'rate'), 1)
def test_timer_with_rate(self):
name = self.timer_name
@self.client.timer(name, rate=0.01)
def timed():
time.sleep(0.001)
# leverage chance by using a large sample
# instead of just 10 samples
for i in range(1000):
timed()
# this is a weak test, but not quite sure how else to
# test explicitly random behaviour
ok_(len(self.mock_stream.msgs) < 200)
def test_incr(self):
name = 'incr'
self.client.incr(name)
full_msg = self._extract_full_msg()
eq_(full_msg.type, 'counter')
eq_(full_msg.logger, self.logger)
eq_(first_value(full_msg, 'name'), name)
# You have to have a rate set here
eq_(first_value(full_msg, 'rate'), 1)
eq_(full_msg.payload, '1')
self.client.incr(name, 10)
full_msg = self._extract_full_msg()
eq_(full_msg.payload, '10')