class TestUnicode(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.mock_sender.send_message.side_effect = \
UnicodeError("UnicodeError encoding user data")
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
self.old_stderr = sys.stderr
sys.stderr = StringIO.StringIO()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
sys.stderr = self.old_stderr
def test_unicode_failure(self):
msg = "mock will raise unicode error here"
self.client.send_message(msg)
sys.stderr.seek(0)
err = sys.stderr.read()
ok_('Error sending' in err)
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def setUp(self):
self.mock_sender = Mock()
self.mock_sender.send_message.side_effect = \
UnicodeError("UnicodeError encoding user data")
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
self.old_stderr = sys.stderr
sys.stderr = StringIO.StringIO()
class TestMetlog(object):
logger = 'tests'
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.client.timer._local = threading.local()
plugin = config_plugin({'net':True})
self.client.add_method('procinfo', plugin)
def test_add_procinfo(self):
HOST = 'localhost' # Symbolic name meaning the local host
PORT = 50017 # Arbitrary non-privileged port
def echo_serv():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
data = conn.recv(1024)
conn.send(data)
conn.close()
s.close()
t = threading.Thread(target=echo_serv)
t.start()
time.sleep(1)
def client_code():
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((HOST, PORT))
client.send('Hello, world')
data = client.recv(1024)
client.close()
time.sleep(1)
self.client.procinfo(net=True)
eq_(1, len(self.client.sender.method_calls))
fields = self.client.sender.method_calls[0][1][0]['fields']
assert fields == {u'net': [{u'status': u'LISTEN', u'type': u'TCP', u'local': u'127.0.0.1:50017', u'remote': u'*:*'}]}
# Start the client up just so that the server will die gracefully
tc = threading.Thread(target=client_code)
tc.start()
def get_client(self, name):
"""
Return the specified MetlogClient, creating it if it doesn't exist.
*NOTE*: Auto-created MetlogClient instances will *not* yet be usable,
it is the downstream developer's responsibility to provide them with a
working sender.
:param name: String token identifying the client, also used as the
client's `logger` value.
"""
client = self._clients.get(name)
if client is None:
with self.lock:
# check again to make sure nobody else got the lock first
client = self._clients.get(name)
if client is None:
# TODO: there is no sender set here - grab one
# based on the globalconfig
client = MetlogClient(sender=None, logger=name)
if (not self._clients
and not self.global_config.get('default')):
# first one, set as default
self.global_config['default'] = name
self._clients[name] = client
return client
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.client.timer._local = threading.local()
plugin = config_plugin({'net':False})
self.client.add_method('procinfo', plugin)
class TestConfiguration(object):
"""
Configuration for plugin based loggers should *override* what the developer
uses. IOTW - developers are overridden by ops.
"""
logger = 'tests'
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.client.timer._local = threading.local()
plugin = config_plugin({'net':False})
self.client.add_method('procinfo', plugin)
def test_no_netlogging(self):
self.client.procinfo(net=True)
eq_(0, len(self.client.sender.method_calls))
class TestMetlogClient(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = DebugCaptureSender()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def _extract_full_msg(self):
return self.mock_sender.msgs[0]
def test_panel(self):
name = 'incr'
self.client.incr('foo')
self.client.incr('foo')
self.client.incr('bar', rate=5)
self.client.incr('batz', rate=2)
self.client.incr('batz', rate=3)
results = munge(self.client.sender.msgs)
expected = [
{'count': 2,
'name': u'foo',
'total': 2.0,
'type': u'counter',
'values': [(1.0, 1.0), (1.0, 1.0)]},
{'count': 1,
'name': u'bar',
'total': 5.0,
'type': u'counter',
'values': [(1.0, 5)]},
{'count': 2,
'name': u'batz',
'total': 5.0,
'type': u'counter',
'values': [(1.0, 2), (1.0, 3)]}
]
eq_(results, expected)
def __init__(self, ini_path=None, ini_dir=None, load_sections=None):
"""
:param ini_dir: Directory path in which to start looking for the ini
file. Will climb the file tree from here looking for 'tests.ini' file,
unless 'WEAVE_TESTFILE' env var is set, in which case it will climb the
file tree from here looking for 'tests_${WEAVE_TESTFILE}.ini'.
:param ini_path: Full path to configuration file. Takes precedence
over ini_dir, if both are provided. Raises IOError if file doesn't
exist.
One or the other of `ini_dir` or `ini_path` arguments MUST be provided.
:param load_sections: A sequence of strings that name the configuration
sections that should be dynamically loaded. Any entry in this sequence
could alternately be a 2-tuple containing the name of the section and
the corresponding class parameter value to use.
"""
self.start_dir = ini_dir
if ini_path:
if not os.path.isfile(ini_path):
raise IOError("invalid config file: %s" % ini_path)
ini_dir = os.path.dirname(ini_path)
elif ini_dir:
if 'WEAVE_TESTFILE' in os.environ:
test_filename = 'tests_%s.ini' % os.environ['WEAVE_TESTFILE']
else:
test_filename = 'tests.ini'
while True:
ini_path = os.path.join(ini_dir, test_filename)
if os.path.exists(ini_path):
break
if ini_path == ("/%s" % test_filename) \
or ini_path == test_filename:
raise IOError("cannot locate %s" % test_filename)
ini_dir = os.path.split(ini_dir)[0]
else:
raise ValueError('No ini_path or ini_dir specified.')
self.ini_dir = ini_dir
self.ini_path = ini_path
ini_cfg = RawConfigParser()
ini_cfg.read(ini_path)
# loading loggers
self.config = self.convert_config(ini_cfg, ini_path)
# Ensure that metlog is available, either from the config
# or by setting up a default client.
try:
loader = load_and_configure(self.config, "metlog_loader")
client = loader.default_client
except KeyError:
sender = DebugCaptureSender()
client = MetlogClient(sender, "syncserver")
CLIENT_HOLDER.set_client(client.logger, client)
if not hasattr(client, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
client.add_method(log_cef_fn)
if load_sections is not None:
for section in load_sections:
if isinstance(section, tuple):
self.add_class(section[0], cls_param=section[1])
else:
self.add_class(section)
def __init__(self, urls, controllers, config=None, auth_class=None):
self.mapper = Mapper()
if config is None:
self.config = Config()
elif isinstance(config, Config):
self.config = config
else:
# try to convert to config object
self.config = Config(config)
# global config
self.retry_after = self.config.get('global.retry_after', 1800)
# heartbeat page
self.heartbeat_page = self.config.get('global.heartbeat_page',
'__heartbeat__')
# debug page, if any
self.debug_page = self.config.get('global.debug_page')
# check if we want to clean when the app ends
self.sigclean = self.config.get('global.clean_shutdown', True)
# load the specified plugin modules
self.modules = dict()
app_modules = self.config.get('app.modules', [])
if isinstance(app_modules, basestring):
app_modules = [app_modules]
for module in app_modules:
self.modules[module] = load_and_configure(self.config, module)
if self.modules.get('metlog_loader') is not None:
# stash the metlog client in a more convenient spot
self.logger = self.modules.get('metlog_loader').default_client
else:
# there was no metlog config, default to using StdLibLoggingSender
sender = StdLibLoggingSender('syncserver', json_types=[])
metlog = MetlogClient(sender, 'syncserver')
CLIENT_HOLDER.set_client(metlog.logger, metlog)
self.logger = metlog
if not hasattr(self.logger, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
self.logger.add_method(log_cef_fn)
# XXX: this should be converted to auto-load in self.modules
# loading the authentication tool
self.auth = None if auth_class is None else auth_class(self.config)
# loading and connecting controllers
self.controllers = dict([(name, klass(self))
for name, klass in controllers.items()])
for url in urls:
if len(url) == 4:
verbs, match, controller, action = url
extras = {}
elif len(url) == 5:
verbs, match, controller, action, extras = url
else:
msg = "Each URL description needs 4 or 5 elements. Got %s" \
% str(url)
raise ValueError(msg)
if isinstance(verbs, str):
verbs = [verbs]
# wrap action methods w/ metlog decorators
controller_instance = self.controllers.get(controller)
if controller_instance is not None:
wrapped_name = '_%s_wrapped' % action
method = getattr(controller_instance, action, None)
if ((method is not None)
and (not hasattr(controller_instance, wrapped_name))):
# add wrapped method
wrapped = svc_timeit(method)
wrapped = incr_count(wrapped)
wrapped = send_services_data(wrapped)
setattr(controller_instance, wrapped_name, wrapped)
self.mapper.connect(None,
match,
controller=controller,
action=action,
conditions=dict(method=verbs),
**extras)
# loads host-specific configuration
self._host_configs = {}
# heartbeat & debug pages
self.standard_controller = StandardController(self)
# rehooked overridable points so they can be overridden in the base app
self.standard_controller._debug_server = self._debug_server
self.standard_controller._check_server = self._check_server
# hooking callbacks when the app shuts down
self.killing = self.shutting = False
self.graceful_shutdown_interval = self.config.get(
'global.graceful_shutdown_interval', 1.)
self.hard_shutdown_interval = self.config.get(
'global.hard_shutdown_interval', 1.)
if self.sigclean:
signal.signal(signal.SIGTERM, self._sigterm)
signal.signal(signal.SIGINT, self._sigterm)
def __init__(self, ini_path=None, ini_dir=None, load_sections=None):
"""
:param ini_dir: Directory path in which to start looking for the ini
file. Will climb the file tree from here looking for 'tests.ini' file,
unless 'WEAVE_TESTFILE' env var is set, in which case it will climb the
file tree from here looking for 'tests_${WEAVE_TESTFILE}.ini'.
:param ini_path: Full path to configuration file. Takes precedence
over ini_dir, if both are provided. Raises IOError if file doesn't
exist.
One or the other of `ini_dir` or `ini_path` arguments MUST be provided.
:param load_sections: A sequence of strings that name the configuration
sections that should be dynamically loaded. Any entry in this sequence
could alternately be a 2-tuple containing the name of the section and
the corresponding class parameter value to use.
"""
self.start_dir = ini_dir
if ini_path:
if not os.path.isfile(ini_path):
raise IOError("invalid config file: %s" % ini_path)
ini_dir = os.path.dirname(ini_path)
elif ini_dir:
if 'WEAVE_TESTFILE' in os.environ:
test_filename = 'tests_%s.ini' % os.environ['WEAVE_TESTFILE']
else:
test_filename = 'tests.ini'
while True:
ini_path = os.path.join(ini_dir, test_filename)
if os.path.exists(ini_path):
break
if ini_path == ("/%s" % test_filename) \
or ini_path == test_filename:
raise IOError("cannot locate %s" % test_filename)
ini_dir = os.path.split(ini_dir)[0]
else:
raise ValueError('No ini_path or ini_dir specified.')
self.ini_dir = ini_dir
self.ini_path = ini_path
ini_cfg = RawConfigParser()
ini_cfg.read(ini_path)
# loading loggers
self.config = self.convert_config(ini_cfg, ini_path)
# Ensure that metlog is available, either from the config
# or by setting up a default client.
try:
loader = load_and_configure(self.config, "metlog_loader")
client = loader.default_client
except KeyError:
sender = DebugCaptureSender()
client = MetlogClient(sender, "syncserver")
CLIENT_HOLDER.set_client(client.logger, client)
if not hasattr(client, "cef"):
log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
client.add_method(log_cef_fn)
if load_sections is not None:
for section in load_sections:
if isinstance(section, tuple):
self.add_class(section[0], cls_param=section[1])
else:
self.add_class(section)
class TestMetlogClient(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
def _extract_full_msg(self):
return self.mock_sender.send_message.call_args[0][0]
def test_metlog_bare(self):
payload = 'this is a test'
before = datetime.utcnow().isoformat()
msgtype = 'testtype'
self.client.metlog(msgtype, payload=payload)
after = datetime.utcnow().isoformat()
full_msg = self._extract_full_msg()
# check the payload
eq_(full_msg['payload'], payload)
# check the various default values
ok_(before < full_msg['timestamp'] < after)
eq_(full_msg['type'], msgtype)
eq_(full_msg['severity'], self.client.severity)
eq_(full_msg['logger'], self.logger)
eq_(full_msg['metlog_pid'], os.getpid())
eq_(full_msg['metlog_hostname'], socket.gethostname())
eq_(full_msg['env_version'], self.client.env_version)
def test_metlog_full(self):
metlog_args = dict(payload='this is another test',
timestamp=datetime.utcnow(),
logger='alternate',
severity=2,
fields={'foo': 'bar',
'boo': 'far'})
msgtype = 'bawlp'
self.client.metlog(msgtype, **metlog_args)
actual_msg = self._extract_full_msg()
metlog_args.update({'type': msgtype,
'env_version': self.client.env_version,
'metlog_pid': os.getpid(),
'metlog_hostname': socket.gethostname(),
'timestamp': metlog_args['timestamp'].isoformat()})
eq_(actual_msg, metlog_args)
def test_oldstyle(self):
payload = 'debug message'
self.client.debug(payload)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], payload)
eq_(full_msg['severity'], SEVERITY.DEBUG)
def test_oldstyle_args(self):
payload = '1, 2: %s\n3, 4: %s'
args = ('buckle my shoe', 'shut the door')
self.client.warn(payload, *args)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], payload % args)
def test_oldstyle_mapping_arg(self):
payload = '1, 2: %(onetwo)s\n3, 4: %(threefour)s'
args = {'onetwo': 'buckle my shoe',
'threefour': 'shut the door'}
self.client.warn(payload, args)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], payload % args)
def test_oldstyle_exc_info(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.error(payload, exc_info=True)
full_msg = self._extract_full_msg()
ok_(full_msg['payload'].startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg['payload'])
ok_('test_client.py' in full_msg['payload'])
def test_oldstyle_exc_info_auto(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.exception(payload)
full_msg = self._extract_full_msg()
ok_(full_msg['payload'].startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg['payload'])
ok_('test_client.py' in full_msg['payload'])
def test_oldstyle_exc_info_passed(self):
def name_error():
try:
a = b # NOQA
except NameError:
return sys.exc_info()
ei = name_error()
payload = 'traceback ahead -->'
self.client.critical(payload, exc_info=ei)
full_msg = self._extract_full_msg()
ok_(full_msg['payload'].startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg['payload'])
ok_('test_client.py' in full_msg['payload'])
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_decorator(self):
@self.client.timer(self.timer_name)
def timed():
time.sleep(0.01)
ok_(not self.mock_sender.send_message.called)
timed()
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], self.timer_name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_with_rate(self):
name = self.timer_name
@self.client.timer(name, rate=0.01)
def timed():
time.sleep(0.001)
for i in range(10):
timed()
# this is a weak test, but not quite sure how else to
# test explicitly random behaviour
ok_(self.mock_sender.send_message.call_count < 10)
def test_incr(self):
name = 'incr'
self.client.incr(name)
full_msg = self._extract_full_msg()
eq_(full_msg['type'], 'counter')
eq_(full_msg['logger'], self.logger)
eq_(full_msg['fields']['name'], name)
# You have to have a rate set here
eq_(full_msg['fields']['rate'], 1)
eq_(full_msg['payload'], '1')
self.client.incr(name, 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], '10')
def setUp(self):
self.sender = DebugCaptureSender()
self.client = MetlogClient(self.sender, self.logger)
def setUp(self):
self.sender = DebugCaptureSender()
self.client = MetlogClient(self.sender, self.logger)
from metlog.client import MetlogClient
from metlog.senders import ZmqPubSender
from metlog.backchannel import ZmqBackchannel
import time
import os
SUB_BIND, PUB_BIND = "ipc:///tmp/feeds/1", "ipc:///tmp/feeds/2"
bc = ZmqBackchannel(SUB_BIND, PUB_BIND)
sender = ZmqPubSender("ipc:///tmp/feeds/0")
client = MetlogClient(sender, back_channel=bc)
while True:
time.sleep(1)
client.metlog("msg_type", payload="pid [%d]" % os.getpid())
print "send messages"
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
class TestMetlogClient(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
def _extract_full_msg(self):
return self.mock_sender.send_message.call_args[0][0]
def test_metlog_bare(self):
payload = 'this is a test'
before = datetime.utcnow().isoformat()
msgtype = 'testtype'
self.client.metlog(msgtype, payload=payload)
after = datetime.utcnow().isoformat()
full_msg = self._extract_full_msg()
# check the payload
eq_(full_msg['payload'], payload)
# check the various default values
ok_(before < full_msg['timestamp'] < after)
eq_(full_msg['type'], msgtype)
eq_(full_msg['severity'], self.client.severity)
eq_(full_msg['logger'], self.logger)
eq_(full_msg['metlog_pid'], os.getpid())
eq_(full_msg['metlog_hostname'], socket.gethostname())
eq_(full_msg['env_version'], self.client.env_version)
def test_metlog_full(self):
metlog_args = dict(payload='this is another test',
logger='alternate',
severity=2,
fields={'foo': 'bar',
'boo': 'far'})
msgtype = 'bawlp'
self.client.metlog(msgtype, **metlog_args)
actual_msg = self._extract_full_msg()
metlog_args.update({'type': msgtype,
'env_version': self.client.env_version,
'metlog_pid': os.getpid(),
'metlog_hostname': socket.gethostname(),
'timestamp': actual_msg['timestamp']})
eq_(actual_msg, metlog_args)
def test_oldstyle(self):
payload = 'debug message'
self.client.debug(payload)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], payload)
eq_(full_msg['severity'], SEVERITY.DEBUG)
def test_oldstyle_args(self):
payload = '1, 2: %s\n3, 4: %s'
args = ('buckle my shoe', 'shut the door')
self.client.warn(payload, *args)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], payload % args)
def test_oldstyle_mapping_arg(self):
payload = '1, 2: %(onetwo)s\n3, 4: %(threefour)s'
args = {'onetwo': 'buckle my shoe',
'threefour': 'shut the door'}
self.client.warn(payload, args)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], payload % args)
def test_oldstyle_exc_info(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.error(payload, exc_info=True)
full_msg = self._extract_full_msg()
ok_(full_msg['payload'].startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg['payload'])
ok_('test_client.py' in full_msg['payload'])
def test_oldstyle_exc_info_auto(self):
payload = 'traceback ahead -->'
try:
a = b # NOQA
except NameError:
self.client.exception(payload)
full_msg = self._extract_full_msg()
ok_(full_msg['payload'].startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg['payload'])
ok_('test_client.py' in full_msg['payload'])
def test_oldstyle_exc_info_passed(self):
def name_error():
try:
a = b # NOQA
except NameError:
return sys.exc_info()
ei = name_error()
payload = 'traceback ahead -->'
self.client.critical(payload, exc_info=ei)
full_msg = self._extract_full_msg()
ok_(full_msg['payload'].startswith(payload))
ok_("NameError: global name 'b' is not defined" in full_msg['payload'])
ok_('test_client.py' in full_msg['payload'])
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_decorator(self):
@self.client.timer(self.timer_name)
def timed():
time.sleep(0.01)
ok_(not self.mock_sender.send_message.called)
timed()
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], self.timer_name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_with_rate(self):
name = self.timer_name
@self.client.timer(name, rate=0.01)
def timed():
time.sleep(0.001)
# leverage chance by using a large sample
# instead of just 10 samples
for i in range(1000):
timed()
# this is a weak test, but not quite sure how else to
# test explicitly random behaviour
ok_(self.mock_sender.send_message.call_count < 100)
def test_incr(self):
name = 'incr'
self.client.incr(name)
full_msg = self._extract_full_msg()
eq_(full_msg['type'], 'counter')
eq_(full_msg['logger'], self.logger)
eq_(full_msg['fields']['name'], name)
# You have to have a rate set here
eq_(full_msg['fields']['rate'], 1)
eq_(full_msg['payload'], '1')
self.client.incr(name, 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], '10')
def client_from_dict_config(config, client=None, clear_global=False):
"""
Configure a metlog client, fully configured w/ sender and plugins.
:param config: Configuration dictionary.
:param client: MetlogClient instance to configure. If None, one will be
created.
:param clear_global: If True, delete any existing global config on the
CLIENT_HOLDER before applying new config.
The configuration dict supports the following values:
logger
Metlog client default logger value.
severity
Metlog client default severity value.
disabled_timers
Sequence of string tokens identifying timers that are to be deactivated.
filters
Sequence of 2-tuples `(filter_provider, config)`. Each `filter_provider`
is a dotted name referring to a function which, when called and passed
the associated `config` dict as kwargs, will return a usable MetlogClient
filter function.
plugins
Nested dictionary containing plugin configuration. Keys are the plugin
names (i.e. the name the method will be given when attached to the
client). Values are 2-tuples `(plugin_provider, config)`. Each
`plugin_provider` is a dotted name referring to a function which, when
called and passed the associated `config`, will return the usable plugin
method.
sender
Nested dictionary containing sender configuration.
global
Dictionary to be applied to CLIENT_HOLDER's `global_config` storage.
New config will overwrite any conflicting values, but will not delete
other config entries. To delete, calling code should call the function
with `clear_global` set to True.
All of the configuration values are optional, but failure to include a
sender may result in a non-functional Metlog client. Any unrecognized keys
will be ignored.
Note that any top level config values starting with `sender_` will be added
to the `sender` config dictionary, overwriting any values that may already
be set.
The sender configuration supports the following values:
class (required)
Dotted name identifying the sender class to instantiate.
args
Sequence of non-keyword args to pass to sender constructor.
<kwargs>
All remaining key-value pairs in the sender config dict will be passed as
keyword arguments to the sender constructor.
"""
# Make a deep copy of the configuration so that subsequent uses of
# the config won't blow up
config = nest_prefixes(copy.deepcopy(config))
config_copy = json.dumps(copy.deepcopy(config))
sender_config = config.get('sender', {})
logger = config.get('logger', '')
severity = config.get('severity', 6)
disabled_timers = config.get('disabled_timers', [])
filter_specs = config.get('filters', [])
plugins_data = config.pop('plugins', {})
global_conf = config.get('global', {})
# update global config stored in CLIENT_HOLDER
from metlog.holder import CLIENT_HOLDER
if clear_global:
CLIENT_HOLDER.global_config = {}
CLIENT_HOLDER.global_config.update(global_conf)
resolver = DottedNameResolver()
# instantiate sender
sender_clsname = sender_config.pop('class')
sender_cls = resolver.resolve(sender_clsname)
sender_args = sender_config.pop('args', tuple())
sender = sender_cls(*sender_args, **sender_config)
# initialize filters
filters = [
resolver.resolve(dotted_name)(**cfg)
for (dotted_name, cfg) in filter_specs
]
# instantiate and/or configure client
if client is None:
client = MetlogClient(sender, logger, severity, disabled_timers,
filters)
else:
client.setup(sender, logger, severity, disabled_timers, filters)
# initialize plugins and attach to client
for section_name, plugin_spec in plugins_data.items():
# each plugin spec is a 2-tuple: (dotted_name, cfg)
plugin_config = plugin_spec[1]
plugin_override = plugin_config.pop('override', False)
plugin_fn = resolver.resolve(plugin_spec[0])(plugin_config)
client.add_method(plugin_fn, plugin_override)
# We bind the configuration into the client itself to ease
# debugging
client._config = config_copy
return client
class TestDisabledTimer(object):
logger = 'tests'
timer_name = 'test'
def _extract_full_msg(self):
return json.loads(self.mock_sender.msgs[0])
def setUp(self):
self.mock_sender = DebugCaptureSender()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
self.client.sender.msgs.clear()
del self.timer_ob.__dict__['_local']
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
# Now disable it
self.client._disabled_timers.add(name)
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result is None)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.client.sender.msgs.clear()
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_decorator(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.client.sender.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10,
"Got: %d" % int(full_msg['payload']))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
# Now disable it
self.client._disabled_timers.add(name)
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.mock_sender.msgs), 0)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo3():
time.sleep(0.01)
foo3()
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_disable_all_timers(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.client.sender.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
# Now disable everything
self.client._disabled_timers.add('*')
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.mock_sender.msgs), 0)
def client_from_dict_config(config, client=None, clear_global=False):
"""
Configure a metlog client, fully configured w/ sender and plugins.
:param config: Configuration dictionary.
:param client: MetlogClient instance to configure. If None, one will be
created.
:param clear_global: If True, delete any existing global config on the
CLIENT_HOLDER before applying new config.
The configuration dict supports the following values:
logger
Metlog client default logger value.
severity
Metlog client default severity value.
disabled_timers
Sequence of string tokens identifying timers that are to be deactivated.
filters
Sequence of 2-tuples `(filter_provider, config)`. Each `filter_provider`
is a dotted name referring to a function which, when called and passed
the associated `config` dict as kwargs, will return a usable MetlogClient
filter function.
plugins
Nested dictionary containing plugin configuration. Keys are the plugin
names (i.e. the name the method will be given when attached to the
client). Values are 2-tuples `(plugin_provider, config)`. Each
`plugin_provider` is a dotted name referring to a function which, when
called and passed the associated `config`, will return the usable plugin
method.
sender
Nested dictionary containing sender configuration.
global
Dictionary to be applied to CLIENT_HOLDER's `global_config` storage.
New config will overwrite any conflicting values, but will not delete
other config entries. To delete, calling code should call the function
with `clear_global` set to True.
All of the configuration values are optional, but failure to include a
sender may result in a non-functional Metlog client. Any unrecognized keys
will be ignored.
Note that any top level config values starting with `sender_` will be added
to the `sender` config dictionary, overwriting any values that may already
be set.
The sender configuration supports the following values:
class (required)
Dotted name identifying the sender class to instantiate.
args
Sequence of non-keyword args to pass to sender constructor.
<kwargs>
All remaining key-value pairs in the sender config dict will be passed as
keyword arguments to the sender constructor.
"""
# Make a deep copy of the configuration so that subsequent uses of
# the config won't blow up
config = nest_prefixes(copy.deepcopy(config))
sender_config = config.get('sender', {})
logger = config.get('logger', '')
severity = config.get('severity', 6)
disabled_timers = config.get('disabled_timers', [])
filter_specs = config.get('filters', [])
plugins_data = config.pop('plugins', {})
global_conf = config.get('global', {})
# update global config stored in CLIENT_HOLDER
from metlog.holder import CLIENT_HOLDER
if clear_global:
CLIENT_HOLDER.global_config = {}
CLIENT_HOLDER.global_config.update(global_conf)
resolver = DottedNameResolver()
# instantiate sender
sender_clsname = sender_config.pop('class')
sender_cls = resolver.resolve(sender_clsname)
sender_args = sender_config.pop('args', tuple())
sender = sender_cls(*sender_args, **sender_config)
# initialize filters
filters = [resolver.resolve(dotted_name)(**cfg)
for (dotted_name, cfg) in filter_specs]
# instantiate and/or configure client
if client is None:
client = MetlogClient(sender, logger, severity, disabled_timers,
filters)
else:
client.setup(sender, logger, severity, disabled_timers, filters)
# initialize plugins and attach to client
for section_name, plugin_spec in plugins_data.items():
# each plugin spec is a 2-tuple: (dotted_name, cfg)
plugin_config = plugin_spec[1]
plugin_override = plugin_config.pop('override', False)
plugin_fn = resolver.resolve(plugin_spec[0])(plugin_config)
client.add_method(plugin_fn, plugin_override)
return client
class TestMetlogClient(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.mock_sender = Mock()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
del self.timer_ob.__dict__['_local']
del self.mock_sender
def _extract_full_msg(self):
return self.mock_sender.send_message.call_args[0][0]
def test_metlog_bare(self):
payload = 'this is a test'
before = datetime.utcnow().isoformat()
msgtype = 'testtype'
self.client.metlog(msgtype, payload=payload)
after = datetime.utcnow().isoformat()
full_msg = self._extract_full_msg()
# check the payload
eq_(full_msg['payload'], payload)
# check the various default values
ok_(before < full_msg['timestamp'] < after)
eq_(full_msg['type'], msgtype)
eq_(full_msg['severity'], self.client.severity)
eq_(full_msg['logger'], self.logger)
eq_(full_msg['metlog_pid'], os.getpid())
eq_(full_msg['metlog_hostname'], socket.gethostname())
eq_(full_msg['env_version'], self.client.env_version)
def test_metlog_full(self):
metlog_args = dict(payload='this is another test',
timestamp=datetime.utcnow(),
logger='alternate',
severity=2,
fields={'foo': 'bar',
'boo': 'far'})
msgtype = 'bawlp'
self.client.metlog(msgtype, **metlog_args)
actual_msg = self._extract_full_msg()
metlog_args.update({'type': msgtype,
'env_version': self.client.env_version,
'metlog_pid': os.getpid(),
'metlog_hostname': socket.gethostname(),
'timestamp': metlog_args['timestamp'].isoformat()})
eq_(actual_msg, metlog_args)
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_decorator(self):
@self.client.timer(self.timer_name)
def timed():
time.sleep(0.01)
ok_(not self.mock_sender.send_message.called)
timed()
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], self.timer_name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_with_rate(self):
name = self.timer_name
@self.client.timer(name, rate=0.01)
def timed():
time.sleep(0.001)
for i in range(10):
timed()
# this is a weak test, but not quite sure how else to
# test explicitly random behaviour
ok_(self.mock_sender.send_message.call_count < 10)
def test_incr(self):
name = 'incr'
self.client.incr(name)
full_msg = self._extract_full_msg()
eq_(full_msg['type'], 'counter')
eq_(full_msg['logger'], self.logger)
eq_(full_msg['fields']['name'], name)
# You have to have a rate set here
eq_(full_msg['fields']['rate'], 1)
eq_(full_msg['payload'], '1')
self.client.incr(name, 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], '10')
class TestMetlogClientFilters(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.sender = DebugCaptureSender()
self.client = MetlogClient(self.sender, self.logger)
def tearDown(self):
del self.sender
del self.client
def test_severity_max(self):
from metlog.filters import severity_max_provider
self.client.filters = [severity_max_provider(severity=SEVERITY.ERROR)]
payload = 'foo'
self.client.debug(payload)
self.client.info(payload)
self.client.warn(payload)
self.client.error(payload)
self.client.exception(payload)
self.client.critical(payload)
# only half of the messages should have gone out
eq_(len(self.sender.msgs), 3)
# make sure it's the right half
for json_msg in self.sender.msgs:
msg = json.loads(json_msg)
ok_(msg['severity'] <= SEVERITY.ERROR)
def test_type_blacklist(self):
from metlog.filters import type_blacklist_provider
type_blacklist = type_blacklist_provider(types=set(['foo']))
self.client.filters = [type_blacklist]
choices = ['foo', 'bar']
notfoos = 0
for i in range(10):
choice = random.choice(choices)
if choice != 'foo':
notfoos += 1
self.client.metlog(choice, payload='msg')
eq_(len(self.sender.msgs), notfoos)
def test_type_whitelist(self):
from metlog.filters import type_whitelist_provider
type_whitelist = type_whitelist_provider(types=set(['foo']))
self.client.filters = [type_whitelist]
choices = ['foo', 'bar']
foos = 0
for i in range(10):
choice = random.choice(choices)
if choice == 'foo':
foos += 1
self.client.metlog(choice, payload='msg')
eq_(len(self.sender.msgs), foos)
def test_type_severity_max(self):
from metlog.filters import type_severity_max_provider
config = {'types': {'foo': {'severity': 3},
'bar': {'severity': 5},
},
}
type_severity_max = type_severity_max_provider(**config)
self.client.filters = [type_severity_max]
for msgtype in ['foo', 'bar']:
for sev in range(8):
self.client.metlog(msgtype, severity=sev, payload='msg')
eq_(len(self.sender.msgs), 10)
msgs = [json.loads(msg) for msg in self.sender.msgs]
foos = [msg for msg in msgs if msg['type'] == 'foo']
eq_(len(foos), 4)
bars = [msg for msg in msgs if msg['type'] == 'bar']
eq_(len(bars), 6)
class TestDisabledTimer(object):
logger = 'tests'
timer_name = 'test'
def _extract_full_msg(self):
return json.loads(self.mock_sender.msgs[0])
def setUp(self):
self.mock_sender = DebugCaptureSender()
self.client = MetlogClient(self.mock_sender, self.logger)
# overwrite the class-wide threadlocal w/ an instance one
# so values won't persist btn tests
self.timer_ob = self.client.timer(self.timer_name)
self.timer_ob.__dict__['_local'] = threading.local()
def tearDown(self):
self.client.sender.msgs.clear()
del self.timer_ob.__dict__['_local']
def test_timer_contextmanager(self):
name = self.timer_name
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
# Now disable it
self.client._disabled_timers.add(name)
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result is None)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.client.sender.msgs.clear()
with self.client.timer(name) as timer:
time.sleep(0.01)
ok_(timer.result >= 10)
full_msg = self._extract_full_msg()
eq_(full_msg['payload'], str(timer.result))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_timer_decorator(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.client.sender.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10,
"Got: %d" % int(full_msg['payload']))
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
# Now disable it
self.client._disabled_timers.add(name)
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.mock_sender.msgs), 0)
# Now re-enable it
self.client._disabled_timers.remove(name)
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo3():
time.sleep(0.01)
foo3()
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
def test_disable_all_timers(self):
name = self.timer_name
@self.client.timer(name)
def foo():
time.sleep(0.01)
foo()
eq_(len(self.client.sender.msgs), 1)
full_msg = self._extract_full_msg()
ok_(int(full_msg['payload']) >= 10)
eq_(full_msg['type'], 'timer')
eq_(full_msg['fields']['name'], name)
eq_(full_msg['fields']['rate'], 1)
# Now disable everything
self.client._disabled_timers.add('*')
self.client.sender.msgs.clear()
@self.client.timer(name)
def foo2():
time.sleep(0.01)
foo2()
eq_(len(self.mock_sender.msgs), 0)
#!/usr/bin/env python
from metlog.senders import ZmqPubSender
from metlog.client import MetlogClient
import time
sender = ZmqPubSender('tcp://127.0.0.1:5565')
client = MetlogClient(sender, 'testy')
while True:
time.sleep(1)
client.metlog('cmd', payload='come in here watson, I need you!')
class TestMetlogClientFilters(object):
logger = 'tests'
timer_name = 'test'
def setUp(self):
self.sender = DebugCaptureSender()
self.client = MetlogClient(self.sender, self.logger)
def tearDown(self):
del self.sender
del self.client
def test_severity_max(self):
from metlog.filters import severity_max_provider
self.client.filters = [severity_max_provider(severity=SEVERITY.ERROR)]
payload = 'foo'
self.client.debug(payload)
self.client.info(payload)
self.client.warn(payload)
self.client.error(payload)
self.client.exception(payload)
self.client.critical(payload)
# only half of the messages should have gone out
eq_(len(self.sender.msgs), 3)
# make sure it's the right half
for json_msg in self.sender.msgs:
msg = json.loads(json_msg)
ok_(msg['severity'] <= SEVERITY.ERROR)
def test_type_blacklist(self):
from metlog.filters import type_blacklist_provider
type_blacklist = type_blacklist_provider(types=set(['foo']))
self.client.filters = [type_blacklist]
choices = ['foo', 'bar']
notfoos = 0
for i in range(10):
choice = random.choice(choices)
if choice != 'foo':
notfoos += 1
self.client.metlog(choice, payload='msg')
eq_(len(self.sender.msgs), notfoos)
def test_type_whitelist(self):
from metlog.filters import type_whitelist_provider
type_whitelist = type_whitelist_provider(types=set(['foo']))
self.client.filters = [type_whitelist]
choices = ['foo', 'bar']
foos = 0
for i in range(10):
choice = random.choice(choices)
if choice == 'foo':
foos += 1
self.client.metlog(choice, payload='msg')
eq_(len(self.sender.msgs), foos)
def test_type_severity_max(self):
from metlog.filters import type_severity_max_provider
config = {
'types': {
'foo': {
'severity': 3
},
'bar': {
'severity': 5
},
},
}
type_severity_max = type_severity_max_provider(**config)
self.client.filters = [type_severity_max]
for msgtype in ['foo', 'bar']:
for sev in range(8):
self.client.metlog(msgtype, severity=sev, payload='msg')
eq_(len(self.sender.msgs), 10)
msgs = [json.loads(msg) for msg in self.sender.msgs]
foos = [msg for msg in msgs if msg['type'] == 'foo']
eq_(len(foos), 4)
bars = [msg for msg in msgs if msg['type'] == 'bar']
eq_(len(bars), 6)
