class TestCLogScribeReportStatus(object):
@pytest.yield_fixture(autouse=True)
def setup_sandbox(self):
self.scribe_logdir = tempfile.mkdtemp()
self.stream = 'foo'
self.scribe_port = find_open_port()
self.log_path = get_log_path(self.scribe_logdir, self.stream)
self.logger = ScribeLogger(
'localhost',
self.scribe_port,
retry_interval=10,
report_status = mock.Mock()
)
with scribed_sandbox(self.scribe_port, self.scribe_logdir):
yield
shutil.rmtree(self.scribe_logdir)
def test_exception_in_raise_status(self):
"""Make sure socket is closed if exception is raised in report_status function."""
def raise_exception_on_error(is_error, message):
if is_error:
raise Exception(message)
self.logger.report_status = raise_exception_on_error
self.logger.client.Log = mock.Mock(side_effect=IOError)
try:
self.logger.log_line(self.stream, '12345678')
except Exception:
assert not self.logger.connected
class ScribeHandler(logging.Handler):
"""Handler for sending python standard logging messages to a scribe
stream.
.. code-block:: python
import clog.handlers, logging
log = logging.getLogger(name)
log.addHandler(clog.handlers.ScribeHandler('localhost', 3600, 'stream', retry_interval=3))
:param host: hostname of scribe server
:param port: port number of scribe server
:param stream: name of the scribe stream logs will be sent to
:param retry_interval: default 0, number of seconds to wait between retries
"""
def __init__(self, host, port, stream, retry_interval=0):
logging.Handler.__init__(self)
self.stream = stream
self.logger = ScribeLogger(host, port, retry_interval)
def emit(self, record):
try:
msg = self.format(record)
self.logger.log_line(self.stream, msg)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
class TestCLogScribeLoggerLineSize(object):
@pytest.yield_fixture(autouse=True)
def setup_sandbox(self):
self.scribe_logdir = tempfile.mkdtemp()
self.stream = "foo"
self.scribe_port = find_open_port()
self.log_path = get_log_path(self.scribe_logdir, self.stream)
self.logger = ScribeLogger("localhost", self.scribe_port, retry_interval=10, report_status=mock.Mock())
with scribed_sandbox(self.scribe_port, self.scribe_logdir):
yield
shutil.rmtree(self.scribe_logdir)
def test_line_size_constants(self):
assert MAX_LINE_SIZE_IN_BYTES == 50 * 1024 * 1024
assert WARNING_LINE_SIZE_IN_BYTES == 5 * 1024 * 1024
assert WHO_CLOG_LARGE_LINE_STREAM == "tmp_who_clog_large_line"
def test_log_line_no_size_limit(self):
line = create_test_line()
self.logger._log_line_no_size_limit(self.stream, line)
wait_on_log_data(self.log_path, line + b"\n")
assert not self.logger.report_status.called
@mock.patch("clog.loggers.ScribeLogger._log_line_no_size_limit")
def test_normal_line_size(self, mock_log_line_no_size_limit):
line = create_test_line()
assert len(line) <= WARNING_LINE_SIZE_IN_BYTES
self.logger.log_line(self.stream, line)
assert not self.logger.report_status.called
mock_log_line_no_size_limit.assert_called_once_with(self.stream, line)
@mock.patch("clog.loggers.ScribeLogger._log_line_no_size_limit")
def test_max_line_size(self, mock_log_line_no_size_limit):
line = create_test_line(MAX_LINE_SIZE_IN_BYTES)
assert len(line) > MAX_LINE_SIZE_IN_BYTES
with pytest.raises(LogLineIsTooLongError):
self.logger.log_line(self.stream, line)
assert self.logger.report_status.called_with(
True, "The log line is dropped (line size larger than %r bytes)" % MAX_LINE_SIZE_IN_BYTES
)
assert not mock_log_line_no_size_limit.called
def test_large_msg(self):
# We advertise support of messages up to 50 megs, so let's test that
# we actually are able to log a 50 meg message to a real scribe server
test_str = "0" * MAX_LINE_SIZE_IN_BYTES
self.logger.log_line(self.stream, test_str)
expected = test_str.encode("UTF-8")
wait_on_log_data(self.log_path, expected + b"\n")
@mock.patch("traceback.format_stack")
@mock.patch("clog.loggers.ScribeLogger._log_line_no_size_limit")
def test_warning_line_size(self, mock_log_line_no_size_limit, mock_traceback):
line = create_test_line(WARNING_LINE_SIZE_IN_BYTES)
assert len(line) > WARNING_LINE_SIZE_IN_BYTES
assert len(line) <= MAX_LINE_SIZE_IN_BYTES
self.logger.log_line(self.stream, line)
assert self.logger.report_status.called_with(
False,
"The log line size is larger than %r bytes (monitored in '%s')"
% (WARNING_LINE_SIZE_IN_BYTES, WHO_CLOG_LARGE_LINE_STREAM),
)
assert mock_log_line_no_size_limit.call_count == 2
call_1 = mock.call(self.stream, line)
origin_info = {}
origin_info["stream"] = self.stream
origin_info["line_size"] = len(line)
origin_info["traceback"] = "".join(mock_traceback)
origin_info_line = json.dumps(origin_info).encode("UTF-8")
call_2 = mock.call(WHO_CLOG_LARGE_LINE_STREAM, origin_info_line)
mock_log_line_no_size_limit.assert_has_calls([call_1, call_2])
class TestCLogScribeLoggerLineSize(object):
@pytest.yield_fixture(autouse=True)
def setup_sandbox(self):
self.scribe_logdir = tempfile.mkdtemp()
self.stream = 'foo'
self.scribe_port = find_open_port()
self.log_path = get_log_path(self.scribe_logdir, self.stream)
self.logger = ScribeLogger('localhost',
self.scribe_port,
retry_interval=10,
report_status=mock.Mock())
with scribed_sandbox(self.scribe_port, self.scribe_logdir):
yield
shutil.rmtree(self.scribe_logdir)
def test_line_size_constants(self):
assert MAX_LINE_SIZE_IN_BYTES == 50 * 1024 * 1024
assert WARNING_LINE_SIZE_IN_BYTES == 5 * 1024 * 1024
assert WHO_CLOG_LARGE_LINE_STREAM == 'tmp_who_clog_large_line'
def test_log_line_no_size_limit(self):
line = create_test_line()
self.logger._log_line_no_size_limit(self.stream, line)
wait_on_log_data(self.log_path, line + b'\n')
assert not self.logger.report_status.called
@mock.patch('clog.loggers.ScribeLogger._log_line_no_size_limit')
def test_normal_line_size(self, mock_log_line_no_size_limit):
line = create_test_line()
assert len(line) <= WARNING_LINE_SIZE_IN_BYTES
self.logger.log_line(self.stream, line)
assert not self.logger.report_status.called
mock_log_line_no_size_limit.assert_called_once_with(self.stream, line)
@mock.patch('clog.loggers.ScribeLogger._log_line_no_size_limit')
def test_max_line_size(self, mock_log_line_no_size_limit):
line = create_test_line(MAX_LINE_SIZE_IN_BYTES)
assert len(line) > MAX_LINE_SIZE_IN_BYTES
with pytest.raises(LogLineIsTooLongError):
self.logger.log_line(self.stream, line)
assert self.logger.report_status.called_with(
True, 'The log line is dropped (line size larger than %r bytes)' %
MAX_LINE_SIZE_IN_BYTES)
assert not mock_log_line_no_size_limit.called
def test_large_msg(self):
# We advertise support of messages up to 50 megs, so let's test that
# we actually are able to log a 50 meg message to a real scribe server
test_str = '0' * MAX_LINE_SIZE_IN_BYTES
self.logger.log_line(self.stream, test_str)
expected = test_str.encode('UTF-8')
wait_on_log_data(self.log_path, expected + b'\n')
@mock.patch('traceback.format_stack')
@mock.patch('clog.loggers.ScribeLogger._log_line_no_size_limit')
def test_warning_line_size(self, mock_log_line_no_size_limit,
mock_traceback):
line = create_test_line(WARNING_LINE_SIZE_IN_BYTES)
assert len(line) > WARNING_LINE_SIZE_IN_BYTES
assert len(line) <= MAX_LINE_SIZE_IN_BYTES
self.logger.log_line(self.stream, line)
assert self.logger.report_status.called_with(
False,
'The log line size is larger than %r bytes (monitored in \'%s\')' %
(WARNING_LINE_SIZE_IN_BYTES, WHO_CLOG_LARGE_LINE_STREAM))
assert mock_log_line_no_size_limit.call_count == 2
call_1 = mock.call(self.stream, line)
origin_info = {}
origin_info['stream'] = self.stream
origin_info['line_size'] = len(line)
origin_info['traceback'] = ''.join(mock_traceback)
origin_info_line = json.dumps(origin_info).encode('UTF-8')
call_2 = mock.call(WHO_CLOG_LARGE_LINE_STREAM, origin_info_line)
mock_log_line_no_size_limit.assert_has_calls([call_1, call_2])
