def __init__(self,
stream,
host='localhost',
port=12201,
protocol='udp',
_filters=None,
environment=None,
json_only=False):
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
self.buffer = deque()
self.environment = str(environment)
if _filters is not None:
self.filters = _filters.split(',')
else:
self.filters = None
self.json_only = bool(json_only)
self.protocol = str(protocol)
self.stream = stream
self.handleropts = {
'host': str(host),
'port': int(port),
'protocol': str(protocol)
}
if str(protocol) == 'udp':
self.gelfhandler = GelfUdpHandler(host=str(host),
port=int(port),
include_extra_fields=True,
compress=True)
elif str(protocol) == 'tcp':
self.gelfhandler = GelfTcpHandler(host=str(host),
port=int(port),
include_extra_fields=True)
elif str(protocol) == 'tls':
self.gelfhandler = GelfTlsHandler(host=str(host),
port=int(port),
include_extra_fields=True)
start_ts = end_ts - (timespan * 1000)
_dg_debug = config.getboolean('debug', False)
# Setup GELF logger
h = None
gelf_host = config.get('host', '127.0.0.1')
gelf_port = int(config.get('port', 11201))
gelf_protocol = config.get('protocol', 'udp') # (tcp|udp|tls)
gelf_source = config.get('source_label', 'Dropbox-audit')
if gelf_protocol == 'udp':
h = GelfUdpHandler(host=gelf_host, port=gelf_port, source=gelf_source)
elif gelf_protocol == 'tcp':
h = GelfTcpHandler(host=gelf_host, port=gelf_port, source=gelf_source)
elif gelf_protocol == 'tls':
tls_cafile = config.get('tls_cafile',
'/etc/ssl/certs/ca-certificates.crt')
h = GelfTlsHandler(host=gelf_host,
port=gelf_port,
source=gelf_source,
validate=True,
ca_certs=tls_cafile)
else:
sys.exit('Unknown protocol')
logger = logging.getLogger()
logger.addHandler(h)
# Import audit entries
n = dropbox_to_graylog(token, start_ts, end_ts, limit=limit, handler=h)
h.flush()
logging.shutdown()
if _dg_debug:
print("Injected {} entries to {}".format(n, gelf_host))
import logging
import json
import pytest
import mock
@pytest.yield_fixture
def send(extra_fields_handler):
with mock.patch.object(extra_fields_handler, 'send') as mock_send:
yield mock_send
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12000, include_extra_fields=True),
GelfUdpHandler(host='127.0.0.1', port=12000, compress=False, include_extra_fields=True),
GelfTlsHandler(host='127.0.0.1', port=12000, include_extra_fields=True)
])
def extra_fields_handler(request):
return request.param
class ContextFilter(logging.Filter):
def filter(self, record):
record.ozzy = 'diary of a madman'
record.van_halen = 1984
record.id = 123
return True
@pytest.yield_fixture
def extra_fields_logger(extra_fields_handler):
def handler():
return GelfTlsHandler(host='127.0.0.1', port=12000)
def test_handler_creation():
with pytest.raises(ValueError):
GelfTlsHandler(host='127.0.0.1', port=12001, validate=True)
import socket
ADDITIONAL_FIELDS = {
'_ozzy': 'diary of a madman',
'_van_halen': 1984,
'_id': '123'
}
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12000, **ADDITIONAL_FIELDS),
GelfUdpHandler(host='127.0.0.1',
port=12000,
compress=False,
**ADDITIONAL_FIELDS),
GelfTlsHandler(host='127.0.0.1', port=12000, **ADDITIONAL_FIELDS)
])
def handler(request):
return request.param
@pytest.yield_fixture
def send(handler):
with mock.patch.object(handler, 'send') as mock_send:
yield mock_send
@pytest.yield_fixture
def logger(handler):
logger = logging.getLogger('test')
logger.addHandler(handler)
STATIC_FIELDS = {'_ozzy': 'diary of a madman', '_van_halen': 1984, '_id': 42}
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, **STATIC_FIELDS),
GelfUdpHandler(host='127.0.0.1', port=12202, **STATIC_FIELDS),
GelfUdpHandler(host='127.0.0.1',
port=12202,
compress=False,
**STATIC_FIELDS),
GelfHttpHandler(host='127.0.0.1', port=12203, **STATIC_FIELDS),
GelfHttpHandler(host='127.0.0.1',
port=12203,
compress=False,
**STATIC_FIELDS),
GelfTlsHandler(host='127.0.0.1', port=12204, **STATIC_FIELDS),
GelfTlsHandler(host='127.0.0.1',
port=12204,
validate=True,
ca_certs='tests/config/cert.pem',
**STATIC_FIELDS),
GelfTcpHandler(host='127.0.0.1',
port=12201,
static_fields=STATIC_FIELDS,
_ozzy='billie jean'),
GelfUdpHandler(host='127.0.0.1',
port=12202,
static_fields=STATIC_FIELDS,
_ozzy='billie jean'),
GelfUdpHandler(host='127.0.0.1',
port=12202,
import pytest
import mock
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler
from tests.helper import logger, get_unique_message, log_warning, log_exception
SYSLOG_LEVEL_ERROR = 3
SYSLOG_LEVEL_WARNING = 4
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201),
GelfUdpHandler(host='127.0.0.1', port=12202),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False),
GelfHttpHandler(host='127.0.0.1', port=12203),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False),
GelfTlsHandler(host='127.0.0.1', port=12204),
# GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem'),
])
def handler(request):
return request.param
def test_simple_message(logger):
message = get_unique_message()
graylog_response = log_warning(logger, message)
assert graylog_response['message'] == message
assert graylog_response['level'] == SYSLOG_LEVEL_WARNING
assert 'full_message' not in graylog_response
assert 'file' not in graylog_response
assert 'module' not in graylog_response
assert 'func' not in graylog_response
def main(args, loglevel):
"""
:param args: see main
:param loglevel:
:return:
"""
# config file loader
config_dict = read_config(file=args.config)
### Loggers ###
# graylog logger
logging.basicConfig(level=loglevel)
logger = logging.getLogger()
logger.addHandler(GelfTlsHandler(host=config_dict['graylog_server'], port=config_dict['gralog_port'],
include_extra_fields=True))
format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# console logger not needed, GELF handler writes to console by default
#ch = logging.StreamHandler(sys.stdout)
#ch.setFormatter(format)
#logger.addHandler(ch)
fh = RotatingFileHandler('backlog.txt', maxBytes=(1048576 * 5), backupCount=7)
fh.setFormatter(format)
logger.addHandler(fh)
### Loggers END ###
logger.info('Graylog site monitor activated with config file: {}'.format(args.config))
logger.debug('Contains {} websites'.format(len(config_dict['websites'])))
for website in config_dict['websites']:
web_domain = website['domain']
web_lookups = website['lookups']
logger.debug("Starting check of website: {}, has {} lookups".format(web_domain,len(web_lookups)))
requests_session = init_session(website.get("login"))
for counter, web_lookup in enumerate(web_lookups):
web_url = web_lookup['url']
lookup_status, lookup_message = check_lookup(web_lookup, requests_session)
lookup_description = web_lookup.get('description')
if lookup_status == "OK":
logger.debug("Lookup n.{}: OK, url: {}".format(counter, web_url),
extra={'url': web_url, 'domain': web_domain, 'lookup_message': lookup_message})
elif lookup_status == "WARNING":
logger.warning("Warning in {} (Desc:{}), {}, url: {}".format(web_domain,lookup_description,lookup_message, web_url),
extra={'url':web_url,'domain': web_domain, 'lookup_message':lookup_message})
elif lookup_status == "ERROR":
logger.error("Error in {} (Desc:{}), {}, url: {}".format(web_domain,lookup_description,lookup_message, web_url),
extra={'url':web_url,'domain': web_domain, 'lookup_message':lookup_message})
else:
logger.critical("Malfunction in {} (Desc:{}): lookup_status not recognized: {}, type:".format(web_url, lookup_description,type(lookup_status)),
extra={'url': web_url, 'domain': web_domain, 'lookup_message': lookup_message})
requests_session.close()
class StreamToGelf:
def __init__(self,
stream,
host='localhost',
port=12201,
protocol='udp',
_filters=None,
environment=None,
json_only=False):
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
self.buffer = deque()
self.environment = str(environment)
if _filters is not None:
self.filters = _filters.split(',')
else:
self.filters = None
self.json_only = bool(json_only)
self.protocol = str(protocol)
self.stream = stream
self.handleropts = {
'host': str(host),
'port': int(port),
'protocol': str(protocol)
}
if str(protocol) == 'udp':
self.gelfhandler = GelfUdpHandler(host=str(host),
port=int(port),
include_extra_fields=True,
compress=True)
elif str(protocol) == 'tcp':
self.gelfhandler = GelfTcpHandler(host=str(host),
port=int(port),
include_extra_fields=True)
elif str(protocol) == 'tls':
self.gelfhandler = GelfTlsHandler(host=str(host),
port=int(port),
include_extra_fields=True)
def _send_gelf(self):
try:
record = json_loads_byteified(''.join(self.buffer))
except ValueError as e:
print "ERROR - cannot load json: " + str(e)
else:
try:
assert 'MESSAGE' in record.keys()
except AssertionError:
pass
else:
if self.filters is not None:
matchlist = [
record[filterkey] for filterkey in record.keys()
if filterkey in unfilteredJournalctlKeys
]
for filterelement in self.filters:
for matchelement in matchlist:
if re.search(str(filterelement),
str(matchelement)):
break
else:
continue
break
message = {'version': '1.1'}
for key, value in record.iteritems():
if key == 'MESSAGE':
# try to unnest and index json message keys, if the message is a valid json.
try:
innerrecord = json_loads_byteified(value)
except AttributeError:
if self.json_only:
break
else:
message['short_message'] = value
except ValueError:
if self.json_only:
break
else:
message['short_message'] = value
except Exception as e:
print(e)
break
else:
try:
assert 'message' in innerrecord.keys()
except AssertionError:
message['short_message'] = json.dumps(
innerrecord)
else:
try:
for hashkey, hashvalue in innerrecord.iteritems(
):
if str(hashkey) == 'message':
message[
'short_message'] = hashvalue
else:
message['_' + str(
hashkey).lower()] = hashvalue
except Exception as e:
print "EXCEPTION: " + str(e)
elif key == '__REALTIME_TIMESTAMP':
# convert from systemd's format of microseconds expressed as
# an integer to graylog's float format, eg: "seconds.microseconds"
message['timestamp'] = float(value) / (1000 * 1000)
elif key == '_HOSTNAME':
message['host'] = value
else:
if len(unfilteredJournalctlKeys) > 0 and str(
key) in unfilteredJournalctlKeys:
message['_' + str(key).lower()] = value
if self.environment is not None:
message['_environment'] = str(self.environment)
else:
try:
if self.protocol != 'udp':
self.gelfhandler.send(
zlib.compress(json.dumps(message)))
else:
self.gelfhandler.send(json.dumps(message))
except Exception as e:
print str(e)
finally:
self.buffer.clear()
def run(self):
for line in self.stream:
line = line.strip()
self.buffer.append(line)
self._send_gelf()
def stop(self, *args):
self.buffer.clear()
self.gelfhandler.flush()
self.gelfhandler.close()
print "Exiting..."
sys.exit(0)
return True
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, include_extra_fields=True),
GelfUdpHandler(host='127.0.0.1', port=12202, include_extra_fields=True),
GelfUdpHandler(host='127.0.0.1',
port=12202,
compress=False,
include_extra_fields=True),
GelfHttpHandler(host='127.0.0.1', port=12203, include_extra_fields=True),
GelfHttpHandler(host='127.0.0.1',
port=12203,
compress=False,
include_extra_fields=True),
GelfTlsHandler(host='127.0.0.1', port=12204, include_extra_fields=True),
GelfTlsHandler(host='127.0.0.1',
port=12204,
validate=True,
ca_certs='tests/config/cert.pem',
include_extra_fields=True),
])
def handler(request):
return request.param
@pytest.yield_fixture
def logger(handler):
logger = logging.getLogger('test')
dummy_filter = DummyFilter()
logger.addFilter(dummy_filter)
def test_handler_certfile_required_for_keyfile():
with pytest.raises(ValueError):
GelfTlsHandler(host='127.0.0.1', port=12001, keyfile="./somefile.key")
def test_tls_handler_init():
with pytest.raises(ValueError):
GelfTlsHandler(host='127.0.0.1', port=12204, validate=True)
with pytest.raises(ValueError):
GelfTlsHandler(host='127.0.0.1', port=12204, keyfile='/dev/null')
# mongo.py
from flask import Flask, render_template
from flask import jsonify
from flask import request
from flask_pymongo import PyMongo
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler, GelfHttpHandler
import logging
app = Flask(__name__)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.addHandler(GelfTcpHandler(host='127.0.0.1', port=12201))
logger.addHandler(GelfUdpHandler(host='127.0.0.1', port=12201))
logger.addHandler(GelfTlsHandler(host='127.0.0.1', port=12201))
logger.addHandler(GelfHttpHandler(host='127.0.0.1', port=12201))
logger.info('hello gelf')
app.config['MONGO_DBNAME'] = 'db_teste_itau1'
app.config['MONGO_URI'] = 'mongodb://*****:*****@app.route('/tweets', methods=['GET'])
def get_all_tweets():
teste4 = mongo.db.teste4
output = []
for s in teste4.find():
def setup_logging(root_log_level: Optional[str] = None,
module_log_levels: Optional[Mapping[str, Union[str,
int]]] = None,
add_console_handler: bool = True,
console_handler_format: Optional[str] = None):
"""
Setup log handlers for each application's namespace
:param root_log_level: The log level all loggers use by default. (Default: `"WARNING"`)
:param module_log_levels: A mapping of module names to their desired log levels.
:param add_console_handler: If `True`, enable logging to stdout. (Default: `True`).
:param console_handler_format: Specifies the format of stdout logs. (Default: `DEFAULT_LOG_FORMAT`).
"""
if root_log_level is None:
root_log_level = "WARNING"
root_logger = logging.getLogger()
root_logger.setLevel(root_log_level)
if module_log_levels:
for module_name, log_level in module_log_levels.items():
logging.getLogger(module_name).setLevel(log_level)
if add_console_handler:
console_handler = logging.StreamHandler()
console_handler.setFormatter(
logging.Formatter(
fmt=console_handler_format or DEFAULT_LOG_FORMAT))
root_logger.addHandler(console_handler)
# Add GELF handler if GELF is enabled
if "GRAYLOG_HOST" in os.environ:
common_opts = dict(host=os.environ["GRAYLOG_HOST"],
debug=bool(os.environ.get("GRAYLOG_DEBUG", True)),
include_extra_fields=True,
_app_name=os.environ["GRAYLOG_APP_NAME"],
_env=os.environ["GRAYLOG_ENV"])
handler_type = os.environ.get("GRAYLOG_HANDLER_TYPE", "tls").lower()
if handler_type == "udp":
gelf_handler = GelfUdpHandler(**common_opts,
port=int(
os.environ.get(
"GRAYLOG_UDP_PORT", 12201)),
chunk_size=int(
os.environ.get(
"GRAYLOG_UDP_CHUNK_SIZE",
1300)),
compress=bool(
os.environ.get(
"GRAYLOG_UDP_COMPRESS",
True)))
elif handler_type == "tls":
gelf_handler = GelfTlsHandler(**common_opts,
port=int(
os.environ.get(
"GRAYLOG_TLS_PORT", 12201)),
timeout=float(
os.environ.get(
"GRAYLOG_TLS_TIMEOUT_SECS",
0.3)))
else:
raise ValueError(
"Graylog handler type '{}' not recognized. Valid types are 'udp' and 'tls'."
.format(handler_type))
root_logger.addHandler(gelf_handler)
# Add datadog handler if log to datadog is enabled
if "DATADOG_HOST" in os.environ:
opts = dict(host=os.environ["DATADOG_HOST"],
port=int(os.environ.get("DATADOG_UDP_PORT", 10518)))
datadog_handler = DataDogUdpHandler(**opts)
root_logger.addHandler(datadog_handler)
import pytest
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler
from tests.helper import logger, get_unique_message, log_warning
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, debug=True),
GelfUdpHandler(host='127.0.0.1', port=12202, debug=True),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, debug=True),
GelfHttpHandler(host='127.0.0.1', port=12203, debug=True),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, debug=True),
GelfTlsHandler(host='127.0.0.1', port=12204, debug=True),
GelfTlsHandler(host='127.0.0.1',
port=12204,
debug=True,
validate=True,
ca_certs='tests/config/cert.pem'),
])
def handler(request):
return request.param
def test_debug_mode(logger):
message = get_unique_message()
graylog_response = log_warning(logger, message)
assert graylog_response['message'] == message
assert graylog_response['file'] == 'helper.py'
assert graylog_response['module'] == 'helper'
assert graylog_response['func'] == 'log_warning'
assert graylog_response['logger_name'] == 'test'
assert 'line' in graylog_response
STATIC_FIELDS = {
'_ozzy': 'diary of a madman',
'_van_halen': 1984,
'_id': 42
}
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, **STATIC_FIELDS),
GelfUdpHandler(host='127.0.0.1', port=12202, **STATIC_FIELDS),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, **STATIC_FIELDS),
GelfHttpHandler(host='127.0.0.1', port=12203, **STATIC_FIELDS),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, **STATIC_FIELDS),
GelfTlsHandler(host='127.0.0.1', port=12204, **STATIC_FIELDS),
# GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', **STATIC_FIELDS),
GelfTcpHandler(host='127.0.0.1', port=12201, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfUdpHandler(host='127.0.0.1', port=12202, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfHttpHandler(host='127.0.0.1', port=12203, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, static_fields=STATIC_FIELDS, _ozzy='billie jean'),
GelfTlsHandler(host='127.0.0.1', port=12204, static_fields=STATIC_FIELDS),
# GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', static_fields=STATIC_FIELDS, _ozzy='billie jean'),
])
def handler(request):
return request.param
def test_static_fields(logger):
message = get_unique_message()
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler
import logging
import json
import pytest
import mock
def log_and_decode(_logger, _send, text):
_logger.warning(text)
message = _send.call_args[0][0].replace(b'\x00', b'').decode('utf-8')
return json.loads(message)
@pytest.mark.parametrize('handler,version', [
(GelfTcpHandler(host='127.0.0.1', port=12000, version='10.7'), '10.7'),
(GelfUdpHandler(
host='127.0.0.1', port=12000, compress=False, version='11.5'), '11.5'),
(GelfTlsHandler(host='127.0.0.1', port=12000, version='12.3'), '12.3')
])
def test_custom_version(handler, version):
logger = logging.getLogger('test')
logger.addHandler(handler)
with mock.patch.object(handler, 'send') as send:
message = log_and_decode(logger, send, 'custom version')
assert message['version'] == version
logger.removeHandler(handler)
import pytest
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler
from tests.helper import logger, get_unique_message, log_warning
@pytest.fixture(params=[
GelfTcpHandler(host='127.0.0.1', port=12201, debug=True),
GelfUdpHandler(host='127.0.0.1', port=12202, debug=True),
GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, debug=True),
GelfHttpHandler(host='127.0.0.1', port=12203, debug=True),
GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, debug=True),
GelfTlsHandler(host='127.0.0.1', port=12204, debug=True),
# GelfTlsHandler(host='127.0.0.1', port=12204, debug=True, validate=True, ca_certs='tests/config/cert.pem'),
])
def handler(request):
return request.param
def test_debug_mode(logger):
message = get_unique_message()
graylog_response = log_warning(logger, message)
assert graylog_response['message'] == message
assert graylog_response['file'] == 'helper.py'
assert graylog_response['module'] == 'helper'
assert graylog_response['func'] == 'log_warning'
assert graylog_response['logger_name'] == 'test'
assert 'line' in graylog_response