def __init__(self, stream, host='localhost', port=12201, protocol='udp', _filters=None, environment=None, json_only=False): signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop) self.buffer = deque() self.environment = str(environment) if _filters is not None: self.filters = _filters.split(',') else: self.filters = None self.json_only = bool(json_only) self.protocol = str(protocol) self.stream = stream self.handleropts = { 'host': str(host), 'port': int(port), 'protocol': str(protocol) } if str(protocol) == 'udp': self.gelfhandler = GelfUdpHandler(host=str(host), port=int(port), include_extra_fields=True, compress=True) elif str(protocol) == 'tcp': self.gelfhandler = GelfTcpHandler(host=str(host), port=int(port), include_extra_fields=True) elif str(protocol) == 'tls': self.gelfhandler = GelfTlsHandler(host=str(host), port=int(port), include_extra_fields=True)
start_ts = end_ts - (timespan * 1000) _dg_debug = config.getboolean('debug', False) # Setup GELF logger h = None gelf_host = config.get('host', '127.0.0.1') gelf_port = int(config.get('port', 11201)) gelf_protocol = config.get('protocol', 'udp') # (tcp|udp|tls) gelf_source = config.get('source_label', 'Dropbox-audit') if gelf_protocol == 'udp': h = GelfUdpHandler(host=gelf_host, port=gelf_port, source=gelf_source) elif gelf_protocol == 'tcp': h = GelfTcpHandler(host=gelf_host, port=gelf_port, source=gelf_source) elif gelf_protocol == 'tls': tls_cafile = config.get('tls_cafile', '/etc/ssl/certs/ca-certificates.crt') h = GelfTlsHandler(host=gelf_host, port=gelf_port, source=gelf_source, validate=True, ca_certs=tls_cafile) else: sys.exit('Unknown protocol') logger = logging.getLogger() logger.addHandler(h) # Import audit entries n = dropbox_to_graylog(token, start_ts, end_ts, limit=limit, handler=h) h.flush() logging.shutdown() if _dg_debug: print("Injected {} entries to {}".format(n, gelf_host))
import logging import json import pytest import mock @pytest.yield_fixture def send(extra_fields_handler): with mock.patch.object(extra_fields_handler, 'send') as mock_send: yield mock_send @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12000, include_extra_fields=True), GelfUdpHandler(host='127.0.0.1', port=12000, compress=False, include_extra_fields=True), GelfTlsHandler(host='127.0.0.1', port=12000, include_extra_fields=True) ]) def extra_fields_handler(request): return request.param class ContextFilter(logging.Filter): def filter(self, record): record.ozzy = 'diary of a madman' record.van_halen = 1984 record.id = 123 return True @pytest.yield_fixture def extra_fields_logger(extra_fields_handler):
def handler(): return GelfTlsHandler(host='127.0.0.1', port=12000)
def test_handler_creation(): with pytest.raises(ValueError): GelfTlsHandler(host='127.0.0.1', port=12001, validate=True)
import socket ADDITIONAL_FIELDS = { '_ozzy': 'diary of a madman', '_van_halen': 1984, '_id': '123' } @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12000, **ADDITIONAL_FIELDS), GelfUdpHandler(host='127.0.0.1', port=12000, compress=False, **ADDITIONAL_FIELDS), GelfTlsHandler(host='127.0.0.1', port=12000, **ADDITIONAL_FIELDS) ]) def handler(request): return request.param @pytest.yield_fixture def send(handler): with mock.patch.object(handler, 'send') as mock_send: yield mock_send @pytest.yield_fixture def logger(handler): logger = logging.getLogger('test') logger.addHandler(handler)
STATIC_FIELDS = {'_ozzy': 'diary of a madman', '_van_halen': 1984, '_id': 42} @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12201, **STATIC_FIELDS), GelfUdpHandler(host='127.0.0.1', port=12202, **STATIC_FIELDS), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, **STATIC_FIELDS), GelfHttpHandler(host='127.0.0.1', port=12203, **STATIC_FIELDS), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, **STATIC_FIELDS), GelfTlsHandler(host='127.0.0.1', port=12204, **STATIC_FIELDS), GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', **STATIC_FIELDS), GelfTcpHandler(host='127.0.0.1', port=12201, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfUdpHandler(host='127.0.0.1', port=12202, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfUdpHandler(host='127.0.0.1', port=12202,
import pytest import mock from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler from tests.helper import logger, get_unique_message, log_warning, log_exception SYSLOG_LEVEL_ERROR = 3 SYSLOG_LEVEL_WARNING = 4 @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12201), GelfUdpHandler(host='127.0.0.1', port=12202), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False), GelfHttpHandler(host='127.0.0.1', port=12203), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False), GelfTlsHandler(host='127.0.0.1', port=12204), # GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem'), ]) def handler(request): return request.param def test_simple_message(logger): message = get_unique_message() graylog_response = log_warning(logger, message) assert graylog_response['message'] == message assert graylog_response['level'] == SYSLOG_LEVEL_WARNING assert 'full_message' not in graylog_response assert 'file' not in graylog_response assert 'module' not in graylog_response assert 'func' not in graylog_response
def main(args, loglevel): """ :param args: see main :param loglevel: :return: """ # config file loader config_dict = read_config(file=args.config) ### Loggers ### # graylog logger logging.basicConfig(level=loglevel) logger = logging.getLogger() logger.addHandler(GelfTlsHandler(host=config_dict['graylog_server'], port=config_dict['gralog_port'], include_extra_fields=True)) format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") # console logger not needed, GELF handler writes to console by default #ch = logging.StreamHandler(sys.stdout) #ch.setFormatter(format) #logger.addHandler(ch) fh = RotatingFileHandler('backlog.txt', maxBytes=(1048576 * 5), backupCount=7) fh.setFormatter(format) logger.addHandler(fh) ### Loggers END ### logger.info('Graylog site monitor activated with config file: {}'.format(args.config)) logger.debug('Contains {} websites'.format(len(config_dict['websites']))) for website in config_dict['websites']: web_domain = website['domain'] web_lookups = website['lookups'] logger.debug("Starting check of website: {}, has {} lookups".format(web_domain,len(web_lookups))) requests_session = init_session(website.get("login")) for counter, web_lookup in enumerate(web_lookups): web_url = web_lookup['url'] lookup_status, lookup_message = check_lookup(web_lookup, requests_session) lookup_description = web_lookup.get('description') if lookup_status == "OK": logger.debug("Lookup n.{}: OK, url: {}".format(counter, web_url), extra={'url': web_url, 'domain': web_domain, 'lookup_message': lookup_message}) elif lookup_status == "WARNING": logger.warning("Warning in {} (Desc:{}), {}, url: {}".format(web_domain,lookup_description,lookup_message, web_url), extra={'url':web_url,'domain': web_domain, 'lookup_message':lookup_message}) elif lookup_status == "ERROR": logger.error("Error in {} (Desc:{}), {}, url: {}".format(web_domain,lookup_description,lookup_message, web_url), extra={'url':web_url,'domain': web_domain, 'lookup_message':lookup_message}) else: logger.critical("Malfunction in {} (Desc:{}): lookup_status not recognized: {}, type:".format(web_url, lookup_description,type(lookup_status)), extra={'url': web_url, 'domain': web_domain, 'lookup_message': lookup_message}) requests_session.close()
class StreamToGelf: def __init__(self, stream, host='localhost', port=12201, protocol='udp', _filters=None, environment=None, json_only=False): signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop) self.buffer = deque() self.environment = str(environment) if _filters is not None: self.filters = _filters.split(',') else: self.filters = None self.json_only = bool(json_only) self.protocol = str(protocol) self.stream = stream self.handleropts = { 'host': str(host), 'port': int(port), 'protocol': str(protocol) } if str(protocol) == 'udp': self.gelfhandler = GelfUdpHandler(host=str(host), port=int(port), include_extra_fields=True, compress=True) elif str(protocol) == 'tcp': self.gelfhandler = GelfTcpHandler(host=str(host), port=int(port), include_extra_fields=True) elif str(protocol) == 'tls': self.gelfhandler = GelfTlsHandler(host=str(host), port=int(port), include_extra_fields=True) def _send_gelf(self): try: record = json_loads_byteified(''.join(self.buffer)) except ValueError as e: print "ERROR - cannot load json: " + str(e) else: try: assert 'MESSAGE' in record.keys() except AssertionError: pass else: if self.filters is not None: matchlist = [ record[filterkey] for filterkey in record.keys() if filterkey in unfilteredJournalctlKeys ] for filterelement in self.filters: for matchelement in matchlist: if re.search(str(filterelement), str(matchelement)): break else: continue break message = {'version': '1.1'} for key, value in record.iteritems(): if key == 'MESSAGE': # try to unnest and index json message keys, if the message is a valid json. try: innerrecord = json_loads_byteified(value) except AttributeError: if self.json_only: break else: message['short_message'] = value except ValueError: if self.json_only: break else: message['short_message'] = value except Exception as e: print(e) break else: try: assert 'message' in innerrecord.keys() except AssertionError: message['short_message'] = json.dumps( innerrecord) else: try: for hashkey, hashvalue in innerrecord.iteritems( ): if str(hashkey) == 'message': message[ 'short_message'] = hashvalue else: message['_' + str( hashkey).lower()] = hashvalue except Exception as e: print "EXCEPTION: " + str(e) elif key == '__REALTIME_TIMESTAMP': # convert from systemd's format of microseconds expressed as # an integer to graylog's float format, eg: "seconds.microseconds" message['timestamp'] = float(value) / (1000 * 1000) elif key == '_HOSTNAME': message['host'] = value else: if len(unfilteredJournalctlKeys) > 0 and str( key) in unfilteredJournalctlKeys: message['_' + str(key).lower()] = value if self.environment is not None: message['_environment'] = str(self.environment) else: try: if self.protocol != 'udp': self.gelfhandler.send( zlib.compress(json.dumps(message))) else: self.gelfhandler.send(json.dumps(message)) except Exception as e: print str(e) finally: self.buffer.clear() def run(self): for line in self.stream: line = line.strip() self.buffer.append(line) self._send_gelf() def stop(self, *args): self.buffer.clear() self.gelfhandler.flush() self.gelfhandler.close() print "Exiting..." sys.exit(0)
return True @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12201, include_extra_fields=True), GelfUdpHandler(host='127.0.0.1', port=12202, include_extra_fields=True), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, include_extra_fields=True), GelfHttpHandler(host='127.0.0.1', port=12203, include_extra_fields=True), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, include_extra_fields=True), GelfTlsHandler(host='127.0.0.1', port=12204, include_extra_fields=True), GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', include_extra_fields=True), ]) def handler(request): return request.param @pytest.yield_fixture def logger(handler): logger = logging.getLogger('test') dummy_filter = DummyFilter() logger.addFilter(dummy_filter)
def test_handler_certfile_required_for_keyfile(): with pytest.raises(ValueError): GelfTlsHandler(host='127.0.0.1', port=12001, keyfile="./somefile.key")
def test_tls_handler_init(): with pytest.raises(ValueError): GelfTlsHandler(host='127.0.0.1', port=12204, validate=True) with pytest.raises(ValueError): GelfTlsHandler(host='127.0.0.1', port=12204, keyfile='/dev/null')
# mongo.py from flask import Flask, render_template from flask import jsonify from flask import request from flask_pymongo import PyMongo from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler, GelfHttpHandler import logging app = Flask(__name__) logging.basicConfig(level=logging.INFO) logger = logging.getLogger() logger.addHandler(GelfTcpHandler(host='127.0.0.1', port=12201)) logger.addHandler(GelfUdpHandler(host='127.0.0.1', port=12201)) logger.addHandler(GelfTlsHandler(host='127.0.0.1', port=12201)) logger.addHandler(GelfHttpHandler(host='127.0.0.1', port=12201)) logger.info('hello gelf') app.config['MONGO_DBNAME'] = 'db_teste_itau1' app.config['MONGO_URI'] = 'mongodb://*****:*****@app.route('/tweets', methods=['GET']) def get_all_tweets(): teste4 = mongo.db.teste4 output = [] for s in teste4.find():
def setup_logging(root_log_level: Optional[str] = None, module_log_levels: Optional[Mapping[str, Union[str, int]]] = None, add_console_handler: bool = True, console_handler_format: Optional[str] = None): """ Setup log handlers for each application's namespace :param root_log_level: The log level all loggers use by default. (Default: `"WARNING"`) :param module_log_levels: A mapping of module names to their desired log levels. :param add_console_handler: If `True`, enable logging to stdout. (Default: `True`). :param console_handler_format: Specifies the format of stdout logs. (Default: `DEFAULT_LOG_FORMAT`). """ if root_log_level is None: root_log_level = "WARNING" root_logger = logging.getLogger() root_logger.setLevel(root_log_level) if module_log_levels: for module_name, log_level in module_log_levels.items(): logging.getLogger(module_name).setLevel(log_level) if add_console_handler: console_handler = logging.StreamHandler() console_handler.setFormatter( logging.Formatter( fmt=console_handler_format or DEFAULT_LOG_FORMAT)) root_logger.addHandler(console_handler) # Add GELF handler if GELF is enabled if "GRAYLOG_HOST" in os.environ: common_opts = dict(host=os.environ["GRAYLOG_HOST"], debug=bool(os.environ.get("GRAYLOG_DEBUG", True)), include_extra_fields=True, _app_name=os.environ["GRAYLOG_APP_NAME"], _env=os.environ["GRAYLOG_ENV"]) handler_type = os.environ.get("GRAYLOG_HANDLER_TYPE", "tls").lower() if handler_type == "udp": gelf_handler = GelfUdpHandler(**common_opts, port=int( os.environ.get( "GRAYLOG_UDP_PORT", 12201)), chunk_size=int( os.environ.get( "GRAYLOG_UDP_CHUNK_SIZE", 1300)), compress=bool( os.environ.get( "GRAYLOG_UDP_COMPRESS", True))) elif handler_type == "tls": gelf_handler = GelfTlsHandler(**common_opts, port=int( os.environ.get( "GRAYLOG_TLS_PORT", 12201)), timeout=float( os.environ.get( "GRAYLOG_TLS_TIMEOUT_SECS", 0.3))) else: raise ValueError( "Graylog handler type '{}' not recognized. Valid types are 'udp' and 'tls'." .format(handler_type)) root_logger.addHandler(gelf_handler) # Add datadog handler if log to datadog is enabled if "DATADOG_HOST" in os.environ: opts = dict(host=os.environ["DATADOG_HOST"], port=int(os.environ.get("DATADOG_UDP_PORT", 10518))) datadog_handler = DataDogUdpHandler(**opts) root_logger.addHandler(datadog_handler)
import pytest from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler from tests.helper import logger, get_unique_message, log_warning @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12201, debug=True), GelfUdpHandler(host='127.0.0.1', port=12202, debug=True), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, debug=True), GelfHttpHandler(host='127.0.0.1', port=12203, debug=True), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, debug=True), GelfTlsHandler(host='127.0.0.1', port=12204, debug=True), GelfTlsHandler(host='127.0.0.1', port=12204, debug=True, validate=True, ca_certs='tests/config/cert.pem'), ]) def handler(request): return request.param def test_debug_mode(logger): message = get_unique_message() graylog_response = log_warning(logger, message) assert graylog_response['message'] == message assert graylog_response['file'] == 'helper.py' assert graylog_response['module'] == 'helper' assert graylog_response['func'] == 'log_warning' assert graylog_response['logger_name'] == 'test' assert 'line' in graylog_response
STATIC_FIELDS = { '_ozzy': 'diary of a madman', '_van_halen': 1984, '_id': 42 } @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12201, **STATIC_FIELDS), GelfUdpHandler(host='127.0.0.1', port=12202, **STATIC_FIELDS), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, **STATIC_FIELDS), GelfHttpHandler(host='127.0.0.1', port=12203, **STATIC_FIELDS), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, **STATIC_FIELDS), GelfTlsHandler(host='127.0.0.1', port=12204, **STATIC_FIELDS), # GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', **STATIC_FIELDS), GelfTcpHandler(host='127.0.0.1', port=12201, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfUdpHandler(host='127.0.0.1', port=12202, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfHttpHandler(host='127.0.0.1', port=12203, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, static_fields=STATIC_FIELDS, _ozzy='billie jean'), GelfTlsHandler(host='127.0.0.1', port=12204, static_fields=STATIC_FIELDS), # GelfTlsHandler(host='127.0.0.1', port=12204, validate=True, ca_certs='tests/config/cert.pem', static_fields=STATIC_FIELDS, _ozzy='billie jean'), ]) def handler(request): return request.param def test_static_fields(logger): message = get_unique_message()
from pygelf import GelfTcpHandler, GelfUdpHandler, GelfTlsHandler import logging import json import pytest import mock def log_and_decode(_logger, _send, text): _logger.warning(text) message = _send.call_args[0][0].replace(b'\x00', b'').decode('utf-8') return json.loads(message) @pytest.mark.parametrize('handler,version', [ (GelfTcpHandler(host='127.0.0.1', port=12000, version='10.7'), '10.7'), (GelfUdpHandler( host='127.0.0.1', port=12000, compress=False, version='11.5'), '11.5'), (GelfTlsHandler(host='127.0.0.1', port=12000, version='12.3'), '12.3') ]) def test_custom_version(handler, version): logger = logging.getLogger('test') logger.addHandler(handler) with mock.patch.object(handler, 'send') as send: message = log_and_decode(logger, send, 'custom version') assert message['version'] == version logger.removeHandler(handler)
import pytest from pygelf import GelfTcpHandler, GelfUdpHandler, GelfHttpHandler, GelfTlsHandler from tests.helper import logger, get_unique_message, log_warning @pytest.fixture(params=[ GelfTcpHandler(host='127.0.0.1', port=12201, debug=True), GelfUdpHandler(host='127.0.0.1', port=12202, debug=True), GelfUdpHandler(host='127.0.0.1', port=12202, compress=False, debug=True), GelfHttpHandler(host='127.0.0.1', port=12203, debug=True), GelfHttpHandler(host='127.0.0.1', port=12203, compress=False, debug=True), GelfTlsHandler(host='127.0.0.1', port=12204, debug=True), # GelfTlsHandler(host='127.0.0.1', port=12204, debug=True, validate=True, ca_certs='tests/config/cert.pem'), ]) def handler(request): return request.param def test_debug_mode(logger): message = get_unique_message() graylog_response = log_warning(logger, message) assert graylog_response['message'] == message assert graylog_response['file'] == 'helper.py' assert graylog_response['module'] == 'helper' assert graylog_response['func'] == 'log_warning' assert graylog_response['logger_name'] == 'test' assert 'line' in graylog_response