def init_logger(): # init logger for Graylog logger = logging.getLogger() logger.setLevel(logging.INFO) handler = graypy.GELFUDPHandler(config.GRAYLOG_HOST, 12201) logger.addHandler(handler) return logger
def get_log(): my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) handler = graypy.GELFUDPHandler('localhost', 514, debugging_fields=False) my_logger.addHandler(handler) return my_logger
def _generate_loggers(self): info_logger = logging.getLogger(INFO_LOGGER) error_logger = logging.getLogger(ERROR_LOGGER) handler = graypy.GELFUDPHandler(os.getenv("GRAYLOG_HOST_NAME", "localhost"), int(os.getenv("GRAYLOG_PORT", 12201))) info_logger.addHandler(handler) error_logger.addHandler(handler) return info_logger, error_logger
def __init__(self, host='', port=''): host = host or settings.GELF_HOST port = port or settings.GELF_PORT self.gsk_logger = logging.getLogger() self.gsk_logger.setLevel(logging.DEBUG) udp_handler = graypy.GELFUDPHandler(host, int(port)) self.gsk_logger.addHandler(udp_handler) self.gsk_logger_adapter = logging.LoggerAdapter( logging.getLogger(), dict(self.fields))
def _add_graylog_handler(self) -> None: if not self.ip or not self.port or not graypy: if self.log_if_graylog_disabled: self.warning('Graylog not configured! Disabling it') return handler = graypy.GELFUDPHandler(self.ip, self.port, debugging_fields=False) self.logger.addHandler(handler)
def _add_graylog_handler(self, address: Optional[Tuple[str, int]], log_if_disabled: bool) -> None: if not graypy: if address: raise ValueError("Misconfiguration: Graylog configured but graypy not installed") return if not address: if log_if_disabled: self.warning("Graypy installed, but Graylog not configured! Disabling it") return handler = graypy.GELFUDPHandler(*address, debugging_fields=False) self._logger.addHandler(handler)
def main(): """Send a message to a given Graylog server and then check to see if the log was saved.""" args = parse_args() graylog_server = "http://" + args.server_ip if args.host_port: graylog_server += ':' + args.host_port graylog_server_api = graylog_server + '/api/' connection_type = args.connection_type.lower().strip() if connection_type not in ("udp", "tcp"): print("Connection type of " + connection_type + " is invalid. Valid options: UDP/TCP") exit(1) try: response_check = requests.get(graylog_server_api, auth=(args.username, args.password), headers={"accept": "application/json"}, timeout=2) if response_check.status_code != 200: logging.error("Received response code: %s", str(response_check.status_code)) exit(1) response_json = response_check.json() if not response_json['version']: logging.error("Failed to connect to the graylog server at: %s", graylog_server_api) exit(1) print("Connected to Graylog server at " + graylog_server + " with version " + response_json['version']) except requests.exceptions.ConnectionError: logging.error("Timeout connecting to the graylog server at: %s", graylog_server_api) exit(1) my_logger = logging.getLogger('graylog_logger') my_logger.setLevel(logging.DEBUG) if connection_type == "udp": graylog = graypy.GELFUDPHandler(args.server_ip) else: graylog = graypy.GELFTCPHandler(args.server_ip) my_logger.addHandler(graylog) print("Sending: " + args.message + " to the Graylog server!") jsondata = {"message": args.message} my_logger.debug(jsondata)
def init_app(self, app, config=None): """ Configure Graylog logger from a Flask application Available configuration options: GRAYLOG_HOST - the host to send messages to [default: 'localhost'] GRAYLOG_PORT - the port to send messages to [default: 12201] GRAYLOG_FACILITY - the facility to report with [default: 'flask'] GRAYLOG_EXTRA_FIELDS - whether or not to include `extra` fields from the message [default: True] GRAYLOG_ADD_DEBUG_FIELDS - whether extra python debug fields should be added to each message [default: True] GRAYLOG_CONFIGURE_MIDDLEWARE - whether to setup middleware to log each response [default: True] :param app: Flask application to configure this logger for :type app: flask.Flask :param config: An override config to use instead of `app.config` :type config: `dict` or `None` """ # Use the config they provided if config is not None: self.config = config # Use the apps config if `config` was not provided elif app is not None: self.config = app.config self.app = app # Setup default config settings self.config.setdefault('GRAYLOG_HOST', 'localhost') self.config.setdefault('GRAYLOG_PORT', 12201) self.config.setdefault('GRAYLOG_FACILITY', 'flask') self.config.setdefault('GRAYLOG_EXTRA_FIELDS', True) self.config.setdefault('GRAYLOG_ADD_DEBUG_FIELDS', True) self.config.setdefault('GRAYLOG_CONFIGURE_MIDDLEWARE', True) # Configure the logging handler and attach to this logger self.handler = graypy.GELFUDPHandler( host=self.config['GRAYLOG_HOST'], port=self.config['GRAYLOG_PORT'], facility=self.config['GRAYLOG_FACILITY'], extra_fields=self.config['GRAYLOG_EXTRA_FIELDS'], debugging_fields=self.config['GRAYLOG_ADD_DEBUG_FIELDS'], ) self.addHandler(self.handler) # Setup middleware if they asked for it if self.config['GRAYLOG_CONFIGURE_MIDDLEWARE']: self.setup_middleware()
def get_logger(self, logger_name): logger = logging.getLogger("graylog") logger.setLevel(logging.INFO) if self.graylog_ip: graylog_handler = graypy.GELFUDPHandler( self.graylog_ip, 12201, localname=SERVICE_NAME, ) logger.addHandler(graylog_handler) file_handler = logging.FileHandler( f"{self.docker_volume_path}/{logger_name}.log") logger.addHandler(file_handler) return logger
def _add_graylog_handler(self) -> None: if not graypy: if self.graylog_address: raise ValueError( "Misconfiguration: Graylog configured but graypy not installed" ) return if not self.graylog_address: if self.log_if_graylog_disabled: self.warning( "Graypy installed, but Graylog not configured! Disabling it" ) return handler = graypy.GELFUDPHandler(*self.graylog_address, debugging_fields=False) self.logger.addHandler(handler)
def setup_loggers(logger, *args, **kwargs): """Add graylog handler to logger.""" try: import graypy from csh_fantasy_bot.config import GELF_URL log.info(f'Gelf url: {GELF_URL}') if GELF_URL: handler = graypy.GELFUDPHandler(GELF_URL, 12201, facility='fantasy_bot_worker') logger.addHandler(handler) logger.setLevel(level=LOG_LEVEL) except ImportError: log.warn("Could not import graypy, using default logging") except KeyError: log.warn("Could not find gelf url, using default logging") finally: for k,v in logging.Logger.manager.loggerDict.items() : print('+ [%s] {%s} ' % (str.ljust( k, 20) , str(v.__class__)[8:-2]) ) if not isinstance(v, logging.PlaceHolder): for h in v.handlers: print(' +++',str(h.__class__)[8:-2] )
def start_logger(configuration, application_name=getproctitle()): logging.getLogger('pika').setLevel(logging.WARNING) logging.getLogger('kazoo.client').setLevel(logging.INFO) log_queue = Queue() queue_handler = QueueHandler(log_queue) graylog_handler = graypy.GELFUDPHandler(host=configuration['host'], port=configuration['port'], facility=application_name) remote_listener = QueueListener(log_queue, graylog_handler) logging.basicConfig(level=logging.INFO, handlers=[queue_handler]) remote_listener.start() logging.debug("Logging service started!") return remote_listener
def __init__(self): path = Path(dirname(__file__)) self.root_dir = join(path.parent, '.env') self.current_training_status = {} if self.root_dir is not None: # Create .env file path. print(self.root_dir) # Load file from the path. load_dotenv(self.root_dir) self.celery = Celery('exchange.machine.learning') self.config = Config() self.mi_host = os.getenv("MI_HOST") self.mi_port = os.getenv("MI_PORT") if os.getenv("MI_GRAYLOG_HOST") is not None: self.mi_graylog_host = os.getenv("MI_GRAYLOG_HOST") self.mi_graylog_port = os.getenv("MI_GRAYLOG_PORT") handler = graypy.GELFUDPHandler(self.mi_graylog_host, int(self.mi_graylog_port)) self.mi_logger = logging.getLogger('mi_logger') self.mi_logger.setLevel(logging.ERROR) self.mi_logger.setLevel(logging.INFO) self.mi_logger.addHandler(handler) if os.getenv("MI_BROKER_URL") is not None: self.mi_broker_url = os.getenv("MI_BROKER_URL") self.mi_result_backend = os.getenv("MI_RESULT_BACKEND") self.celery.conf.update(broker_url=self.mi_broker_url, result_backend=self.mi_result_backend, broker_use_ssl=False, authSource='celery', authMechanism="SCRAM-SHA-1", user='******', password='******', database_name='celery', taskmeta_collection='celery_taskmeta', groupmeta_collection='celery_groupmeta', max_pool_size=10, options=None) if os.getenv("MI_GRAYLOG_HOST") is not None: self.mi_logger.info('Exchange loading') super(Exchange, self).__init__()
def create_logger(log=None, config=None, handler_filter=None): config = config if config else {} app_name = config['APP_NAME'] if 'APP_NAME' in config and config[ 'APP_NAME'] != '' else None if log is None: log = logging.getLogger(app_name) log.handlers = [] if 'LOGSTASH' in config and config['LOGSTASH'] != "": try: port = 12201 host = config.get('LOGSTASH') if ":" in config["LOGSTASH"]: host = config["LOGSTASH"].split(":")[0] port = int(config["LOGSTASH"].split(":")[1]) handler = graypy.GELFUDPHandler(host, port) if handler_filter: handler.addFilter(handler_filter) log.addHandler(handler) log.info(f"Starting new logger on logstash {config['LOGSTASH']}") except Exception as err: log.handlers = [] handler = logging.StreamHandler() log.addHandler(handler) log.error( f"Failed logging to logstash {config['LOGSTASH']}, error {err} fall back to stdout" ) else: handler = logging.StreamHandler() log.addHandler(handler) if 'LOG_LEVEL' in config and config['LOG_LEVEL'] == 'DEBUG': log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) return log
def main(): env = os.environ try: host = env['GRAYLOG_SERVER'] port = int(env['GRAYLOG_PORT']) except KeyError: sys.exit("GRAYLOG_SERVER and GRAYLOG_PORT are required.") sys.stderr.write("Starting with host: %s, port: %d" % (host, port)) sys.stderr.flush() handler = graypy.GELFUDPHandler(host, port) level_match = re.compile(level_match_expr) for event_headers, event_data in supervisor_events(sys.stdin, sys.stdout): level, body = split_msg_and_get_log_level(event_data, level_match) event = logging.LogRecord( name=event_headers['processname'], level=level, pathname=None, lineno=0, msg=body, args=(), exc_info=None, ) # Set the processName to the name of the value of 'program:name' in the # supervisor config. event.processName = event_headers['groupname'] event.process = int( event_headers['pid']) if 'pid' in event_headers else 0 handler.handle(event)
import logging import sys import graypy import prometheus_client from proxy_pool import settings, app prometheus_client.start_http_server(int(settings.PROMETHEUS_PORT)) logging_handlers = [] if settings.GRAYLOG_GELFUDP_LOGGER is not None: graylog_handler = graypy.GELFUDPHandler( settings.GRAYLOG_GELFUDP_LOGGER['host'], int(settings.GRAYLOG_GELFUDP_LOGGER['port'])) logging_handlers.append(graylog_handler) if settings.CONSOLE_LOGGER: console_handler = logging.StreamHandler(sys.stdout) logging_handlers.append(console_handler) logging.basicConfig(level=int(settings.LOGGING_LEVEL), handlers=logging_handlers, format=settings.LOGGING_FORMAT) app.bootstrap()
import logging import sys import random import time import graypy my_logger = logging.getLogger(__name__) my_logger.setLevel(logging.DEBUG) handler = graypy.GELFUDPHandler('graylog', 12201) my_logger.addHandler(handler) my_logger.addHandler(logging.StreamHandler(stream=sys.stdout)) possible_logging_messages = [ lambda: my_logger.debug('Debug Graylog.'), lambda: my_logger.info('Info Graylog.'), lambda: my_logger.warning('Warning Graylog.'), lambda: my_logger.error('Error Graylog.'), lambda: my_logger.exception('Exception Graylog.'), ] def main(): while True: try: random.choice(possible_logging_messages)() b = random.randint(0, 3) print(3/b)
from werkzeug.exceptions import HTTPException, default_exceptions import app.base.QException as qex cwd = os.getcwd() templates = '/'.join((cwd, 'app/templates/')) app = Flask(__name__, template_folder=templates, static_url_path="") packs = [dataTemplate, dataFilter, dataFile, dataUser, dataQuery] [app.register_blueprint(pack) for pack in packs] def error_handling(error): if isinstance(error, HTTPException): result = {'code': error.code, 'description': error.description} else: description = default_exceptions[500].description result = {'code': 500, 'description': description} app.logger.exception(str(error), extra=result) return jsonify(result) for code in default_exceptions.keys(): app.register_error_handler(code, error_handling) if __name__ == '__main__': handler = graypy.GELFUDPHandler('localhost', 12201) app.logger.addHandler(handler) app.run()
import logging import os import graypy from flask import Flask graylog = logging.getLogger('graylog') graylog.setLevel(logging.INFO) handler = graypy.GELFUDPHandler(os.environ['GRAYLOG_IP'], 12201, localname="crawler_server") graylog.addHandler(handler) app = Flask(__name__) graylog.info('Pipeline test server running') @app.route('/') def info(): graylog.info('Pipeline test server called') return os.getenv('GRAYLOG_IP', 'variable not assigned')
## config, custom logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s [%(name)s] %(levelname)s: %(message)s') stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) config = configparser.ConfigParser() config.read(APP_CONFIG_FILEPATH) graylog_handler = graypy.GELFUDPHandler(config['Graylog']['host'], int(config['Graylog']['port'])) graylog_handler.setLevel(logging.ERROR) # set logging.ERROR later graylog_handler.setFormatter(formatter) logger.addHandler(stream_handler) logger.addHandler(graylog_handler) ## amazon.com related AMAZON_COM_ITEM_LINK_PATTERN = r'^(https?://www.amazon.com)?/([^/]+/[^/]+|dp)/([A-Z0-9]{10})(/.*$)?' ## amazon.ca related AMAZON_CA_ITEM_LINK_PATTERN = r'^(https?://www.amazon.ca)?/([^/]+/[^/]+|dp)/([A-Z0-9]{10})(/.*$)?' AMAZON_ITEM_LINK_FORMAT = 'https://www.{}/dp/{}{}' AMAZON_ITEM_VARIATION_LINK_POSTFIX = '/?th=1&psc=1' AMAZON_ITEM_IMAGE_CONVERT_PATTERN_FROM = r'\._([^_]+)_\.'
def init_app(self, app, config=None, extra=None): """ Configure Graylog logger from a Flask application Available configuration options: GRAYLOG_HOST - the host to send messages to [default: 'localhost'] GRAYLOG_PORT - the port to send messages to [default: 12201] GRAYLOG_FACILITY - the facility to report with [default: 'flask'] GRAYLOG_EXTRA_FIELDS - whether or not to include `extra` fields from the message [default: True] GRAYLOG_ADD_DEBUG_FIELDS - whether extra python debug fields should be added to each message [default: True] GRAYLOG_CONFIGURE_MIDDLEWARE - whether to setup middleware to log each response [default: True] :param app: Flask application to configure this logger for :type app: flask.Flask :param config: An override config to use instead of `app.config` :type config: `dict` or `None` :param extra: Additional Graylog fields included in messages :type extra: `dict` or `None` """ # Use the config they provided if config is not None: self.config = config # Use the apps config if `config` was not provided elif app is not None: self.config = app.config self.app = app # Setup default config settings self.config.setdefault("GRAYLOG_HOST", "localhost") self.config.setdefault("GRAYLOG_PORT", 12201) self.config.setdefault("GRAYLOG_FACILITY", "flask") self.config.setdefault("GRAYLOG_EXTRA_FIELDS", True) self.config.setdefault("GRAYLOG_ADD_DEBUG_FIELDS", True) self.config.setdefault("GRAYLOG_CONFIGURE_MIDDLEWARE", True) # Configure the logging handler and attach to this logger self.handler = graypy.GELFUDPHandler( host=self.config["GRAYLOG_HOST"], port=self.config["GRAYLOG_PORT"], facility=self.config["GRAYLOG_FACILITY"], extra_fields=self.config["GRAYLOG_EXTRA_FIELDS"], debugging_fields=self.config["GRAYLOG_ADD_DEBUG_FIELDS"], ) self.addHandler(self.handler) # Setup middleware if they asked for it if self.config["GRAYLOG_CONFIGURE_MIDDLEWARE"]: self.setup_middleware() # Additional Graylog fields self._static_extra = {} if extra is not None: self._static_extra = extra # Register custom log record factory to get those extra arguments # into records old_factory = logging.getLogRecordFactory() def record_factory(*args, **kwargs): record = old_factory(*args, **kwargs) for name, val, in self._static_extra.items(): setattr(record, name, val) return record logging.setLogRecordFactory(record_factory)
import logging import graypy graylog_server = os.environ.get("LOG_SERVER", "127.0.0.1") graylog_port = os.environ.get("LOG_PORT", "12201") my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) handler = graypy.GELFUDPHandler(graylog_server, graylog_port) my_logger.addHandler(handler) my_logger.debug('This is a test from a Python script!')
return Handler class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): """Handle requests in a separate thread.""" address_family = socket.AF_INET6 daemon_threads = True def handle_error(self, request, client_address): exc = sys.exc_info() print(f'[{client_address}] {exc[0].__name__}: {exc[1]}', file=sys.stderr) if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('--graylog-host', default='localhost') parser.add_argument('--graylog-port', type=int, default=12201) parser.add_argument('--listen-host', default='::') parser.add_argument('--listen-port', type=int, default=8080) parser.add_argument('--x-forwarded-for', action='store_true') args = parser.parse_args() gelf_handler = graypy.GELFUDPHandler(args.graylog_host, args.graylog_port) server = ThreadedHTTPServer( (args.listen_host, args.listen_port), get_http_request_handler(gelf_handler, args.x_forwarded_for), ) server.serve_forever()
def addGrayLogHandler(logger): server = UtilsConfig.get("Logging", "graylog_server") port = UtilsConfig.get("Logging", "graylog_port") if server is not None and port is not None: graylogHandler = graypy.GELFUDPHandler(server, int(port)) logger.addHandler(graylogHandler)
def getLogger(level=None): if level is None: level = UtilsConfig.get('Logging', 'level') if level is None: level = 'INFO' # Check if graylog handler already added: logger = logging.getLogger('edna2') hasGraylogHandler = False hasStreamHandler = False hasFileHandler = False for handler in logger.handlers: if isinstance(handler, graypy.GELFUDPHandler): hasGraylogHandler = True elif isinstance(handler, logging.handlers.RotatingFileHandler): hasFileHandler = True elif isinstance(handler, logging.StreamHandler): hasStreamHandler = True if not hasGraylogHandler: server = UtilsConfig.get('Logging', 'graylog_server') port = UtilsConfig.get('Logging', 'graylog_port') if server is not None and port is not None: graylogHandler = graypy.GELFUDPHandler(server, int(port)) logger.addHandler(graylogHandler) if not hasStreamHandler: streamHandler = logging.StreamHandler() logFileFormat = '%(asctime)s %(levelname)-8s %(message)s' formatter = logging.Formatter(logFileFormat) streamHandler.setFormatter(formatter) logger.addHandler(streamHandler) if not hasFileHandler: logPath = UtilsConfig.get('Logging', 'log_file_path') if logPath is not None: if not os.path.exists(os.path.dirname(logPath)): os.makedirs(os.path.dirname(logPath)) maxBytes = int(UtilsConfig.get('Logging', 'log_file_maxbytes', 1e6)) backupCount = int(UtilsConfig.get('Logging', 'log_file_backupCount', 10)) fileHandler = logging.handlers.RotatingFileHandler( logPath, maxBytes=maxBytes, backupCount=backupCount) logFileFormat = UtilsConfig.get('Logging', 'log_file_format') if logFileFormat is None: logFileFormat = '%(asctime)s %(levelname)-8s %(message)s' formatter = logging.Formatter(logFileFormat) fileHandler.setFormatter(formatter) logger.addHandler(fileHandler) # Set level if level == 'DEBUG': loggingLevel = logging.DEBUG elif level == 'INFO': loggingLevel = logging.INFO elif level == 'WARNING': loggingLevel = logging.WARNING elif level == 'ERROR': loggingLevel = logging.ERROR elif level == 'CRITICAL': loggingLevel = logging.CRITICAL elif level == 'FATAL': loggingLevel = logging.FATAL else: raise RuntimeError('Unknown logging level: "{0}"'.format(level)) logger.setLevel(loggingLevel) return logger
try: gelf_url = config['gelf-url'] except KeyError: gelf_url = None # if we have log configuration for log servers, # add that, otherwise let's use basic logging isLogConfigInfo = False # TODO should test setting the url, but not having graypy avail as lib if gelf_url is not None: logging_facility = os.getenv('LOGGING_FACILITY', 'lacrosse-default-facility') handler = graypy.GELFUDPHandler(gelf_url, 12201, localname='lacrosse-processing', facility=logging_facility) log.addHandler(handler) isLogConfigInfo = True if not isLogConfigInfo: logging.basicConfig(level=logging.INFO) logging_level = config['log-level'] or "INFO" log.setLevel(level=logging_level or logging.INFO) # used for dumping the dicts, probably could move over to MonitorMQTT class class SetEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, TempSensor):
import logging import graypy import pickle import re import sys import time from xmlrpc.client import ServerProxy, Error from .models.usagelog import UsageLog from pyramid import threadlocal logging.basicConfig(filename='webhook.log', level=logging.DEBUG) graylogger = logging.getLogger('gray_logger') graylogger.setLevel(logging.DEBUG) handler = graypy.GELFUDPHandler('5.9.19.231', 5090) graylogger.addHandler(handler) log = logging.getLogger(__name__) registry = threadlocal.get_current_registry() def from_hex(mystr): print(mystr) try: return bytes.fromhex(mystr).decode('utf-8') except TypeError: return "Unregistered Carrier" except ValueError: return "Unregistered Carrier"
import hmac import os.path import subprocess import threading import graypy import logging import socket from flask import Flask, request, Response myLogger = logging.getLogger('webhook-listener') myLogger.setLevel(logging.DEBUG) grayhandler = graypy.GELFUDPHandler('ncpfast-logs.edc.renci.org', 12201) myLogger.addHandler(grayhandler) # handler = logging.StreamHandler(sys.stdout) # handler.setLevel(logging.DEBUG) # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # handler.setFormatter(formatter) # myLogger.addHandler(handler) application = Flask(__name__) application.config['DEBUG'] = True secretToken = os.getenv('WEBHOOK_TOKEN', 'change_me___preferably_set_in_a_.env_file') queryToken = os.getenv('QUERY_TOKEN', 'change_me___preferably_set_in_a_.env_file') myLogger.debug('Token: ' + secretToken + ' Type: ' + str(type(secretToken)))