def initialize_logging(): kafkalogger = logging.getLogger("kafka") kafkalogger.addHandler(logging.StreamHandler(sys.stdout)) kafkalogger.setLevel(os.getenv("LOG_LEVEL", "INFO")) if any("KUBERNETES" in k for k in os.environ): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(LogstashFormatterV1()) logging.root.setLevel(os.getenv("LOG_LEVEL", "INFO")) logging.root.addHandler(handler) else: logging.basicConfig( level="INFO", format="%(threadName)s %(levelname)s %(name)s - %(message)s" ) logger = logging.getLogger(app_config.get_app_name()) if config.aws_access_key_id and config.aws_secret_access_key: logger.warn("configuring cloudwatch logging") config_cloudwatch(logger) logger.warn("cloudwatch logging ENABLED") else: logger.warn("cloudwatch logging DISABLED") return logger
def detectQR(image_name): host = '127.0.0.1' test_logger = logging.getLogger() handler = logging.StreamHandler() formatter = LogstashFormatterV1() handler.setFormatter(formatter) test_logger.setLevel(logging.INFO) test_logger.addHandler(logstash.LogstashHandler(host, 5000, version=1)) script_path = os.getcwd() detect_result = os.popen("java -jar QRDetect.jar %s" % image_name) csv = detect_result.read().split(',') timestamp = image_name.split('_')[1].split('.')[0] if (len(csv) == 2): print("NO QR DETECETED") else: for i in range(0, int(len(csv) / 3)): data = { 'cartID': int(csv[i + 1]), 'camID': int(csv[0]), 'x': int(csv[i + 2]), 'y': int(csv[i + 3]), 'time': int(timestamp), } #data_json = json.dumps(data, indent = 2) test_logger.info('python-logstash: test extra fields', extra=data)
def init_logger(): logger = logging.getLogger() handler = logging.StreamHandler() formatter = LogstashFormatterV1() handler.setFormatter(formatter) logger.addHandler(handler) return logger
def configure_logging(service_name): """ Configure logging based on the settings in the settings file. This sets up a handler for each logging mode that is enabled. See `microservice.core.settings.LoggingMode` for the supported logging types. :param str service_name: Name of the service being served by this instance. """ logger = logging.getLogger() logger.setLevel(settings.logging_level) formatter_kwargs = { 'fmt': json.dumps({'extra': { 'local_service': service_name, # Basic way to distinguish logs between instances of the same microservice. 'instance_id': random.randint(100000, 999999) }}) } formatter = LogstashFormatterV1(**formatter_kwargs) if settings.LoggingMode.FILE in settings.logging_modes: file_handler = logging.FileHandler('{}.log'.format(service_name)) file_handler.setFormatter(formatter) file_handler.addFilter(RequestIDLogFilter()) logger.addHandler(file_handler) if settings.LoggingMode.HUMAN in settings.logging_modes: stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(HumanReadableLogstashFormatter(**formatter_kwargs)) stdout_handler.addFilter(RequestIDLogFilter()) logger.addHandler(stdout_handler) if settings.LoggingMode.STDOUT in settings.logging_modes: stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(formatter) stdout_handler.addFilter(RequestIDLogFilter()) logger.addHandler(stdout_handler) if settings.LoggingMode.LOGSTASH in settings.logging_modes: # TODO: test this raise Exception("Warning: untested") logstash_handler = AsynchronousLogstashHandler( **settings.logstash_settings) logstash_handler.setFormatter(formatter) logstash_handler.addFilter(RequestIDLogFilter()) logger.addHandler(logstash_handler) if settings.LoggingMode.FLUENTD in settings.logging_modes: # TODO: test this raise Exception("Warning: untested") fluentd_handler = handler.FluentHandler( 'pycroservices.follow', **settings.fluentd_settings, buffer_overflow_handler=overflow_handler) fluentd_handler.setFormatter(formatter) fluentd_handler.addFilter(RequestIDLogFilter()) logger.addHandler(fluentd_handler)
def config_cloudwatch(logger): CW_SESSION = Session(aws_access_key_id=config.AWS_ACCESS_KEY_ID, aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY, region_name=config.AWS_REGION_NAME) cw_handler = watchtower.CloudWatchLogHandler(boto3_session=CW_SESSION, log_group=config.LOG_GROUP, stream_name=config.HOSTNAME) cw_handler.setFormatter(LogstashFormatterV1()) logger.addHandler(cw_handler)
def initialize_logging(): kafkalogger = logging.getLogger("kafka") kafkalogger.setLevel("ERROR") if any("OPENSHIFT" in k for k in os.environ): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(LogstashFormatterV1()) logging.root.setLevel(os.getenv("LOG_LEVEL", "INFO")) logging.root.addHandler(handler) else: logging.basicConfig( level=config.LOG_LEVEL, format="%(threadName)s %(levelname)s %(name)s - %(message)s", ) if os.environ.get("ACG_CONFIG"): f = clowder_config else: f = non_clowder_config aws_access_key_id, aws_secret_access_key, aws_region_name, aws_log_group, create_log_group = f( ) if all((aws_access_key_id, aws_secret_access_key, aws_region_name, aws_log_group)): from boto3.session import Session import watchtower boto3_session = Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name) cw_handler = watchtower.CloudWatchLogHandler( boto3_session=boto3_session, log_group=aws_log_group, stream_name=socket.gethostname(), create_log_group=create_log_group) cw_handler.setFormatter(LogstashFormatterV1()) logging.root.addHandler(cw_handler) logger = logging.getLogger(config.APP_NAME) return logger
def calculate(things_to_buy_count, request_data, customer_id, coupons_list, final_payment_amount, nocoupon_payment_amount): host = '127.0.0.1' test_logger = logging.getLogger() handler = logging.StreamHandler() formatter = LogstashFormatterV1() handler.setFormatter(formatter) test_logger.setLevel(logging.INFO) test_logger.addHandler(logstash.LogstashHandler(host, 5000, version=1)) i = 0 while i < things_to_buy_count: item_serial = str(request_data['serial' + str(i + 1)]) try: print(item_serial) item_ob_info = Item_Info.objects.get(serial_num=item_serial) item_ob = Item_Info.objects.get(serial_num=item_serial).item except Item_Info.DoesNotExist: print('Invalid Item Serial\n') return HttpResponse('Invalid Item Serial\n') print(customer_id) logdata = { 'serial': item_serial, } data = Pur_History(customer=customer_id, item=item_ob_info) print(data) data.save() test_logger.info('python-logstash: test extra fields', extra=logdata) coupon_use_checker = False for coupon in coupons_list: if coupon.coupon_item.item == item_ob and coupon.coupon_use == None: Coupon_Item_Info.objects.filter( serial_num=coupon.serial_num).update(coupon_use=True) final_payment_amount = final_payment_amount + int( (float(item_ob.price) * (float(1) - float(coupon.coupon_item.discount_rate) / float(100)))) print( int((float(item_ob.price) * (float(1) - float(coupon.coupon_item.discount_rate) / 100)))) print('coupon adjust') coupon_use_checker = True break if coupon_use_checker == False: print(item_ob.price) print('coupon not adjust') final_payment_amount = final_payment_amount + item_ob.price nocoupon_payment_amount = nocoupon_payment_amount + item_ob.price i = i + 1 return final_payment_amount
def detectQR(image_name): host = '127.0.0.1' test_logger = logging.getLogger() handler = logging.StreamHandler() formatter = LogstashFormatterV1() handler.setFormatter(formatter) test_logger.setLevel(logging.INFO) test_logger.addHandler(logstash.LogstashHandler(host, 5000, version=1)) #image_name = '1527589403.6_1.jpg'; script_path = os.getcwd() detect_result = os.popen("java -jar QRDetect.jar %s" % image_name) csv = detect_result.read().split(',') #timestamp = image_name.split('_')[1].split('.')[0] timestamp = image_name.split('_')[0] if (len(csv) == 2): print("NO QR DETECETED") print(csv) else: print("QR DETECTED############################") print(csv) for i in range(0, int(len(csv) / 3)): logdata = { 'cartID': int(csv[i + 1]), 'camID': int(csv[0]), 'x': int(csv[i + 2]), 'y': int(csv[i + 3]), 'time': int(timestamp), } #data_json = json.dumps(data, indent = 2) time_num = int(timestamp) #serial = 'cart128644' serial = str(int(csv[i + 1])) camera_num = int(csv[0]) coor_x = int(csv[i + 2]) coor_y = int(csv[i + 3]) cart_customer = Cart_Info.objects.get(serial_num=serial).owner camera = Camera_Info.objects.get(num=camera_num) data = Mv_History(time=time_num, customer=cart_customer, camera_num=camera, x=coor_x, y=coor_y) data.save() #receive_cartqrcode(serial,camera_num,coor_x,coor_y) print("DB SAVED") test_logger.info('python-logstash: test extra fields', extra=logdata) print("LOG SAVED")
def config_cloudwatch(logger): CW_SESSION = Session( aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key, region_name=config.aws_region_name, ) cw_handler = watchtower.CloudWatchLogHandler( boto3_session=CW_SESSION, log_group=config.log_group, stream_name=config.namespace, ) cw_handler.setFormatter(LogstashFormatterV1()) logger.addHandler(cw_handler)
def create_logger() -> logging.Logger: """Set up logstash formatted logger. Returns: logging.Logger: Python logger for __name__ """ logger = logging.getLogger(__name__) handler = logging.StreamHandler(stream=sys.stdout) handler.setFormatter(LogstashFormatterV1()) logger.setLevel(logging.INFO) logger.handlers = [handler] return logger
def initialize_logging(): kafkalogger = logging.getLogger("kafka") kafkalogger.setLevel("ERROR") if any("KUBERNETES" in k for k in os.environ): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(LogstashFormatterV1()) logging.root.setLevel(os.getenv("LOG_LEVEL", "INFO")) logging.root.addHandler(handler) else: logging.basicConfig( level=config.LOG_LEVEL, format="%(threadName)s %(levelname)s %(name)s - %(message)s") logger = logging.getLogger(config.APP_NAME) if (config.AWS_ACCESS_KEY_ID and config.AWS_SECRET_ACCESS_KEY): config_cloudwatch(logger) return logger
def init(logstash_type, level=logging.DEBUG, logpath=None, logger_name=None, extra_fields=None): """ Initializes and returns configured LogstashFormatter logger :param logstash_type: required extra field used for logstash configuration to Kafka topic output :type logstash_type: str :param level: defaulted to DEBUG :type level: int :param logpath: optional, filepath that the log will be written to, prints to stdout if logpath=None :type logpath: str :param logger_name: name of logger to be configured :type logger_name: str :param extra_fields: provide extra fields to be always present in logs :type extra_fields: dict """ if logpath: handler = logging.FileHandler(logpath) else: handler = logging.StreamHandler(sys.stdout) # since we require that type is declared, it is always defined as an extra field fmt = {'extra': {'type': logstash_type}} # if additional extra fields are defined they are appended if extra_fields: if not isinstance(extra_fields, dict): raise Exception('extra_fields must be of type dict') fmt['extra'].update(extra_fields) handler.setFormatter(LogstashFormatterV1(fmt=json.dumps(fmt))) handler.setLevel(level) logger = logging.getLogger(logger_name) logger.addHandler(handler) logger.setLevel(level) return logger
import sys import json import logging from logstash_formatter import LogstashFormatterV1 with open('/home/wenbo/Desktop/Tap-News/config/config.json') as config_file: config = json.load(config_file) SERVER_HOST = config['backend_server']['service']['SERVER_HOST'] SERVER_PORT = config['backend_server']['service']['SERVER_PORT'] # Initialize logger logger = logging.getLogger('backend-server') logger.setLevel(logging.INFO) logFile = logging.FileHandler('main.log') formatter = LogstashFormatterV1() logFile.setFormatter(formatter) logger.addHandler(logFile) class RequestHandler(pyjsonrpc.HttpRequestHandler): """ Test Method """ @pyjsonrpc.rpcmethod def add(self, a, b): print "add is called with %d and %d" % (a, b) return a + b """ Get news summaries for a user """ @pyjsonrpc.rpcmethod def getNewsSummariesForUser(self, user_id, page_num): logger.info('get news summaries for user')
from logstash_formatter import LogstashFormatterV1 from concurrent.futures import ThreadPoolExecutor from aiokafka import AIOKafkaConsumer, AIOKafkaProducer from kafka.errors import KafkaError from kafkahelpers import ReconnectingClient from prometheus_async.aio import time from boto3.session import Session from pup.utils import mnm, configuration from pup.utils.fact_extract import extract_facts from pup.utils.get_commit_date import get_commit_date # Logging if any("KUBERNETES" in k for k in os.environ): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(LogstashFormatterV1()) logging.root.setLevel(os.getenv("LOGLEVEL", "INFO")) logging.root.addHandler(handler) else: logging.basicConfig( level=os.getenv("LOGLEVEL", "INFO"), format="%(threadName)s %(levelname)s %(name)s - %(message)s") logger = logging.getLogger('advisor-pup') try: with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as f: NAMESPACE = f.read() except EnvironmentError: logger.info('Not Running on Openshift')
# place. I kept this for reference, but I prefer the refactored code in # logsetup which makes it more readable and re-usable. import logging from logging import StreamHandler from logstash_formatter import LogstashFormatterV1 from logstash_async.handler import AsynchronousLogstashHandler import sys import os # most of this is from the python logstash example: # https://pypi.python.org/pypi/python-logstash host = os.getenv('LOGSTASH_HOST', 'localhost') port = int(os.getenv('LOGSTASH_PORT', 5959)) formatter = LogstashFormatterV1(fmt='{"extra": {"appname": "lstest"}}') handler = AsynchronousLogstashHandler(host, port, database_path='logstash.db') handler.setFormatter(formatter) test_logger = logging.getLogger('python-logstash-logger') test_logger.setLevel(logging.INFO) test_logger.addHandler(handler) test_logger.addHandler(StreamHandler(stream=sys.stdout)) # this part is from SO: # https://stackoverflow.com/questions/6234405/logging-uncaught-exceptions-in-python def handle_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback)
def main(): signal.signal(signal.SIGTERM, signal_handler) parser = argparse.ArgumentParser( description='Export ES query results to Prometheus.') parser.add_argument( '-e', '--es-cluster', default='localhost', help= 'addresses of nodes in a Elasticsearch cluster to run queries on. Nodes should be separated by commas e.g. es1,es2. Ports can be provided if non-standard (9200) e.g. es1:9999 (default: localhost)' ) parser.add_argument( '-p', '--port', type=int, default=8080, help='port to serve the metrics endpoint on. (default: 8080)') parser.add_argument( '--query-disable', action='store_true', help= 'disable query monitoring. Config file does not need to be present if query monitoring is disabled.' ) parser.add_argument( '-c', '--config-file', default='exporter.cfg', help= 'path to query config file. Can be absolute, or relative to the current working directory. (default: exporter.cfg)' ) parser.add_argument('--cluster-health-disable', action='store_true', help='disable cluster health monitoring.') parser.add_argument( '--cluster-health-interval', type=float, default=10, help= 'polling interval for cluster health monitoring in seconds. (default: 10)' ) parser.add_argument( '--cluster-health-level', default='indices', choices=['cluster', 'indices', 'shards'], help= 'level of detail for cluster health monitoring. (default: indices)') parser.add_argument('--nodes-stats-disable', action='store_true', help='disable nodes stats monitoring.') parser.add_argument( '--nodes-stats-interval', type=float, default=10, help= 'polling interval for nodes stats monitoring in seconds. (default: 10)' ) parser.add_argument('--indices-stats-disable', action='store_true', help='disable indices stats monitoring.') parser.add_argument( '--indices-stats-interval', type=float, default=10, help= 'polling interval for indices stats monitoring in seconds. (default: 10)' ) parser.add_argument( '--indices-stats-mode', default='cluster', choices=['cluster', 'indices'], help='detail mode for indices stats monitoring. (default: cluster)') parser.add_argument('-j', '--json-logging', action='store_true', help='turn on json logging.') parser.add_argument('-v', '--verbose', action='store_true', help='turn on verbose logging.') args = parser.parse_args() log_handler = logging.StreamHandler() log_format = '[%(asctime)s] %(name)s.%(levelname)s %(threadName)s %(message)s' formatter = LogstashFormatterV1( ) if args.json_logging else logging.Formatter(log_format) log_handler.setFormatter(formatter) logging.basicConfig(handlers=[log_handler], level=logging.DEBUG if args.verbose else logging.INFO) logging.captureWarnings(True) port = args.port es_cluster = args.es_cluster.split(',') es_client = Elasticsearch(es_cluster, verify_certs=False) scheduler = sched.scheduler() if not args.query_disable: config = configparser.ConfigParser() config.read_file(open(args.config_file)) query_prefix = 'query_' queries = {} for section in config.sections(): if section.startswith(query_prefix): query_name = section[len(query_prefix):] query_interval = config.getfloat(section, 'QueryIntervalSecs') query_indices = config.get(section, 'QueryIndices', fallback='_all') query = json.loads(config.get(section, 'QueryJson')) queries[query_name] = (query_interval, query_indices, query) if queries: for name, (interval, indices, query) in queries.items(): func = partial(run_query, es_client, name, indices, query) run_scheduler(scheduler, interval, func) else: logging.warn('No queries found in config file %s', args.config_file) if not args.cluster_health_disable: cluster_health_func = partial(get_cluster_health, es_client, args.cluster_health_level) run_scheduler(scheduler, args.cluster_health_interval, cluster_health_func) if not args.nodes_stats_disable: nodes_stats_func = partial(get_nodes_stats, es_client) run_scheduler(scheduler, args.nodes_stats_interval, nodes_stats_func) if not args.indices_stats_disable: parse_indices = args.indices_stats_mode == 'indices' indices_stats_func = partial(get_indices_stats, es_client, parse_indices) run_scheduler(scheduler, args.indices_stats_interval, indices_stats_func) logging.info('Starting server...') start_http_server(port) logging.info('Server started on port %s', port) try: scheduler.run() except KeyboardInterrupt: pass shutdown()
defaults={ 'count': settings.LOG_BACKUP_COUNT, 'db_dump': settings.DB_BACKUP_FILE, 'db_setup': settings.DB_BACKUP_FILE + '.db_setup', 'log_file': join(LOG_DIR, 'bank.log'), 'when': settings.LOG_ROTATE_WHEN, }) if settings.LOG_SQL_STATEMENTS: logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) # TODO Explore and add Tornado logs to the log file as well logger = logging.getLogger('app') handler = logging.FileHandler(join(LOG_DIR, 'bank.log.json')) formatter = LogstashFormatterV1(fmt=json.dumps( {'extra': { 'app': settings.APP_NAME, 'env': settings.ENVIRONMENT }})) handler.setFormatter(formatter) logger.addHandler(handler) # db dump logger db_dump_logger = logging.getLogger('dbdump') # db setup logger db_logger = logging.getLogger('dbsetupdump')
def get_formatter(extra={}): return LogstashFormatterV1(fmt=json.dumps({"extra": extra}))
logger = logging.getLogger('orator.connection.queries') logger.setLevel(logging.DEBUG) logging.warning('Orator query log started') # Log handlers = [ logging.handlers.RotatingFileHandler( "%s/%s" % (app.root_path, app.config['LOG_RESOURCE_PATH']), encoding=app.config['BLUEPRINT']['LOGS']['FILE_ENCODING'], maxBytes=int(app.config['BLUEPRINT']['LOGS']['FILE_MAX_BYTES']), backupCount=int(app.config['BLUEPRINT']['LOGS']['FILE_BACKUP_COUNT'])), logging.StreamHandler() ] fmt = {'extra': {'type': app.config['APP_NAME'] + '-local'}} jfmt = json.dumps(fmt) logsts_formatter = LogstashFormatterV1(fmt=jfmt, datefmt="%Y-%m-%d %H:%M:%S") app.logger.setLevel(logging.DEBUG) for h in handlers: h.setFormatter(logsts_formatter) app.logger.addHandler(h) # Exception Handler # Invalid response @app.errorhandler(ConnectionTimeoutException) def handle_exception(error): data = error.get_body() logRequest(data, error.get_code()) return jsonify(data), error.get_code()
def setup_logging(app, config): handler = logging.StreamHandler(sys.stdout) handler.setFormatter(LogstashFormatterV1()) app.logger.addHandler(handler)