def get_json_log_handler(path, app_name, json_fields):
handler = logging.FileHandler(path)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['collector', app_name]
formatter.defaults['@fields'] = json_fields
handler.setFormatter(formatter)
return handler
def get_json_log_handler(path, app_name):
handler = RotatingFileHandler(
path, maxBytes=1024 * 1024 * 10, backupCount=5)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['application', app_name]
handler.setFormatter(formatter)
return handler
def get_json_log_handler(path, app_name, json_fields):
handler = logging.FileHandler(path)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['collector', app_name]
formatter.defaults['@fields'] = json_fields
handler.setFormatter(formatter)
return handler
def get_json_log_handler(path, app_name, json_fields):
handler = ConcurrentRotatingFileHandler(
path, "a", 2 * 1024 * 1024 * 1024, 1)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['collector', app_name]
formatter.defaults['@fields'] = json_fields
handler.setFormatter(formatter)
return handler
def get_json_log_handler(path, app_name, json_fields):
handler = ConcurrentRotatingFileHandler(path, "a", 2 * 1024 * 1024 * 1024,
1)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['collector', app_name]
formatter.defaults['@fields'] = json_fields
handler.setFormatter(formatter)
return handler
def create_logger(name,
filehandler_config=None,
environment='',
stream_config=None,
level=None):
if name in _log_registy:
return _log_registy[name]
if level is None:
level = LOGSTASH
with _log_lock:
logger = logging.getLogger(name)
logger.environment = sanitize_name(environment)
logger.setLevel(level)
logstash_formatter = LogstashFormatter()
if filehandler_config:
fh_handler = logging.handlers.RotatingFileHandler(
**filehandler_config)
fh_handler.setLevel(level)
fh_handler.setFormatter(logstash_formatter)
logger.addHandler(fh_handler)
if stream_config:
stream_handler = logging.StreamHandler(
stream=stream_config.get('stream'))
stream_handler.setLevel(stream_config.get('level', level))
stream_formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
stream_handler.setFormatter(stream_formatter)
logger.addHandler(stream_handler)
_log_registy[name] = logger
return logger
REDIS=redis.StrictRedis(**flask_app.config['REDIS_SETTINGS']),
MONGO=pymongo.MongoClient(**flask_app.config['MONGODB_SETTINGS']),
CELERY_BROKER_URL='redis://%(host)s:%(port)i/%(db)i' %
flask_app.config['REDIS_SETTINGS'])
celery = make_celery(flask_app)
def environment():
return os.getenv("GOVUK_ENV", "development")
handler = logging.FileHandler("log/%s.json.log" % environment())
logging.worker_hijack_root_logger = False
formatter = LogstashFormatter()
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
flask_app.logger.addHandler(handler)
flask_app.logger.setLevel(logging.INFO)
if flask_app.debug:
handler = logging.FileHandler("log/development.log")
handler.setLevel(logging.DEBUG)
flask_app.logger.addHandler(handler)
flask_app.logger.setLevel(logging.DEBUG)
def logstasher_request(request):
"""Returns the REQUEST line for logstasher"""
def get_json_log_handler(path):
handler = FileHandler(path)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['application']
handler.setFormatter(formatter)
return handler
import time
import json
from datetime import datetime
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
from elasticsearch_dsl import F, Q
from testing.clients import guid_lookup_table
from testing.testing import Inventory
from logstash_formatter import LogstashFormatter
# from base import Inventory
es_endpoint = '%s:9200' % Inventory().es
# es_endpoint = '54.152.5.133:9200' # FIXME speedup hack
client = Elasticsearch(es_endpoint)
lsformatter = LogstashFormatter(defaults=dict())
es_index_name = 'logstash-%s' % datetime.utcnow().strftime('%Y.%m.%d')
es_doc_type = 'ethlog'
def pprint(x):
print json.dumps(x.to_dict(), indent=2)
def ip_from_guid(guid):
return guid_lookup_table[guid]['guid_short'] + ' @ ' + guid_lookup_table[
guid]['ip'] + '/' + guid_lookup_table[guid]['impl']
def time_range_filter(field="@timestamp", offset=60):
start_time = datetime.utcfromtimestamp(time.time() - offset).strftime(
sys.exit(1)
es_endpoint = sys.argv[1]
extra = dict(x.split(',') for x in sys.argv[2:])
# ES
es = elasticsearch.Elasticsearch(es_endpoint)
# logstash-YYYY.MM.DD
es_index_name = 'logstash-%s' % datetime.datetime.utcnow().strftime('%Y.%m.%d')
es_doc_type = 'ethlog'
def es_log(doc):
es.create(index=es_index_name, doc_type=es_doc_type, body=doc)
lsformatter = LogstashFormatter(defaults=extra)
while True:
l = sys.stdin.readline().strip()
if not l:
continue
# workaround for multi-line jsons from cpp
if l == '{':
ml = l
while True:
l = sys.stdin.readline()
# workaround for wrong time format in cpp, add subseconds and Z
if '\"ts\"' in l and not 'Z\"' in l:
l = l[:-2] + '.000000Z\"'
def get_json_log_handler(path, app_name):
handler = FileHandler(path)
formatter = LogstashFormatter()
formatter.defaults['@tags'] = ['application', app_name]
handler.setFormatter(formatter)
return handler
import falcon
import os
import socket
import json
import logging
from logstash_formatter import LogstashFormatter
logger = logging.getLogger('cf-env')
logger.setLevel(logging.INFO)
ADDR = str(os.environ['SYSLOG_URL'])
PORT = int(os.environ['SYSLOG_PORT'])
handler = logging.handlers.SysLogHandler(address=(ADDR, PORT))
formatter = LogstashFormatter()
handler.setFormatter(formatter)
logger.addHandler(handler)
class EnvResources:
def on_get(self, req, resp):
"""
This get will show yo the env vars that will be logged to the stdout
and stderr
"""
resp.status = falcon.HTTP_200
msg = """Hey I'm running at: %s:%s""" \
% (self.get_host(), self.get_port())
resp.body = msg
env_vars = self.get_vcap_env()
logger.info("Test message",
extra={
import logging
#import logging.handlers
import logging.handlers as handlers
#import logstash
from logstash_formatter import LogstashFormatter
logger = logging.getLogger(__name__)
# Set logging level across the logger. Set to INFO in production
logger.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
formatter1 = LogstashFormatter()
# LogstashLog Handler
#logstash_handler = logstash.LogstashHandler("54.69.130.37",9299, version=1)
#logstash_handler.setFormatter(formatter1)
#Local File Handler
#create file handler which logs even debug messages
file_handler = logging.FileHandler(__name__ + '_file.log')
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
# create console handler with debug level
# This should be changed to ERROR in production
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.DEBUG)
console_handler.setFormatter(formatter)
#adding Rotating file handler
def get_logger(logger_name=DEFAULT_LOGGER_NAME,
log_level=DEFAULT_LOG_LEVEL,
enable_machine_log_file=False,
enable_human_log_file=False,
enable_stream_log=True,
ignore_log_failures=True,
machine_log_file_path=None,
human_log_file_path=None,
human_log_format=DEFAULT_HUMAN_LOG_FORMAT,
logstash_host=None,
logstash_port=None,
enable_logstash_log=False,
**kwargs):
# IMPORTANT NOTE:
# depending on environment settings, log failure may be silently ignored.
# typically, this is done when log failure should not fail the
# parent process
try:
lgr = logging.getLogger(logger_name)
# adding logger handlers only if no handlers exist.
# the logger object acts as a singleton by nature,
# but handlers may be unintentionally duplicated
if not len(lgr.handlers):
lgr.setLevel(int(log_level))
# add stream handler if enabled
if str(enable_stream_log).lower() == 'true':
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(
logging.Formatter(human_log_format))
lgr.addHandler(stream_handler)
# add human readable log file handler
if str(enable_human_log_file).lower() == 'true':
file_handler = logging.FileHandler(human_log_file_path)
file_handler.setFormatter(logging.Formatter(human_log_format))
lgr.addHandler(file_handler)
# add machine log handler
if str(enable_machine_log_file).lower() == 'true':
from logstash_formatter import LogstashFormatter
file_handler = logging.FileHandler(machine_log_file_path)
file_handler.setFormatter(LogstashFormatter())
lgr.addHandler(file_handler)
# add LogStash handler
if str(enable_logstash_log).lower() == 'true':
from logstash import LogstashHandler
# from logstash_formatter import LogstashFormatter
logstash_handler = LogstashHandler(logstash_host,
int(logstash_port),
version=1)
# logstash_handler.setFormatter(LogstashFormatter())
lgr.addHandler(logstash_handler)
return lgr
except:
if str(ignore_log_failures).lower() == 'true':
pass
else:
raise
from logstash_formatter import LogstashFormatter
from prometheus_client import REGISTRY, generate_latest, Summary
from spacy.symbols import ENT_TYPE, TAG, DEP
import spacy.about
import spacy.util
from .parse import Parse, Entities, Sentences, SentencesDependencies
logging.basicConfig(
level=logging.DEBUG,
format="[%(asctime)s] [%(process)d] [%(levelname)s] %(message)s")
log = logging.getLogger()
if os.getenv('REWE_BDP_STAGE') is not None:
loghandler = logging.StreamHandler()
loghandler.setFormatter(LogstashFormatter())
log.handlers = []
log.addHandler(loghandler)
MODELS = os.getenv("languages", "").split()
# Create a metric to track time spent and requests made.
REQUEST_TIME = Summary('request_processing_seconds',
'Time spent processing request',
labelnames=['method', 'endpoint'])
_models = {}
def get_model(model_name):
if model_name not in _models: