def test_es_log_extra_argument_insertion(self):
self.log.info("About to test elasticsearch insertion")
handler = CMRESHandler(hosts=[{
'host': self.getESHost(),
'port': self.getESPort()
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
es_index_name="pythontest",
es_additional_fields={
'App': 'Test',
'Environment': 'Dev'
},
raise_on_indexing_exceptions=True)
es_test_server_is_up = handler.test_es_source()
self.log.info(
"ES services status is: {0!s}".format(es_test_server_is_up))
self.assertEqual(True, es_test_server_is_up)
log = logging.getLogger("PythonTest")
log.addHandler(handler)
log.warning("Extra arguments Message",
extra={
"Arg1": 300,
"Arg2": 400
})
self.assertEqual(1, len(handler._buffer))
self.assertEqual(handler._buffer[0]['Arg1'], 300)
self.assertEqual(handler._buffer[0]['Arg2'], 400)
self.assertEqual(handler._buffer[0]['App'], 'Test')
self.assertEqual(handler._buffer[0]['Environment'], 'Dev')
handler.flush()
self.assertEqual(0, len(handler._buffer))
def createHandler(self, parameters=None):
"""
Each backend can initialize its attributes and create its handler with them.
:params parameters: dictionary of parameters. ex: {'FileName': file.log}
"""
if parameters is not None:
self.__host = parameters.get('Host', self.__host)
self.__user = parameters.get('User', self.__user)
self.__passwd = parameters.get('Password', self.__passwd)
self.__port = int(parameters.get('Port', self.__port))
self.__index = parameters.get('Index', self.__index)
self.__bufferSize = int(parameters.get('BufferSize', self.__bufferSize))
self.__flushTime = int(parameters.get('FlushTime', self.__flushTime))
if self.__user is not None and self.__passwd is not None:
self._handler = CMRESHandler(hosts=[{'host': self.__host, 'port': self.__port}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(self.__user, self.__passwd),
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
else:
self._handler = CMRESHandler(hosts=[{'host': self.__host, 'port': self.__port}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
# We give a format containing only asctime to add the field in elasticsearch
# asctime is not created at the initialization of the LogRecords but built in the format process
self._handler.setFormatter(logging.Formatter('%(asctime)s'))
def test_buffered_log_insertion_flushed_when_buffer_full(self):
handler = CMRESHandler(hosts=[{
'host': self.getESHost(),
'port': self.getESPort()
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
buffer_size=2,
flush_frequency_in_sec=1000,
es_index_name="pythontest",
es_additional_fields={
'App': 'Test',
'Environment': 'Dev'
},
raise_on_indexing_exceptions=True)
es_test_server_is_up = handler.test_es_source()
self.log.info(
"ES services status is: {0!s}".format(es_test_server_is_up))
self.assertEqual(True, es_test_server_is_up)
log = logging.getLogger("PythonTest")
log.setLevel(logging.DEBUG)
log.addHandler(handler)
log.warning("First Message")
log.info("Seccond Message")
self.assertEqual(0, len(handler._buffer))
handler.close()
def logData(self, data):
if self.es_user != "null":
self.handler = CMRESHandler(
hosts=[{"host": str(self.es_host), "port": int(self.es_port)}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
use_ssl=True,
verify_ssl=False,
auth_details=(self.es_user, self.es_pwd),
es_index_name="dockeree",
es_additional_fields=data,
)
else:
self.handler = CMRESHandler(
hosts=[{"host": str(self.es_host), "port": int(self.es_port)}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="dockeree",
es_additional_fields=data,
)
# log that data
self.eslog.addHandler(self.handler)
self.eslog.debug(
"[{}] Logging namespace {} quota to {}:{}".format(
time.time(), data["metadata"]["namespace"], self.es_host, self.es_port
)
)
self.eslog.removeHandler(self.handler)
def test_ping(self):
handler = CMRESHandler(hosts=[{'host': 'localhost', 'port': 9200}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="pythontest",
use_ssl=False)
es_test_server_is_up = handler.test_es_source()
self.assertEquals(True, es_test_server_is_up)
def test_ping(self):
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="pythontest",
use_ssl=False,
raise_on_indexing_exceptions=True)
es_test_server_is_up = handler.test_es_source()
self.assertEqual(True, es_test_server_is_up)
def test_ping(self):
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="pythontest",
use_ssl=False,
raise_on_indexing_exceptions=True)
es_test_server_is_up = handler.test_es_source()
self.assertEqual(True, es_test_server_is_up)
def test_index_name_frequency_functions(self):
index_name = "pythontest"
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.DAILY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_daily_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.WEEKLY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_weekly_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.MONTHLY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_monthly_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.YEARLY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_yearly_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.DISABLED,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_disabled_index_name(index_name)
)
def _initialize_logger(self):
# initialize handler
self.handler = CMRESHandler(
hosts=[{
'host': self._get_option('host'),
'port': self._get_option('port')
}],
auth_details=(self._get_option('username'),
self._get_option('password')),
auth_type=CMRESHandler.AuthType(self._get_option('auth_type')),
use_ssl=self._get_option('use_ssl'),
verify_ssl=self._get_option('verify_ssl'),
es_index_name=self._get_option('es_index_name'),
es_additional_fields={
'app': self._get_option('app'),
'env': self._get_option('env')
},
raise_on_indexing_exceptions=self._get_option(
'raise_on_indexing_exceptions'))
# check if elastic server is up
self._es_server_is_up = self.handler.test_es_source()
# initialize logger
self._logger = logging.getLogger(self._get_option('es_logger_name'))
# FIXME self._logger.setLevel(self._get_option('es_logger_level'))
self._logger.setLevel(logging.DEBUG)
self._logger.addHandler(self.handler)
#
self.ansible_session = str(uuid.uuid1())
self.ansible_version = os.popen("ansible --version | head -1").read()
self.errors = 0
self.base_data = {
'ansible_global_session': self.ansible_session,
'ansible_global_version': self.ansible_version
}
#
self.options = {}
if cli:
self._options = cli.options
self.base_data['ansible_global_checkmode'] = self._options.check
self.base_data['ansible_global_tags'] = self._options.tags
self.base_data[
'ansible_global_skip_tags'] = self._options.skip_tags
self.base_data[
'ansible_global_inventory'] = self._options.inventory
class Logger:
__is_initialized = False
class __CustomFormatter(logging.Formatter):
"""Logging Formatter to add colors and count warning / errors"""
grey = "\x1b[38;21m"
yellow = "\x1b[33;21m"
red = "\x1b[31;21m"
bold_red = "\x1b[31;1m"
reset = "\x1b[0m"
format = "%(asctime)s %(name)s [%(levelname)s] %(message)s"
FORMATS = {
logging.DEBUG: grey + format + reset,
logging.INFO: grey + format + reset,
logging.WARNING: yellow + format + reset,
logging.ERROR: red + format + reset,
logging.CRITICAL: bold_red + format + reset
}
def format(self, record):
log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record)
def __init__(self):
self.__root = logging.getLogger("BackupCreator")
if Logger.__is_initialized is False:
Logger.__is_initialized = True
self.__root.setLevel(logging.DEBUG)
self.__handler = logging.StreamHandler(sys.stdout)
self.__handler.setLevel(logging.DEBUG)
self.__handler.setFormatter(self.__CustomFormatter())
self.__root.addHandler(self.__handler)
self.__handler2 = CMRESHandler(
hosts=[{
'host': 'localhost',
'port': 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="logstash_mailsender-{0:yyyy.MM.dd}")
self.__handler2.setLevel(logging.DEBUG)
self.__root.addHandler(self.__handler2)
def get_logger(self):
return self.__root
def test_fast_insertion_of_hundred_logs(self):
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
buffer_size=500,
flush_frequency_in_sec=0.5,
es_index_name="pythontest",
raise_on_indexing_exceptions=True)
log = logging.getLogger("PythonTest")
log.setLevel(logging.DEBUG)
log.addHandler(handler)
for i in range(100):
log.info("Logging line {0:d}".format(i), extra={'LineNum': i})
handler.flush()
self.assertEqual(0, len(handler._buffer))
def test_fast_insertion_of_hundred_logs(self):
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
buffer_size=500,
flush_frequency_in_sec=0.5,
es_index_name="pythontest",
raise_on_indexing_exceptions=True)
log = logging.getLogger("PythonTest")
log.setLevel(logging.DEBUG)
log.addHandler(handler)
for i in range(100):
log.info("Logging line {0:d}".format(i), extra={'LineNum': i})
handler.flush()
self.assertEqual(0, len(handler._buffer))
class ESLoggerConfig(BaseConfig):
USE_ELASTIC_SEARCH = bool(get_configuration_value('USE_ES_LOGS', False))
ES_HANDLER = None
if USE_ELASTIC_SEARCH:
from cmreslogging.handlers import CMRESHandler
es_host = get_configuration_value('ES_HOST', 'localhost')
es_port = int(get_configuration_value('ES_PORT', 9200))
es_user = get_configuration_value('ES_USER', None)
es_password = get_configuration_value('ES_PASSWORD', None)
es_use_ssl = bool(int(get_configuration_value('ES_USE_SSL', 0)))
es_verify_ssl = get_configuration_value('ES_VERIFY_SSL', False)
es_index_name = get_configuration_value('ES_INDEX_NAME',
'tesla_admin_index')
es_auth_type = CMRESHandler.AuthType.NO_AUTH
if es_user is not None and es_password is not None:
es_auth_type = CMRESHandler.AuthType.BASIC_AUTH
es_handler = CMRESHandler(hosts=[{
'host': es_host,
'port': es_port
}],
auth_type=es_auth_type,
auth_details=(es_user, es_password),
use_ssl=es_use_ssl,
verify_ssl=es_verify_ssl,
es_index_name=es_index_name,
flush_frequency_in_sec=15,
es_additional_fields={
'Module': BaseConfig.MODULE_NAME,
'Version': BaseConfig.MODULE_VERSION,
'Environment': BaseConfig.CONFIG_NAME
})
if es_handler.test_es_source():
es_handler.setLevel(logging.INFO)
ES_HANDLER = es_handler
def add_elasticsearch_handler(self):
if ELASTICSEARCH_INSTALLED:
es_hosts = self._env('HANDLERS_ELASTICSEARCH_HOST')
if not es_hosts:
self.logger.warning('Elasticsearch handler host not set')
return
name = self._env('HANDLERS_ELASTICSEARCH_NAME',
default='elasticsearch')
level = self._env('HANDLERS_ELASTICSEARCH_LEVEL', default='info')
formatter = self._env('HANDLER_ELASTICSEARCH_FORMATTER',
default='json')
hosts = []
es_hosts = es_hosts.split(',')
for es_host in es_hosts:
host, port = es_host.split(':', 1)
hosts.append({'host': host, 'port': port})
handler_args = {
'hosts': hosts,
'auth_type': CMRESHandler.AuthType.NO_AUTH
}
handler = CMRESHandler(**handler_args)
self.add_handler(handler=handler,
name=name,
formatter=formatter,
level=level)
else:
self.logger.error(
'It seems that the elasticsearch handler is not installed. '
'You can install it by running `pip install '
'wryte[elasticsearch]`')
def create_app():
from project.models import db
from project.views import views_bp as views_blueprint
app = Flask(__name__)
app.config.from_object(config[ENVIRONMENT])
if not app.config["DEBUG"]:
handler = CMRESHandler(hosts=[{
'host': '192.168.99.100',
'port': 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="my_python_index",
es_additional_fields={
'App': app.config["APP_NAME"],
'Environment': ENVIRONMENT
})
app.logger.addHandler(handler)
db.init_app(app)
bcrypt.init_app(app)
jwt = JWT(app, authenticate, identity)
swagger = Swagger(app, config=swagger_config)
app.register_blueprint(views_blueprint)
return app, db
def configure_logger(app):
logs_folder = app.config.get('LOG_PATH')
if not logs_folder:
logs_folder = os.getcwd()
logs_folder = os.path.join(logs_folder, 'ecm_logs')
if not os.path.isdir(logs_folder):
os.makedirs(logs_folder)
handler = RotatingFileHandler(os.path.join(logs_folder, 'ecm.log'),
maxBytes=1 * 1024 * 1024,
backupCount=10,
encoding='UTF-8')
# 如果不是debug模式，则捕捉INFO以上的日志信息，默认NOTSET
if not app.config.get('DEBUG'):
handler.setLevel(logging.INFO)
app.logger.setLevel(logging.INFO)
logging_format = logging.Formatter(
'[%(asctime)s] [%(levelname)s] [%(pathname)s:%(lineno)d] [%(message)s]'
)
handler.setFormatter(logging_format)
app.logger.addHandler(handler)
es_handler = CMRESHandler(hosts=[{
'host': 'localhost',
'port': 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="central_management_log")
app.logger.setLevel(logging.INFO)
app.logger.addHandler(es_handler)
return
def test_index_name_frequency_functions(self):
index_name = "pythontest"
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.DAILY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_daily_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.WEEKLY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_weekly_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.MONTHLY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_monthly_index_name(index_name)
)
handler = CMRESHandler(hosts=[{'host': self.getESHost(), 'port': self.getESPort()}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
use_ssl=False,
index_name_frequency=CMRESHandler.IndexNameFrequency.YEARLY,
raise_on_indexing_exceptions=True)
self.assertEqual(
handler._index_name_func.__func__(index_name),
CMRESHandler._get_yearly_index_name(index_name)
)
def createHandler(self, parameters=None):
"""
Each backend can initialize its attributes and create its handler with them.
:params parameters: dictionary of parameters. ex: {'FileName': file.log}
"""
if parameters is not None:
self.__host = parameters.get('Host', self.__host)
self.__user = parameters.get('User', self.__user)
self.__passwd = parameters.get('Password', self.__passwd)
self.__port = int(parameters.get('Port', self.__port))
self.__index = parameters.get('Index', self.__index)
self.__bufferSize = int(
parameters.get('BufferSize', self.__bufferSize))
self.__flushTime = int(
parameters.get('FlushTime', self.__flushTime))
if self.__user is not None and self.__passwd is not None:
self._handler = CMRESHandler(
hosts=[{
'host': self.__host,
'port': self.__port
}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(self.__user, self.__passwd),
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
else:
self._handler = CMRESHandler(
hosts=[{
'host': self.__host,
'port': self.__port
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
# We give a format containing only asctime to add the field in elasticsearch
# asctime is not created at the initialization of the LogRecords but built in the format process
self._handler.setFormatter(logging.Formatter('%(asctime)s'))
def get_db_logger(
host='search-parsl-logging-test-2yjkk2wuoxukk2wdpiicl7mcrm.us-east-1.es.amazonaws.com',
port=443,
enable_es_logging=False,
index_name="my_python_index",
version='1.0.0',
**kwargs):
"""
Parameters
----------
host : str, optional
URL to the elasticsearch cluster. Skip the http(s)://
port : int, optional
Port to use to access the elasticsearch cluster
enable_es_logging : Bool, optional
Set to True to enable logging to elasticsearch
index_name : str, optional
Index name to use for elasticsearch
Returns
-------
logging.logger object
Raises
------
OptionalModuleMissing
"""
logger = logging.getLogger(__file__)
if enable_es_logging:
if not _es_logging_enabled:
raise OptionalModuleMissing(
['CMRESHandler'],
"Logging to ElasticSearch requires the cmreslogging module")
handler = CMRESHandler(
hosts=[{
'host': host,
'port': port
}],
use_ssl=True,
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
es_additional_fields={
'Campaign': "test",
# use the name of the user's home directory as their username since there
# does not seem to be a portable way to do this
'Version': version,
'Username': getpass.getuser()
})
logger = logging.getLogger("ParslElasticsearch")
logger.setLevel(logging.INFO)
logger.addHandler(handler)
else:
logger.addHandler(NullHandler())
return logger
def enrich_es_handler():
return CMRESHandler(hosts=[{
'host_ip': ES_HOST,
'port': ES_PORT
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=ES_LOG_INDEX,
es_additional_fields={
'App': APP,
'Environment': ENV
})
def __init__(self):
self.__root = logging.getLogger("BackupCreator")
if Logger.__is_initialized is False:
Logger.__is_initialized = True
self.__root.setLevel(logging.DEBUG)
self.__handler = logging.StreamHandler(sys.stdout)
self.__handler.setLevel(logging.DEBUG)
self.__handler.setFormatter(self.__CustomFormatter())
self.__root.addHandler(self.__handler)
self.__handler2 = CMRESHandler(
hosts=[{
'host': 'localhost',
'port': 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="logstash_mailsender-{0:yyyy.MM.dd}")
self.__handler2.setLevel(logging.DEBUG)
self.__root.addHandler(self.__handler2)
def test_buffered_log_insertion_flushed_when_buffer_full(self):
handler = CMRESHandler(hosts=[{'host': 'localhost', 'port': 9200}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
buffer_size=2,
flush_frequency_in_sec=1000,
es_index_name="pythontest",
es_additional_fields={'App': 'Test', 'Environment': 'Dev'})
es_test_server_is_up = handler.test_es_source()
self.log.info("ES services status is: {0!s}".format(es_test_server_is_up))
self.assertEquals(True, es_test_server_is_up)
log = logging.getLogger("PythonTest")
log.setLevel(logging.DEBUG)
log.addHandler(handler)
log.warning("First Message")
log.info("Seccond Message")
self.assertEquals(0, len(handler._buffer))
handler.close()
def test_es_log_extra_argument_insertion(self):
self.log.info("About to test elasticsearch insertion")
handler = CMRESHandler(hosts=[{'host': 'localhost', 'port': 9200}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
es_index_name="pythontest",
es_additional_fields={'App': 'Test', 'Environment': 'Dev'})
es_test_server_is_up = handler.test_es_source()
self.log.info("ES services status is: {0!s}".format(es_test_server_is_up))
self.assertEquals(True, es_test_server_is_up)
log = logging.getLogger("PythonTest")
log.addHandler(handler)
log.warning("Extra arguments Message", extra={"Arg1": 300, "Arg2": 400})
self.assertEquals(1, len(handler._buffer))
self.assertEquals(handler._buffer[0]['Arg1'], 300)
self.assertEquals(handler._buffer[0]['Arg2'], 400)
self.assertEquals(handler._buffer[0]['App'], 'Test')
self.assertEquals(handler._buffer[0]['Environment'], 'Dev')
handler.flush()
self.assertEquals(0, len(handler._buffer))
def test_buffered_log_insertion_after_interval_expired(self):
handler = CMRESHandler(hosts=[{'host': 'localhost', 'port': 9200}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
flush_frequency_in_sec=0.1,
es_index_name="pythontest",
es_additional_fields={'App': 'Test', 'Environment': 'Dev'})
es_test_server_is_up = handler.test_es_source()
self.log.info("ES services status is: {0!s}".format(es_test_server_is_up))
self.assertEquals(True, es_test_server_is_up)
log = logging.getLogger("PythonTest")
log.addHandler(handler)
log.warning("Extra arguments Message", extra={"Arg1": 300, "Arg2": 400})
self.assertEquals(1, len(handler._buffer))
self.assertEquals(handler._buffer[0]['Arg1'], 300)
self.assertEquals(handler._buffer[0]['Arg2'], 400)
self.assertEquals(handler._buffer[0]['App'], 'Test')
self.assertEquals(handler._buffer[0]['Environment'], 'Dev')
time.sleep(1)
self.assertEquals(0, len(handler._buffer))
def __init__(self):
es_config = G_CONFIG.config['elasticsearch']
hosts = [{'host': es_config['host'], 'port': es_config['port']}]
self.handler = CMRESHandler(
hosts=hosts,
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.ES_INDEX_NAME,
index_name_frequency=CMRESHandler.IndexNameFrequency.MONTHLY)
self.es = Elasticsearch(hosts=hosts)
self.log = logging.getLogger(self.LOGGER_NAME)
self.log.setLevel(logging.INFO)
self.log.addHandler(self.handler)
self.es.indices.put_template(name='qlogger_template',
body=self._index_template())
def elasticsearch():
import logging
from cmreslogging.handlers import CMRESHandler
handler = CMRESHandler(
hosts=[{
"host": "localhost",
"port": 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="minc_index",
)
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO)
root_logger.addHandler(handler)
def get_logger():
# Enable logging
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
level=logging.INFO)
logger = logging.getLogger(__name__)
if os.getenv('ELASTIC_HOST') and os.getenv('ELASTIC_USER') and os.getenv('ELASTIC_PASS') and \
os.getenv('SEND_LOGS_TO_ELASTIC'):
handler = CMRESHandler(
hosts=[{
'host': os.getenv('ELASTIC_HOST'),
'port': 9200
}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(os.getenv('ELASTIC_USER'),
os.getenv('ELASTIC_PASS')),
es_index_name="munich-tg-logs",
index_name_frequency=CMRESHandler.IndexNameFrequency.MONTHLY)
logger.addHandler(handler)
return logger
def main(args=None):
####################
# Synchronize time #
####################
try:
import ntplib
client = ntplib.NTPClient()
response = client.request('pool.ntp.org')
os.system(
'date ' +
time.strftime('%m%d%H%M%Y.%S', time.localtime(response.tx_time)))
except:
print('Could not sync with time server.')
"""The main routine."""
if args is None:
args = sys.argv[1:]
print "version=" + version
################
# Parse config #
################
config = ConfigParser.RawConfigParser()
config.file = '/etc/ble_positioning_node/config.conf'
config.read(config.file)
###############################
# We want reporting to kibana #
###############################
LOG_CONN_POINTS = [{
'host': config.get('Communication', 'elastic'),
'port': config.getint('Communication', 'elastic_port')
}]
handler = CMRESHandler(hosts=LOG_CONN_POINTS,
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=True,
es_index_name="beacon-scanner")
scan = BLEScanner(handler, config)
scan.main()
def init_logging(application):
if application.config['ELASTIC_HOST']:
env_name = 'development'
if 'APP_CONFIG_FILE' in os.environ:
env_name = os.environ['APP_CONFIG_FILE']
handler = CMRESHandler(hosts=[{
'host':
application.config["ELASTIC_HOST"],
'port':
9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="logs-nisse",
es_additional_fields={'environment': env_name})
application.logger.addHandler(handler)
# application.logger.setLevel(logging.DEBUG)
def create_app():
from project.models import db
from project.views import views_bp as views_blueprint
environment = os.environ.get("ENVIRONMENT", "default")
app = Flask(__name__)
app.config.from_object(CONFIG[environment])
app.wsgi_app = PrefixMiddleware(app.wsgi_app,
prefix=app.config["APPLICATION_ROOT"])
db.init_app(app)
if not app.config["DEBUG"]:
handler = CMRESHandler(hosts=[{
'host': '192.168.99.100',
'port': 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="my_python_index",
es_additional_fields={
'App': app.config["APP_NAME"],
'Environment': ENVIRONMENT
})
app.logger.addHandler(handler)
SWAGGER_CONFIG["specs"][0]["route"] = SWAGGER_CONFIG["specs"][0][
"route"].format(application_root=app.config["APPLICATION_ROOT"])
SWAGGER_CONFIG["static_url_path"] = SWAGGER_CONFIG[
"static_url_path"].format(
application_root=app.config["APPLICATION_ROOT"])
SWAGGER_CONFIG["specs_route"] = SWAGGER_CONFIG["specs_route"].format(
application_root=app.config["APPLICATION_ROOT"])
SWAGGER_CONFIG["basePath"] = SWAGGER_CONFIG["basePath"].format(
application_root=app.config["APPLICATION_ROOT"])
Swagger(app, config=SWAGGER_CONFIG)
app.register_blueprint(views_blueprint)
with app.test_request_context():
db.create_all()
return app, db
def test_new_index_has_messages(self):
index_name = "pythontest-index-messages"
handler = CMRESHandler(hosts=[{
'host': self.getESHost(),
'port': self.getESPort()
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
use_ssl=False,
es_index_name=index_name,
es_additional_fields={
'App': 'Test',
'Environment': 'Dev'
},
raise_on_indexing_exceptions=True)
message = "Specific message %s"
es_test_server_is_up = handler.test_es_source()
index = handler._index_name_func.__func__(index_name)
client = handler._client
client.indices.delete(index, ignore=404)
self.log.info(
"ES services status is: {0!s}".format(es_test_server_is_up))
self.assertEqual(True, es_test_server_is_up)
log = logging.getLogger("PythonTest")
log.setLevel(logging.DEBUG)
log.addHandler(handler)
log.info(message, 1)
log.info(message, 'str')
handler.flush()
self.assertEqual(0, len(handler._buffer))
# Wait a little while elasticsearch creating mapping and indexing messages
time.sleep(1)
self.assertEqual(
2,
client.search(index,
body={"query": {
"match": {
'message': message
}
}})['hits']['total'])
handler.close()
def send_log(message, level='info'):
from cmreslogging.handlers import CMRESHandler
handler = CMRESHandler(hosts=[{
'host': 'es01',
'port': 9200
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="my_python_index")
log = logging.getLogger("PythonTest")
log.setLevel(logging.INFO)
log.addHandler(handler)
if level == 'info':
log.info(str(message))
else:
log.error(str(message))
#log.debug()
#log.info()
#log.warning()
#log.error()
#log.critical()
#send_log('123456789')
def getLogger(self):
handler = logging.handlers.SysLogHandler(
address=('logsene-receiver-syslog.sematext.com', 514))
handler2 = CMRESHandler(hosts=[{
'host': os.getenv('BMU_ES_HOST'),
'port': 9200
}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(os.getenv('BMU_ES_USR'),
os.getenv('BMU_ES_PWD')),
es_index_name="i_bmu")
formater = logging.Formatter(
"4c971c88-73a9-4557-b534-4fb4cebc6d48:%(message)s")
console_formatter = logging.Formatter(
fmt='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
handler.setFormatter(formater)
console = logging.StreamHandler(sys.stdout)
log = logging.getLogger("BackupMeUp")
log.setLevel(logging.INFO)
log.addHandler(console)
log.addHandler(handler)
log.addHandler(handler2)
return log
def get_db_logger(logger_name='parsl_db_logger',
is_logging_server=False,
monitoring_config=None,
**kwargs):
"""
Parameters
----------
logger_name : str, optional
Name of the logger to use. Prevents adding repeat handlers or incorrect handlers
is_logging_server : Bool, optional
Used internally to determine which handler to return when using local db logging
monitoring_config : MonitoringConfig, optional
Pass in a logger class object to use for generating loggers.
Returns
-------
logging.logger object
Raises
------
OptionalModuleMissing
"""
logger = logging.getLogger(logger_name)
if monitoring_config is None:
logger.addHandler(NullHandler())
return logger
if monitoring_config.database_type == 'elasticsearch':
if not _es_logging_enabled:
raise OptionalModuleMissing(
['CMRESHandler'],
"Logging to ElasticSearch requires the cmreslogging module")
handler = CMRESHandler(hosts=[{
'host': monitoring_config.host,
'port': monitoring_config.port
}],
use_ssl=monitoring_config.enable_ssl,
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=monitoring_config.index_name,
es_additional_fields={
'Campaign': "test",
'Version': monitoring_config.version,
'Username': getpass.getuser()
})
logger = logging.getLogger(monitoring_config.logger_name)
logger.setLevel(logging.INFO)
logger.addHandler(handler)
elif monitoring_config.database_type == 'local_database' and not is_logging_server:
# add a handler that will pass logs to the logging server
handler = RemoteHandler(monitoring_config.web_app_host,
monitoring_config.web_app_port)
# use the specific name generated by the server or the monitor wrapper
logger = logging.getLogger(logger_name)
logger.setLevel(logging.INFO)
logger.addHandler(handler)
elif monitoring_config.database_type == 'local_database' and is_logging_server:
# add a handler that will take logs being recieved on the server and log them to the database
handler = DatabaseHandler(monitoring_config.eng_link)
# use the specific name generated by the server or the monitor wrapper
logger = logging.getLogger(logger_name)
logger.setLevel(logging.INFO)
logger.addHandler(handler)
else:
raise ValueError(
'database_type must be one of ["local_database", "elasticsearch"]')
return logger
class ElasticSearchBackend(AbstractBackend):
"""
ElasticsearchBackend is used to create an abstraction of the handler and the formatter concepts from logging.
Here, we have a CMRESHandler which is part of an external library named 'cmreslogging' based on 'logging'.
CMRESHandler is a specific handler created to send log records to an ElasticSearch DB. It does not need a Formatter
object.
"""
def __init__(self):
"""
CMRESHandler needs, at least, a hostname, a username, a password, a port and a specific index
from the ElasticSearch DB to send log records.
"""
super(ElasticSearchBackend, self).__init__(None, None)
self.__host = ''
self.__user = None
self.__passwd = None
self.__port = 9203
self.__index = ''
self.__bufferSize = 1000
self.__flushTime = 1
def createHandler(self, parameters=None):
"""
Each backend can initialize its attributes and create its handler with them.
:params parameters: dictionary of parameters. ex: {'FileName': file.log}
"""
if parameters is not None:
self.__host = parameters.get('Host', self.__host)
self.__user = parameters.get('User', self.__user)
self.__passwd = parameters.get('Password', self.__passwd)
self.__port = int(parameters.get('Port', self.__port))
self.__index = parameters.get('Index', self.__index)
self.__bufferSize = int(parameters.get('BufferSize', self.__bufferSize))
self.__flushTime = int(parameters.get('FlushTime', self.__flushTime))
if self.__user is not None and self.__passwd is not None:
self._handler = CMRESHandler(hosts=[{'host': self.__host, 'port': self.__port}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(self.__user, self.__passwd),
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
else:
self._handler = CMRESHandler(hosts=[{'host': self.__host, 'port': self.__port}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
# We give a format containing only asctime to add the field in elasticsearch
# asctime is not created at the initialization of the LogRecords but built in the format process
self._handler.setFormatter(logging.Formatter('%(asctime)s'))
def setLevel(self, level):
"""
No possibility to set the level of the ElasticSearch handler.
It is not set by default so it can send all Log Records of all levels to ElasticSearch.
"""
pass
def setFormat(self, fmt, datefmt, options):
"""
Each backend give a format to their formatters and attach them to their handlers.
:params fmt: string representing the log format
:params datefmt: string representing the date format
:params component: string represented as "system/component"
:params options: dictionary of logging options. ex: {'Color': True}
"""
pass
sqsChainTxKey = 'SQS_UBIRCH_BIGCHAIN_DB_TX'
sqsRegionKey = 'AWS_REGION'
ipdbAppIdKey = 'IPDB_APP_ID'
ipdbAppKeyKey = 'IPDB_APP_KEY'
bigChainDbHostKey = 'BIG_CHAIN_DB_HOST'
numThreadsKey = 'NUM_TRHEADS'
esLoggerHostKey = 'ES_LOG_HOST'
esLoggerPortKey = 'ES_LOG_PORT'
if (esLoggerHostKey in os.environ.keys()):
esLoggerHost = os.environ[esLoggerHostKey]
esLoggerPort = int(os.environ[esLoggerPortKey])
esLoggerHandler = CMRESHandler(
hosts=[{
'host': esLoggerHost,
'port': esLoggerPort
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name="big-chain-store-service-logs")
logger.addHandler(esLoggerHandler)
if (awsAccessKeyId not in os.environ or awsSecretAccessKey not in os.environ):
logger.error("env vars missing")
logger.info("AWS_ACCESS_KEY_ID -> AWS access key")
logger.info("AWS_SECRET_ACCESS_KEY -> AWS secret key")
logger.info(
"SQS_CHAIN_IN -> AWS SQS queue name of inbound data which should be stored into bigChainDb (optional), default is 'local_dev_ubirch_bigchaindb_in'"
)
logger.info(
"SQS_CHAIN_TX -> AWS SQS queue name for outbound tx hash publishing (optional), default is 'local_dev_ubirch_bigchaindb_tx'"
)
async def InitServer(cls,
servertype="unkown-servertype",
startmysql=True,
startpg=True):
# 创建日志目录
if not os.path.exists('logs'):
os.makedirs('logs')
logging.basicConfig(level=logging.DEBUG) # 默认屏幕输出
filehandler = TimedRotatingFileHandler( # 时间轮转输出
filename="logs/my.log",
when='D',
interval=1,
backupCount=0)
filehandler.suffix = "%Y%m%d-%H%M%S.log"
formatter = logging.Formatter(
"%(asctime)s-%(name)s-%(levelname)s-[%(filename)s:%(lineno)d]-%(message)s"
)
filehandler.setFormatter(formatter)
filehandler.setLevel(logging.INFO)
logging.getLogger().addHandler(filehandler) # 添加时间轮转输出
# 服务器名字 数据文件路径
cls.myservertype = servertype
cls.mydatapath = datafilepath()
# 生成一个新的 guid 可能不会使用，如果有的话
newguid = cls.myservertype + "_" + str(uuid.uuid1())
await cls.loadconfig()
if (cls.elkdatabase == ""):
cls.elkdatabase = cls.myservertype
if (cls.myserverguid == None):
cls.myserverguid = newguid
# 没有配置文件就保存
await cls.saveconfig()
# ELK的日志输出
es_handler = CMRESHandler(hosts=[{
'host': cls.elkhost,
'port': cls.elkport
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=cls.elkdatabase,
es_additional_fields={
'App': cls.myservertype,
'AppGuid': cls.myserverguid
})
es_handler.setLevel(logging.INFO)
formatter2 = logging.Formatter(
"%(asctime)s-%(name)s-%(levelname)s-[%(filename)s:%(lineno)d]-%(message)s"
)
es_handler.setFormatter(formatter2)
logging.getLogger().addHandler(es_handler)
# 禁止如下两个模块乱打调试日志，因为ELK会把 debug 日志打在屏幕，有干扰
logging.getLogger("elasticsearch").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
# 创建 mysql 的工厂
if startmysql:
cls.mysqlengine = await create_engine(user=cls.mysqluser,
password=cls.mysqlpassword,
host=cls.mysqlhost,
port=cls.mysqlport,
db=cls.mysqldb,
loop=cls.asyncioloop)
if startpg:
cls.pg_pool = await asyncpg.create_pool(
user='******',
password='******',
database='goldhonor',
host='factory.goldhonor.com',
port=15432,
command_timeout=60)
pass
load_dotenv()
# load variables
http_base_container = os.getenv("HTTP_BASE_CONTAINER_IMAIS")
elastic_search_host = os.getenv("ELASTIC_SEARCH_HOST")
elastic_search_port = os.getenv("ELASTIC_SEARCH_PORT")
index_name = os.getenv("INDEX_NAME")
info = os.getenv("INFO")
logger_name = os.getenv("LOGGER_NAME")
# add elasticsearch logging capabilities
# parameters coming from env file
handler = CMRESHandler(
hosts=[{
"host": elastic_search_host,
"port": elastic_search_port
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=index_name,
)
# name of the logger
# level and handler
log = logging.getLogger(logger_name)
log.setLevel(logging.INFO)
log.addHandler(handler)
class BlobStorage(object):
"""A class that allows you to connect to Azure Blob Storage
With this class it will be possible to identify the metadata of the files,
class ElasticSearchBackend(AbstractBackend):
"""
ElasticsearchBackend is used to create an abstraction of the handler and the formatter concepts from logging.
Here, we have a CMRESHandler which is part of an external library named 'cmreslogging' based on 'logging'.
CMRESHandler is a specific handler created to send log records to an ElasticSearch DB. It does not need a Formatter
object.
"""
def __init__(self):
"""
CMRESHandler needs, at least, a hostname, a username, a password, a port and a specific index
from the ElasticSearch DB to send log records.
"""
super(ElasticSearchBackend, self).__init__(None, None)
self.__host = ''
self.__user = None
self.__passwd = None
self.__port = 9203
self.__index = ''
self.__bufferSize = 1000
self.__flushTime = 1
def createHandler(self, parameters=None):
"""
Each backend can initialize its attributes and create its handler with them.
:params parameters: dictionary of parameters. ex: {'FileName': file.log}
"""
if parameters is not None:
self.__host = parameters.get('Host', self.__host)
self.__user = parameters.get('User', self.__user)
self.__passwd = parameters.get('Password', self.__passwd)
self.__port = int(parameters.get('Port', self.__port))
self.__index = parameters.get('Index', self.__index)
self.__bufferSize = int(
parameters.get('BufferSize', self.__bufferSize))
self.__flushTime = int(
parameters.get('FlushTime', self.__flushTime))
if self.__user is not None and self.__passwd is not None:
self._handler = CMRESHandler(
hosts=[{
'host': self.__host,
'port': self.__port
}],
auth_type=CMRESHandler.AuthType.BASIC_AUTH,
auth_details=(self.__user, self.__passwd),
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
else:
self._handler = CMRESHandler(
hosts=[{
'host': self.__host,
'port': self.__port
}],
auth_type=CMRESHandler.AuthType.NO_AUTH,
es_index_name=self.__index,
use_ssl=True,
verify_ssl=True,
buffer_size=self.__bufferSize,
flush_frequency_in_sec=self.__flushTime)
# We give a format containing only asctime to add the field in elasticsearch
# asctime is not created at the initialization of the LogRecords but built in the format process
self._handler.setFormatter(logging.Formatter('%(asctime)s'))
def setLevel(self, level):
"""
No possibility to set the level of the ElasticSearch handler.
It is not set by default so it can send all Log Records of all levels to ElasticSearch.
"""
pass
def setFormat(self, fmt, datefmt, options):
"""
Each backend give a format to their formatters and attach them to their handlers.
:params fmt: string representing the log format
:params datefmt: string representing the date format
:params component: string represented as "system/component"
:params options: dictionary of logging options. ex: {'Color': True}
"""
pass