def add_http_logstash_handler(self, host='http://logapi.misa.com.vn', port=80, app_id='test-log', level=logging.ERROR, database_path=None,
headers=None):
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
class HttpTransport:
def __init__(self, host, port, **kwargs):
self._host = host
self._port = port
self._headers = kwargs['headers'] if 'headers' in kwargs and kwargs['headers'] else {'Authorization': 'Basic bWlzYTpNaXNhQDIwMTk=',
'Content-Type': 'application/json; charset=utf-8'}
def close(self):
pass
def send(self, data: dict, use_logging=None):
for v in data:
requests.post(url=self._host, data=v, headers=self._headers)
class CustomFormatter(LogstashFormatter):
def _move_extra_record_fields_to_prefix(self, message):
super()._move_extra_record_fields_to_prefix(message)
message['app_id'] = app_id
handler = AsynchronousLogstashHandler(host, port, transport=HttpTransport(host, port, headers=headers),
database_path=database_path or f'/tmp/logstash_{app_id}.db')
handler.formatter = CustomFormatter(tags=[app_id])
handler.setLevel(level)
self.handlers.append(handler)
return self
def get_logger():
global hd_log
if hd_log is None:
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
print('base_dir', BASE_DIR)
log_file_path = os.path.join(BASE_DIR, 'Log/python-web.log')
err_log_file_path = os.path.join(BASE_DIR, 'Log/python-web-err.log')
logger.add(log_file_path, rotation="50 MB",
encoding='utf-8') # Automatically rotate too big file
logger.add(err_log_file_path,
rotation="50 MB",
encoding='utf-8',
level='ERROR') # Automatically rotate too big file
# logstash
logstash_ip = get_config('logstash', 'ip')
if logstash_ip:
logstash_port = get_config('logstash', 'port')
logstash_handler = AsynchronousLogstashHandler(logstash_ip,
int(logstash_port),
database_path=None)
logstash_formatter = LogstashFormatter(
message_type='python-logstash',
extra_prefix='',
extra=dict(app_name='python-web'))
logstash_handler.setFormatter(logstash_formatter)
logger.add(sink=logstash_handler)
hd_log = logger
return hd_log
def a_async_handler(host='0.0.0.0', port=5000, formatter=None):
async_handler = AsynchronousLogstashHandler(host, port, database_path=None)
formatter = formatter if formatter else a_formatter()
async_handler.setFormatter(formatter)
return async_handler
def configure_logging(self):
# Filters
str_format_filter = dist_zero.logging.StrFormatFilter()
context = {
'env': settings.DIST_ZERO_ENV,
'mode': runners.MODE_SIMULATED,
'runner': True,
'simulator_id': self.id,
'start_at': self._start_datetime,
}
if settings.LOGZ_IO_TOKEN:
context['token'] = settings.LOGZ_IO_TOKEN
context_filter = dist_zero.logging.ContextFilter(context)
# Formatters
human_formatter = dist_zero.logging.HUMAN_FORMATTER
json_formatter = dist_zero.logging.JsonFormatter(
'(asctime) (levelname) (name) (message)')
# Handlers
stdout_handler = logging.StreamHandler(sys.stdout)
human_file_handler = logging.FileHandler('./.tmp/simulator.log')
json_file_handler = logging.FileHandler('./.tmp/simulator.json.log')
logstash_handler = AsynchronousLogstashHandler(
settings.LOGSTASH_HOST,
settings.LOGSTASH_PORT,
database_path='./.tmp/logstash.db',
)
stdout_handler.setLevel(logging.ERROR)
human_file_handler.setLevel(logging.DEBUG)
json_file_handler.setLevel(logging.DEBUG)
logstash_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(human_formatter)
human_file_handler.setFormatter(human_formatter)
json_file_handler.setFormatter(json_formatter)
logstash_handler.setFormatter(json_formatter)
stdout_handler.addFilter(str_format_filter)
human_file_handler.addFilter(str_format_filter)
json_file_handler.addFilter(str_format_filter)
json_file_handler.addFilter(context_filter)
logstash_handler.addFilter(str_format_filter)
logstash_handler.addFilter(context_filter)
# Loggers
dist_zero_logger = logging.getLogger('dist_zero')
root_logger = logging.getLogger()
dist_zero.logging.set_handlers(root_logger, [
json_file_handler,
human_file_handler,
logstash_handler,
stdout_handler,
])
def _bind_handlers(self):
""" Binds Logstash Handlers to the Logging object """
load_dotenv()
handler = AsynchronousLogstashHandler(
host=os.environ["LOGSTASH_HOST"],
port=int(os.environ["LOGSTASH_PORT"]),
database_path=None,
)
handler.setFormatter(FlaskLogstashFormatter())
self._logger.addHandler(handler)
def get_handler(extra={}, formatter=None):
extra['logstash_host'] = logstash_host
extra['logstash_port'] = logstash_port
if not formatter:
formatter = get_formatter(extra)
handler = AsynchronousLogstashHandler(logstash_host,
logstash_port,
database_path=database_path)
handler.setFormatter(formatter)
return handler
def add_logstash_handler(self, host, port, app_id, level=logging.ERROR, database_path=None):
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
class CustomFormatter(LogstashFormatter):
def _move_extra_record_fields_to_prefix(self, message):
super()._move_extra_record_fields_to_prefix(message)
message['app_id'] = app_id
handler = AsynchronousLogstashHandler(host, port, database_path=database_path or f'/tmp/logstash_{app_id}.db')
handler.formatter = CustomFormatter(tags=[app_id])
handler.setLevel(level)
self.handlers.append(handler)
return self
def logger_configure(
log_level: int = logging.DEBUG) -> None: # pragma: no cover
formatter: Formatter = Formatter()
logger.configure(
**{
'handlers': [
{
'sink': sys.stdout,
'colorize': True,
'format': formatter.stdout_format
},
],
'extra': {
'headers': '',
'uuid': None,
},
})
LoggerConfigure.add_levels()
if setting.integrate.logstash.enable:
logger.add(
AsynchronousLogstashHandler(
setting.integrate.logstash.host,
setting.integrate.logstash.port,
setting.integrate.logstash.database_path,
**setting.integrate.logstash.options,
))
def main(interface, logstash_config):
global output
if logstash_config is not None:
global test_logger
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
test_logger.addHandler(
AsynchronousLogstashHandler(logstash_config['host'],
logstash_config['port'],
database_path=None))
test_logger.error(
'python-logstash-async: test logstash error message.')
logit()
elif output is not None:
writeit()
else:
printit()
max_bytes = 1000000
promiscuous = True
read_timeout = 100 # in milliseconds
pc = pcapy.open_live(interface, max_bytes, promiscuous, read_timeout)
packet_limit = -1 # infinite
pc.loop(packet_limit, recv_pkts) # capture packets
def setup_logger(self):
if os.environ.get("FRED_LOG_HOST") and os.environ.get("FRED_LOG_PORT"):
self.logger = logging.getLogger("python-logstash-logger")
self.logger.addHandler(
AsynchronousLogstashHandler(
os.environ.get("FRED_LOG_HOST"),
int(os.environ.get("FRED_LOG_PORT")), ""))
self.logger.addHandler(logging.StreamHandler())
else:
self.logger = logging.Logger("logger")
self.logger.setLevel(logging.INFO)
def __init__(self):
self.logger = logging.getLogger("logstash")
self.logger.setLevel(logging.INFO)
try:
host = os.environ["LOGSTASH_HOST"]
except:
host = "localhost"
try:
port = int(os.environ["LOGSTASH_PORT"])
except:
port = 5044
handler = AsynchronousLogstashHandler(host=host,
port=port,
ssl_enable=False,
ssl_verify=False,
database_path='')
formatter = LogstashFormatter()
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.info("Init logger")
def initiate_log():
# Create the logger and set it's logging level
logger = logging.getLogger("logstash")
logger.setLevel(logging.DEBUG)
# Create the handler
handler = AsynchronousLogstashHandler(
host='ab5413e7-7e28-45a6-bdaa-d6c3e00cab46-ls.logit.io',
port=27421,
ssl_enable=True,
ssl_verify=False,
database_path='')
# Here you can specify additional formatting on your log record/message
formatter = LogstashFormatter()
handler.setFormatter(formatter)
# Assign handler to the logger
logger.addHandler(handler)
return logger
def get_log():
host = 'localhost'
port = 5000
test_logger = logging.getLogger('breath-logger')
# Set it to whatever level you want - default will be info
test_logger.setLevel(logging.DEBUG)
# Create a handler for it
async_handler = AsynchronousLogstashHandler(host, port, database_path=None)
# Add the handler to the logger
test_logger.addHandler(async_handler)
return test_logger
def configure_logging(service_name):
"""
Configure logging based on the settings in the settings file.
This sets up a handler for each logging mode that is enabled.
See `microservice.core.settings.LoggingMode` for the supported logging types.
:param str service_name: Name of the service being served by this instance.
"""
logger = logging.getLogger()
logger.setLevel(settings.logging_level)
formatter_kwargs = {
'fmt': json.dumps({'extra': {
'local_service': service_name,
# Basic way to distinguish logs between instances of the same microservice.
'instance_id': random.randint(100000, 999999)
}})
}
formatter = LogstashFormatterV1(**formatter_kwargs)
if settings.LoggingMode.FILE in settings.logging_modes:
file_handler = logging.FileHandler('{}.log'.format(service_name))
file_handler.setFormatter(formatter)
file_handler.addFilter(RequestIDLogFilter())
logger.addHandler(file_handler)
if settings.LoggingMode.HUMAN in settings.logging_modes:
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(HumanReadableLogstashFormatter(**formatter_kwargs))
stdout_handler.addFilter(RequestIDLogFilter())
logger.addHandler(stdout_handler)
if settings.LoggingMode.STDOUT in settings.logging_modes:
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(formatter)
stdout_handler.addFilter(RequestIDLogFilter())
logger.addHandler(stdout_handler)
if settings.LoggingMode.LOGSTASH in settings.logging_modes:
# TODO: test this
raise Exception("Warning: untested")
logstash_handler = AsynchronousLogstashHandler(
**settings.logstash_settings)
logstash_handler.setFormatter(formatter)
logstash_handler.addFilter(RequestIDLogFilter())
logger.addHandler(logstash_handler)
if settings.LoggingMode.FLUENTD in settings.logging_modes:
# TODO: test this
raise Exception("Warning: untested")
fluentd_handler = handler.FluentHandler(
'pycroservices.follow',
**settings.fluentd_settings,
buffer_overflow_handler=overflow_handler)
fluentd_handler.setFormatter(formatter)
fluentd_handler.addFilter(RequestIDLogFilter())
logger.addHandler(fluentd_handler)
def get_logger(log_name="",
log_path='.',
single_log_file_size=1024 * 1024 * 600,
log_to_file=True,
backup_count=3,
host=None,
port=None,
database_path=None):
""":return a logger"""
if not os.path.exists(log_path):
try:
os.makedirs(log_path)
except Exception as e:
print(str(e))
logger = logging.getLogger("{}".format(log_name))
logger.setLevel(logging.DEBUG)
if log_name and log_to_file:
# file
log_file = "{}/{}.log".format(log_path, log_name)
fh = logging.handlers.RotatingFileHandler(
log_file, maxBytes=single_log_file_size, backupCount=backup_count)
color_formatter = ColoredFormatter(
fmt=
'%(asctime)s %(funcName)s[line:%(lineno)d] [%(levelname)s]: %(message)s'
)
fh.setFormatter(color_formatter)
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
# stdout
sh = logging.StreamHandler()
color_formatter = ColoredFormatter(
fmt=
'%(asctime)s %(funcName)s[line:%(lineno)d] [%(levelname)s]: %(message)s'
)
sh.setFormatter(color_formatter)
sh.setLevel(logging.DEBUG)
logger.addHandler(sh)
if host:
from logstash_async.handler import AsynchronousLogstashHandler
alh = AsynchronousLogstashHandler(host,
port,
database_path=database_path)
logger.addHandler(alh)
return logger
def initLogger(appConfig: dict):
# formatting for log stash
logstashFormatter = LogstashFormatter(
message_type='python-logstash',
extra=dict(application='mis_weekly_report_gen_service'))
# set app logger name and minimum logging level
appLogger = logging.getLogger('python-logstash-logger')
appLogger.setLevel(logging.INFO)
# configure console logging
streamHandler = logging.StreamHandler()
# streamHandler.setFormatter(logstashFormatter)
appLogger.addHandler(streamHandler)
# configure logstash logging
host = appConfig["logstashHost"]
port = appConfig["logstashPort"]
if not (pd.isna(host)) and not (pd.isna(port)):
logstashHandler = AsynchronousLogstashHandler(
host, port, database_path='logstash.db')
logstashHandler.setFormatter(logstashFormatter)
appLogger.addHandler(logstashHandler)
AppLogger.__instance = appLogger
def setup_logging(config):
root_logger = logging.getLogger()
if config.logging_level == 'DEBUG':
root_logger.setLevel(logging.DEBUG)
elif config.logging_level == 'INFO':
root_logger.setLevel(logging.INFO)
elif config.logging_level == 'WARNING':
root_logger.setLevel(logging.WARNING)
elif config.logging_level == 'ERROR':
root_logger.setLevel(logging.ERROR)
elif config.logging_level == 'CRITICAL':
root_logger.setLevel(logging.CRITICAL)
if config.external_logging:
root_logger.addHandler(
AsynchronousLogstashHandler(config.logging_host,
config.logging_port,
database_path=config.logging_local_db))
if USE_LOGGER:
import logging
from logstash_async.handler import AsynchronousLogstashHandler
# Setup elk stack
# host_logger = 'localhost'
host_logger = 'logstash'
port_logger = 5000
# Get you a test logger
test_logger = logging.getLogger('python-logstash-logger')
# Set it to whatever level you want - default will be info
test_logger.setLevel(logging.DEBUG)
# Create a handler for it
async_handler = AsynchronousLogstashHandler(host_logger, port_logger, database_path=None)
# Add the handler to the logger
test_logger.addHandler(async_handler)
###### Define possible cache statuses#####
SUCCESS = 1
CUSTOM_CACHE_FAILED = 2
REDIS_CACHE_FAILED = 3
BOTH_CACHES_FAILED = 4
##########################################
class ResponseCaching():
delta_expire = 15 # save request in cache for delta seconds
import logging
import sys
from logstash_async.handler import AsynchronousLogstashHandler
host = 'logstash'
port = 5005
test_logger = logging.getLogger('logstash')
test_logger.setLevel(logging.INFO)
test_logger.addHandler(
AsynchronousLogstashHandler(host, port, database_path='logstash.db'))
# If you don't want to write to a SQLite database, then you do
# not have to specify a database_path.
# NOTE: Without a database, messages are lost between process restarts.
# test_logger.addHandler(AsynchronousLogstashHandler(host, port))
test_logger.error('python-logstash-async: test logstash error message.')
test_logger.info('python-logstash-async: test logstash info message.')
test_logger.warning('python-logstash-async: test logstash warning message.')
# add extra field to logstash message
extra = {
'test_string': 'python version: ' + repr(sys.version_info),
'test_boolean': True,
'test_dict': {
'a': 1,
'b': 'c'
},
'test_float': 1.23,
'test_integer': 123
logger.info("*************")
try:
compute(" AND Lot:"+str(lot),str(lot))
except Exception as e:
logger.error("Unable to compute Lot "+str(lot),exc_info=True)
logger.info("<== "*10)
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(module)s - %(funcName)s: %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger()
lshandler=None
if os.environ["USE_LOGSTASH"]=="true":
logger.info ("Adding logstash appender")
lshandler=AsynchronousLogstashHandler("logstash", 5001, database_path='logstash_test.db')
lshandler.setLevel(logging.ERROR)
logger.addHandler(lshandler)
handler = TimedRotatingFileHandler("logs/"+MODULE+".log",
when="d",
interval=1,
backupCount=30)
logFormatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s')
handler.setFormatter( logFormatter )
logger.addHandler(handler)
logger.info("==============================")
logger.info("Starting: %s" % MODULE)
logger.info("Module: %s" %(VERSION))
def prepare_logstash(args):
"""Prepare logstash handler based on the configuration stored in the Tango
database.
:param args: process execution arguments
:type args: list<str>
.. note::
The prepare_logstash function has been included in Sardana
on a provisional basis. Backwards incompatible changes
(up to and including its removal) may occur if
deemed necessary by the core developers.
"""
log_messages = []
try:
from logstash_async.handler import AsynchronousLogstashHandler
except ImportError:
msg = ("Unable to import logstash_async. Skipping logstash "
+ "configuration...", )
log_messages.append(msg,)
return log_messages
def get_logstash_conf(dev_name):
try:
props = db.get_device_property(dev_name, "LogstashHost")
host = props["LogstashHost"][0]
except IndexError:
host = None
try:
props = db.get_device_property(dev_name, "LogstashPort")
port = int(props["LogstashPort"][0])
except IndexError:
port = None
try:
props = db.get_device_property(dev_name, "LogstashCacheDbPath")
cache_db_path = props["LogstashCacheDbPath"][0]
except IndexError:
cache_db_path = None
return host, port, cache_db_path
db = Database()
bin_name = args[0]
try:
instance_name = args[1]
except IndexError:
msg = ("Unknown %s instance name. " % bin_name
+ "Skipping logstash configuration...")
log_messages.append(msg, )
return log_messages
server_name = bin_name + "/" + instance_name
if bin_name in ["Pool", "MacroServer"]:
class_name = bin_name
dev_name = get_dev_from_class_server(db, class_name, server_name)[0]
host, port, cache = get_logstash_conf(dev_name)
else:
dev_name = get_dev_from_class_server(db, "Pool", server_name)[0]
host, port, cache = get_logstash_conf(dev_name)
if host is None:
dev_name = get_dev_from_class_server(db, "MacroServer",
server_name)[0]
host, port, cache = get_logstash_conf(dev_name)
if host is not None:
root = Logger.getRootLog()
handler = AsynchronousLogstashHandler(host, port, database_path=cache)
# don't use full path for program_name
handler._create_formatter_if_necessary()
_, handler.formatter._program_name = os.path.split(handler.formatter._program_name)
root.addHandler(handler)
msg = ("Log is being sent to logstash listening on %s:%d",
host, port)
log_messages.append(msg)
return log_messages
)
# Kafka initialize
consumer_obj = KafkaConsumer(
globals.RECEIVE_TOPIC,
bootstrap_servers=[globals.KAFKA_HOSTNAME + ':' + globals.KAFKA_PORT],
auto_offset_reset="earliest",
enable_auto_commit=True,
group_id="my-group",
value_deserializer=lambda x: json.loads(x.decode("utf-8")),
security_protocol="SASL_PLAINTEXT",
sasl_mechanism='PLAIN',
sasl_plain_username=globals.KAFKA_USERNAME,
sasl_plain_password=globals.KAFKA_PASSWORD
)
# Get you a test logger
error_logger = logging.getLogger('python-logstash-logger')
# Set it to whatever level you want - default will be info
error_logger.setLevel(logging.DEBUG)
# Create a handler for it
async_handler = AsynchronousLogstashHandler(globals.LOGSTASH_HOSTNAME, int(globals.LOGSTASH_PORT), database_path=None)
# Add the handler to the logger
error_logger.addHandler(async_handler)
def err_logger(msg):
msg = globals.RECEIVE_TOPIC + " " + msg
error_logger.error(msg)
def setup_bot(backend_name: str, logger, config, restore=None) -> ErrBot:
# from here the environment is supposed to be set (daemon / non daemon,
# config.py in the python path )
bot_config_defaults(config)
if hasattr(config, 'BOT_LOG_FORMATTER'):
format_logs(formatter=config.BOT_LOG_FORMATTER)
else:
format_logs(theme_color=config.TEXT_COLOR_THEME)
if hasattr(config, 'BOT_LOG_FILE') and config.BOT_LOG_FILE:
hdlr = logging.FileHandler(config.BOT_LOG_FILE)
hdlr.setFormatter(
logging.Formatter(
"%(asctime)s %(levelname)-8s %(name)-25s %(message)s"))
logger.addHandler(hdlr)
if hasattr(config, 'BOT_LOG_LOGSTASH') and config.BOT_LOG_LOGSTASH:
try:
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
except ImportError:
log.exception(
"You have BOT_LOG_LOGSTASH enabled, but I couldn't import modules "
"needed for Logstash integration. Did you install python-logstash-async? "
"(See https://python-logstash-async.readthedocs.io/en/latest/installation.html for installation instructions)"
)
exit(-1)
logger.addHandler(
AsynchronousLogstashHandler(config.BOT_LOG_LOGSTASH_HOST,
int(config.BOT_LOG_LOGSTASH_PORT),
database_path=None))
hdlr = logger.handlers[-1]
hdlr.setFormatter(
LogstashFormatter(
extra=dict(application=config.BOT_LOG_LOGSTASH_APP,
environment=config.BOT_LOG_LOGSTASH_ENV)))
if hasattr(config, 'BOT_LOG_SENTRY') and config.BOT_LOG_SENTRY:
sentry_integrations = []
try:
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
except ImportError:
log.exception(
"You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
"needed for Sentry integration. Did you install sentry-sdk? "
"(See https://docs.sentry.io/platforms/python for installation instructions)"
)
exit(-1)
sentry_logging = LoggingIntegration(
level=config.SENTRY_LOGLEVEL, event_level=config.SENTRY_EVENTLEVEL)
sentry_integrations.append(sentry_logging)
if hasattr(config,
'BOT_LOG_SENTRY_FLASK') and config.BOT_LOG_SENTRY_FLASK:
try:
from sentry_sdk.integrations.flask import FlaskIntegration
except ImportError:
log.exception(
"You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
"needed for Sentry integration. Did you install sentry-sdk[flask]? "
"(See https://docs.sentry.io/platforms/python/flask for installation instructions)"
)
exit(-1)
sentry_integrations.append(FlaskIntegration())
try:
if hasattr(config, 'SENTRY_TRANSPORT') and isinstance(
config.SENTRY_TRANSPORT, tuple):
mod = importlib.import_module(config.SENTRY_TRANSPORT[1])
transport = getattr(mod, config.SENTRY_TRANSPORT[0])
sentry_sdk.init(dsn=config.SENTRY_DSN,
integrations=sentry_integrations,
transport=transport)
else:
sentry_sdk.init(dsn=config.SENTRY_DSN,
integrations=sentry_integrations)
except ImportError:
log.exception(
f'Unable to import selected SENTRY_TRANSPORT - {config.SENTRY_TRANSPORT}'
)
exit(-1)
logger.setLevel(config.BOT_LOG_LEVEL)
storage_plugin = get_storage_plugin(config)
# init the botplugin manager
botplugins_dir = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR)
if not path.exists(botplugins_dir):
makedirs(botplugins_dir, mode=0o755)
plugin_indexes = getattr(config, 'BOT_PLUGIN_INDEXES',
(PLUGIN_DEFAULT_INDEX, ))
if isinstance(plugin_indexes, str):
plugin_indexes = (plugin_indexes, )
# Extra backend is expected to be a list type, convert string to list.
extra_backend = getattr(config, 'BOT_EXTRA_BACKEND_DIR', [])
if isinstance(extra_backend, str):
extra_backend = [extra_backend]
backendpm = BackendPluginManager(config, 'errbot.backends', backend_name,
ErrBot, CORE_BACKENDS, extra_backend)
log.info(f'Found Backend plugin: {backendpm.plugin_info.name}')
repo_manager = BotRepoManager(storage_plugin, botplugins_dir,
plugin_indexes)
try:
bot = backendpm.load_plugin()
botpm = BotPluginManager(
storage_plugin,
config.BOT_EXTRA_PLUGIN_DIR, config.AUTOINSTALL_DEPS,
getattr(config, 'CORE_PLUGINS',
None), lambda name, clazz: clazz(bot, name),
getattr(config, 'PLUGINS_CALLBACK_ORDER', (None, )))
bot.attach_storage_plugin(storage_plugin)
bot.attach_repo_manager(repo_manager)
bot.attach_plugin_manager(botpm)
bot.initialize_backend_storage()
# restore the bot from the restore script
if restore:
# Prepare the context for the restore script
if 'repos' in bot:
log.fatal('You cannot restore onto a non empty bot.')
sys.exit(-1)
log.info(f'**** RESTORING the bot from {restore}')
restore_bot_from_backup(restore, bot=bot, log=log)
print('Restore complete. You can restart the bot normally')
sys.exit(0)
errors = bot.plugin_manager.update_plugin_places(
repo_manager.get_all_repos_paths())
if errors:
log.error('Some plugins failed to load:\n' +
'\n'.join(errors.values()))
bot._plugin_errors_during_startup = "\n".join(errors.values())
return bot
except Exception:
log.exception("Unable to load or configure the backend.")
exit(-1)
LOGSTASH_DB_PATH = "python-elk-logstash.db"
LOGSTASH_TRANSPORT = "logstash_async.transport.BeatsTransport"
LOGSTASH_PORT = 5042
LOGSTASH_TRANSPORT = HttpTransport(
LOGSTASH_HOST,
LOGSTASH_PORT,
ssl_verify=False
timeout=5.0,
username="******",
password="******"
)
logstash_handler = AsynchronousLogstashHandler(
LOGSTASH_HOST,
LOGSTASH_PORT,
transport=LOGSTASH_TRANSPORT,
database_path=LOGSTASH_DB_PATH
)
#logstash_formatter = FlaskLogstashFormatter(
# message_type='python-logstash',
# extra_prefix='dev',
# extra=dict(application='example-app', environment='production'))
logstash_formatter = FlaskLogstashFormatter(metadata={"beat": "myapp"})
logstash_handler.setFormatter(logstash_formatter)
test_logger.addHandler(logstash_handler)
# logstash_handler.formatter = FlaskLogstashFormatter(metadata={"beat": "myapp"})
# app.logger.addHandler(logstash_handler)
@app.route('/')
def create_app(settings):
app = Eve(settings=settings, json_encoder=UUIDEncoder, validator=CustomValidator, auth=JWTAuth)
app.name = 'Materials service'
# We are using a document in the counters collection to generate sequential ids to be
# used for barcodes. Here we're "seeding" the collection with the inital document
with app.app_context():
current_app.data.driver.db \
.get_collection('counters') \
.update({'_id': 'barcode'}, {'$setOnInsert': {'seq': 0}}, upsert=True)
# Create a swagger.json
app.register_blueprint(swagger)
# Configure swagger ui to display docs using swagger.json @ SWAGGER_URL
app.register_blueprint(get_swaggerui_blueprint(SWAGGER_URL, API_URL), url_prefix=SWAGGER_URL)
login_manager = LoginManager()
login_manager.init_app(app)
@login_manager.user_loader
def load_user(email):
print email
return User(email)
# Application hooks
def set_uuid(resource_name, items):
for item in items:
item['_id'] = str(uuid.uuid4())
app.on_insert += set_uuid
# Containers hooks
def set_barcode_if_not_present(containers):
for container in containers:
if 'barcode' not in container:
result = app.data.driver.db.counters.find_one_and_update(
{'_id': 'barcode'},
{'$inc': {'seq': 1}},
return_document=ReturnDocument.AFTER)
container['barcode'] = 'AKER-%s' % result['seq']
def insert_empty_slots(containers):
for container in containers:
addresser = Addresser(container['num_of_rows'],
container['num_of_cols'],
bool(container.get('row_is_alpha')),
bool(container.get('col_is_alpha')))
slots = container.get('slots')
if not slots:
container['slots'] = [{'address': address} for address in addresser]
else:
definedaddresses = {slot['address'] for slot in container['slots']}
for address in addresser:
if address not in definedaddresses:
slots.append({'address': address})
app.on_insert_containers += set_barcode_if_not_present
app.on_insert_containers += insert_empty_slots
# Materials hooks
def set_owner_id(materials):
for material in materials:
if not material.get("owner_id"):
material["owner_id"] = current_user.id
app.on_insert_materials += set_owner_id
# Very rudimentary validation method... just for development!
@app.route('/materials/validate', methods=['POST'])
def validate(**lookup):
if 'materials' not in request.json:
abort(422)
if (validate_existence(request.json['materials'])):
return "ok"
else:
return "not ok - some materials not found"
def validate_existence(materials):
validation_set = set(materials)
result_set = set()
for material in app.data.driver.db.materials.find(
{'_id': {'$in': materials}}, {'_id': 1}):
result_set.add(material['_id'])
difference = validation_set - result_set
return not difference
@app.route('/materials/verify_ownership', methods=['POST'])
def verify_ownership(**lookup):
materials = request.json.get('materials')
owner_id = request.json.get('owner_id')
if materials is None or not owner_id:
abort(422)
if len(materials) == 0:
# If materials is an empty list, then the check is logically
# successful
return Response(status=200, mimetype="application/json")
if not validate_existence(materials):
abort(422, description="There was at least one material that did not exist")
find_args = {
'$and': [
{'_id': {'$in': materials}},
{'owner_id': {'$ne': owner_id}}
]
}
materials_cursor = app.data.driver.db.materials.find(find_args)
if materials_cursor.count() > 0:
response_body = json.dumps({
"_status": "ERR",
"_error": "{0} material(s) do not belong to {1}".format(materials_cursor.count(),
owner_id),
"_issues": [material['_id'] for material in materials_cursor]
})
return Response(status=403, response=response_body, mimetype="application/json")
return Response(status=200, mimetype="application/json")
def cerberus_to_json_list(schema, quality):
return [key for key, value in schema.iteritems() if value.get(quality)]
def cerberus_to_json_change_type_for_datetime(schema):
for value in schema.itervalues():
if value['type'] == 'datetime':
value['type'] = 'string'
value['format'] = 'date'
def cerberus_to_json_filter_parameters(schema, filter_list):
for key in filter_list:
schema.pop(key, None)
def cerberus_to_json_change_allowed_with_one_of(schema):
for value in schema.itervalues():
if 'allowed' in value:
value['enum'] = value['allowed']
del value['enum']
def cerberus_to_json_only_id_is_required(schema):
for key, value in schema.iteritems():
if key == '_id':
value['required'] = True
elif value.get('required'):
value['required'] = False
def amend_required_order(required):
if 'supplier_name' in required and required[0] != 'supplier_name':
required.remove('supplier_name')
required.insert(0, 'supplier_name')
def form_field_order(field_name):
"""This function describes the order for the fields shown on the submission
form, with any unspecified fields being displayed after the sorted ones."""
return FORM_FIELD_ORDER.get(field_name, len(FORM_FIELD_ORDER))
def cerberus_to_json_schema(schema_obj, patch=False):
filter_list = ['meta', 'parent', 'ancestors']
if not patch:
filter_list.append('_id')
schema = copy.deepcopy(schema_obj)
cerberus_to_json_change_type_for_datetime(schema)
cerberus_to_json_filter_parameters(schema, filter_list)
if patch:
cerberus_to_json_only_id_is_required(schema)
cerberus_to_json_change_allowed_with_one_of(schema)
required = cerberus_to_json_list(schema, 'required')
searchable = cerberus_to_json_list(schema, 'searchable')
amend_required_order(required)
show_on_form = cerberus_to_json_list(schema, 'show_on_form')
show_on_form.sort(key=form_field_order)
return {'type': 'object',
'properties': schema,
'required': required,
'searchable': searchable,
'show_on_form': show_on_form}
@app.route('/containers/json_schema', methods=['GET'])
def containers_json_schema(**lookup):
return json_schema_request('containers')
@app.route('/materials/json_schema', methods=['GET'])
def materials_json_schema(**lookup):
return json_schema_request('materials')
# Deprecated in favour of json_schema
@app.route('/materials/schema', methods=['GET'])
def bulk_schema(**lookup):
return json_schema_request('materials')
@app.route('/materials/json_patch_schema', methods=['GET'])
def materials_json_patch_schema(**lookup):
return json_schema_request('materials', True)
def json_schema_request(model_name, patch=False):
schema_obj = cerberus_to_json_schema(current_app.config['DOMAIN'][model_name]['schema'],
patch)
schema_str = json.dumps(schema_obj, default=json_util.default)
return Response(response=schema_str, status=200, mimetype="application/json")
def process_where(where, in_date_value=False):
if not where:
return where
if isinstance(where, dict):
for k, v in where.iteritems():
where[k] = process_where(v, in_date_value or k == 'date_of_receipt')
elif isinstance(where, (list, tuple)):
return [process_where(x, in_date_value) for x in where]
elif in_date_value and isinstance(where, basestring):
try:
return str_to_date(where)
except ValueError:
return where
return where
def _bulk_find(resource, args):
find_args = {
'filter': process_where(args.get('where')),
'projection': args.get('projection'),
}
try:
limit = max(int(args['max_results']), 0)
except (ValueError, KeyError):
limit = 0
if limit:
find_args['limit'] = limit
try:
page = max(int(args['page']), 1)
except (ValueError, KeyError):
page = 1
if limit and page > 1:
find_args['skip'] = limit*(page-1)
try:
find_args['sort'] = [(args['sort_by'], args['sort_order'])]
except:
find_args['sort'] = None
cursor = app.data.driver.db[resource].find(**find_args)
total = cursor.count()
pages = ((total + limit-1) // limit) if limit else 1
items = list(cursor)
meta = {'max_results': limit, 'total': total, 'page': page}
links = {}
if page > 1:
links['prev'] = {'page': (page-1)}
if page < pages:
links['next'] = {'page': (page+1)}
links['last'] = {'page': pages}
for item in items:
for k, v in item.iteritems():
if isinstance(v, datetime):
# date_to_str converts a datetime value to the format defined in the
# configuration file
item[k] = date_to_str(v)
if isinstance(v, unicode):
item[k] = str(v)
msg = {'_items': items, '_meta': meta, '_links': links}
msg_json = json.dumps(msg, default=json_util.default)
return Response(response=msg_json,
status=200,
mimetype="application/json")
@app.route('/materials/search', methods=['POST'])
def bulk_find_materials(**lookup):
return _bulk_find('materials', request.json)
@app.route('/containers/search', methods=['POST'])
def bulk_find_containers(**lookup):
return _bulk_find('containers', request.json)
# Logging
app.logger.setLevel(app.config.get('LOGGING_LEVEL', logging.WARNING))
# enable logging to 'app.log' file
log_handlers = [
logging.FileHandler('app.log'),
logging.StreamHandler(sys.stdout),
]
# set a custom log format, and add request
# metadata to each log line
for handler in log_handlers:
handler.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)s: %(message)s '
'[in %(filename)s:%(lineno)d] -- ip: %(clientip)s, '
'url: %(url)s, method: %(method)s'))
for handler in log_handlers:
app.logger.addHandler(handler)
def log_request_start(resource, request, lookup=None):
message = "%s resource=%r, request=%r" % (request.method, resource, request)
app.logger.info(message)
app.logger.info("Request data:\n"+request.data)
def log_request_end(resource, request, response):
message = "%s resource=%r, request=%r, response=%r" % (request.method,
resource,
request,
response)
app.logger.info(message)
if response:
app.logger.debug("Response data:\n"+response.data)
def logstash_logger(resource, request, response):
# Something similar to lograge for Rails apps
# [200] GET /materials?available=true (materials)
message = "[%d] %s %s (%r)" % (response.status_code, request.method, request.full_path, resource)
app.logger.info(message)
if app.config.get('LOGSTASH_ENABLE') is True:
logstash_handler = AsynchronousLogstashHandler(
app.config.get('LOGSTASH_HOST'),
app.config.get('LOGSTASH_PORT'),
database_path='logstash.db', transport='logstash_async.transport.UdpTransport'
)
app.logger.addHandler(logstash_handler)
for method in 'GET POST PATCH PUT DELETE'.split():
if app.config.get('LOGSTASH_ENABLE'):
events = getattr(app, 'on_post_'+method)
events += logstash_logger
else:
events = getattr(app, 'on_pre_'+method)
events += log_request_start
events = getattr(app, 'on_post_'+method)
events += log_request_end
return app
from flask import Flask, jsonify
from pymongo import MongoClient
from flask_login import LoginManager
import logging
from logstash_async.handler import AsynchronousLogstashHandler
import time
app = Flask(__name__)
app.secret_key = 'secret secret'
client = MongoClient('mongodb')
db = client.test_db
users = db.users
deleted_users = db.deleted_users
manager = LoginManager(app)
devices = db.devices
host = 'logstash'
port = 5000
logger = logging.getLogger('simple-app')
logger.setLevel(logging.DEBUG)
async_handler = AsynchronousLogstashHandler(host, port, database_path=None)
logger.addHandler(async_handler)
startup_time = time.time()
from . import routes
def __init__(self, level=logging.DEBUG):
self.xpr_config = XprConfigParser(
config_file_path=XprConfigParser.DEFAULT_CONFIG_PATH_XPR_LOG)
if self.xpr_config[self.LOGGING_SECTION][self.FIND_CONFIG_RECURSIVE]:
self.xpr_config = self.load_config("xpr")
self.name = self.xpr_config[self.PROJECT_NAME]
super(XprLogger, self).__init__(self.name)
self.setLevel(level)
logger_formatter = XprCustomFormatter(
self.xpr_config[self.LOGGING_SECTION][self.FORMATTER])
logstash_formatter = XprLogstashCustomFormatter(
self.xpr_config[self.LOGGING_SECTION][self.FORMATTER])
log_folder = os.path.expanduser(
self.xpr_config[self.LOGGING_SECTION][self.LOGS_FOLDER_PATH])
if not os.path.exists(log_folder):
try:
os.makedirs(log_folder, 0o755)
except IOError as err:
print(
"Permission Denied to create logs folder at the specidied directory. \n{}".format(
str(err)))
# Adding file handler for levels below warning
try:
if self.xpr_config[self.LOGGING_SECTION][self.LOGGING_FILE_BOOL]:
try:
wfh = logging.FileHandler(os.path.join(
log_folder,
'.'.join((self.xpr_config[self.PROJECT_NAME], "log"))), 'w')
except IOError as err:
print("Permission denied to create log files. "
"Saving log files in base directory . \n{}".format(
str(err)))
wfh = logging.FileHandler(
os.path.join(os.path.expanduser("~"),
'.'.join((self.xpr_config[
self.PROJECT_NAME], "log"))),
'w')
wfh.setFormatter(logger_formatter)
wfh.setLevel(logging.DEBUG)
self.addHandler(wfh)
except Exception as err:
print("Unable to add file handler to logger. \n{}".format(str(err)))
raise err
# Adding file handler for levels more critical than warning
try:
if self.xpr_config[self.LOGGING_SECTION][self.LOGGING_FILE_BOOL]:
try:
efh = logging.FileHandler(os.path.join(
log_folder,
'.'.join((self.xpr_config[self.PROJECT_NAME], "err"))), 'w')
except IOError as err:
print("Permission denied to create log files. "
"Saving log files in base directory . \n{}".format(
str(err)))
efh = logging.FileHandler(
os.path.join(os.path.expanduser("~"),
'.'.join((self.xpr_config[
self.PROJECT_NAME], "err"))),
'w')
efh.setFormatter(logger_formatter)
efh.setLevel(logging.ERROR)
self.addHandler(efh)
except Exception as err:
print(
"Unable to add file handler to logger . \n{}".format(str(err)))
raise err
# Adding logstash logging handler
try:
if self.xpr_config[self.LOGGING_SECTION][
self.LOGGING_LOGSTASH_BOOL]:
cache_filename = ""
if self.xpr_config[self.LOGGING_SECTION][
self.LOGSTASH_CACHE_BOOL]:
cache_filename = os.path.join(
log_folder, "cache.persistence")
lh = AsynchronousLogstashHandler(
host=self.xpr_config[self.LOGGING_SECTION][
self.LOGSTASH_HOST],
port=self.xpr_config[self.LOGGING_SECTION][
self.LOGSTASH_PORT],
database_path=cache_filename)
lh.setFormatter(logstash_formatter)
self.addHandler(lh)
except Exception as err:
print("Unable to add logstash handler to logger. \n{}".format(
str(err)))
raise err
# file services object
file_check_obj = FileCheck()
file_extract_obj = FileExtract()
file_convert_obj = FileConvert()
# logger
logging.basicConfig(
format=
'%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
datefmt='%Y-%m-%d:%H:%M:%S',
level=logging.DEBUG)
# Get you a test logger
logger = logging.getLogger('python-logstash-logger')
# Set it to whatever level you want - default will be info
logger.setLevel(logging.DEBUG)
# Create a handler for it
async_handler = AsynchronousLogstashHandler(globals.LOGGER_SERVER_HOST,
int(globals.LOGGER_SERVER_PORT),
database_path=None)
# Add the handler to the logger
logger.addHandler(async_handler)
def send_log_msg(msg, error=False):
msg = "INDEXING_MAIN " + msg
if error:
logger.error(msg)
else:
logger.info(msg)
def prepare_logstash(args):
"""Prepare logstash handler based on the configuration stored in the Tango
database.
:param args: process execution arguments
:type args: list<str>
.. note::
The prepare_logstash function has been included in Sardana
on a provisional basis. Backwards incompatible changes
(up to and including its removal) may occur if
deemed necessary by the core developers.
"""
log_messages = []
try:
from logstash_async.handler import AsynchronousLogstashHandler
except ImportError:
msg = ("Unable to import logstash_async. Skipping logstash " +
"configuration...", )
log_messages.append(msg, )
return log_messages
def get_logstash_conf(dev_name):
try:
props = db.get_device_property(dev_name, "LogstashHost")
host = props["LogstashHost"][0]
except IndexError:
host = None
try:
props = db.get_device_property(dev_name, "LogstashPort")
port = int(props["LogstashPort"][0])
except IndexError:
port = None
try:
props = db.get_device_property(dev_name, "LogstashCacheDbPath")
cache_db_path = props["LogstashCacheDbPath"][0]
except IndexError:
cache_db_path = None
return host, port, cache_db_path
db = Database()
bin_name = args[0]
try:
instance_name = args[1]
except IndexError:
msg = ("Unknown %s instance name. " % bin_name +
"Skipping logstash configuration...")
log_messages.append(msg, )
return log_messages
server_name = bin_name + "/" + instance_name
if bin_name in ["Pool", "MacroServer"]:
class_name = bin_name
dev_name = get_dev_from_class_server(db, class_name, server_name)[0]
host, port, cache = get_logstash_conf(dev_name)
else:
dev_name = get_dev_from_class_server(db, "Pool", server_name)[0]
host, port, cache = get_logstash_conf(dev_name)
if host is None:
dev_name = get_dev_from_class_server(db, "MacroServer",
server_name)[0]
host, port, cache = get_logstash_conf(dev_name)
if host is not None:
root = Logger.getRootLog()
handler = AsynchronousLogstashHandler(host, port, database_path=cache)
# don't use full path for program_name
handler._create_formatter_if_necessary()
_, handler.formatter._program_name = os.path.split(
handler.formatter._program_name)
root.addHandler(handler)
msg = ("Log is being sent to logstash listening on %s:%d", host, port)
log_messages.append(msg)
return log_messages
# formatting for log stash
logstash_formatter = LogstashFormatter(message_type='python-logstash',
extra_prefix='dev',
extra=dict(application='example-app',
environment='production'))
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
streamHandler = logging.StreamHandler()
# streamHandler.setFormatter(logstash_formatter)
test_logger.addHandler(streamHandler)
logstashHandler = AsynchronousLogstashHandler(host,
port,
database_path='logstash.db')
logstashHandler.setFormatter(logstash_formatter)
test_logger.addHandler(logstashHandler)
# If you don't want to write to a SQLite database, then you do
# not have to specify a database_path.
# NOTE: Without a database, messages are lost between process restarts.
# test_logger.addHandler(AsynchronousLogstashHandler(host, port))
test_logger.error('python-logstash-async: test logstash error message.')
test_logger.info('python-logstash-async: test logstash info message.')
test_logger.warning('python-logstash-async: test logstash warning message.')
# add extra field to logstash message
extra = dict(test_string="python_version-" + repr(sys.version_info),
import logging
from flask import Flask
app = Flask(__name__)
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
# Create the logger and set it's logging level
logger = logging.getLogger("logstash")
logger.setLevel(logging.ERROR)
# Create the handler
handler = AsynchronousLogstashHandler(
host='172.18.0.3',
port=5042,
ssl_enable=True,
ssl_verify=False,
database_path='python-elk-logstash.db')
# Here you can specify additional formatting on your log record/message
formatter = LogstashFormatter()
handler.setFormatter(formatter)
# Assign handler to the logger
logger.addHandler(handler)
@app.route('/')
def hello_world():
logger.info("Hello there")
return 'Hello, World!'
# DEFAULT_DOWNLOADS_FOLDER = Path("media/downloads/")
# IMPORT ENV VARIABLES
# config_path = APP_ROOT / "config_environment.py"
# config_path_template = APP_ROOT / "TEMPLATE_config_environment.py"
config_path = os.path.join(APP_ROOT, "config_environment.py")
config_path_template = os.path.join(APP_ROOT, "TEMPLATE_config_environment.py")
# Sending loggs to logit.io
# Create the logger and set it's logging level
logger_logit = logging.getLogger("logstash")
logger_logit.setLevel(logging.ERROR)
# Create the handler
handler = AsynchronousLogstashHandler(
host='fc652908-5b50-4887-8af2-89286e6febe1-ls.logit.io',
port=17326,
ssl_enable=True,
ssl_verify=False,
database_path='')
# Here you can specify additional formatting on your log record/message
formatter = LogstashFormatter(message_type='python-logstash',
extra_prefix='extra',
extra=dict(mikrostoritev='imageUpload',
okolje='production'))
handler.setFormatter(formatter)
# Assign handler to the logger
logger_logit.addHandler(handler)
try:
# if config_path.exists():
if os.path.exists(config_path):
exec(open(config_path).read())
# -------------------------------------------
# Logging setup
# -------------------------------------------
# Create the logger and set it's logging level
logger = logging.getLogger("logstash")
logger.setLevel(logging.INFO)
log_endpoint_uri = str(environ["LOGS_URI"]).strip()
log_endpoint_port = int(environ["LOGS_PORT"].strip())
# Create the handler
handler = AsynchronousLogstashHandler(
host=log_endpoint_uri,
port=log_endpoint_port,
ssl_enable=True,
ssl_verify=False,
database_path='')
# Here you can specify additional formatting on your log record/message
formatter = LogstashFormatter()
handler.setFormatter(formatter)
# Assign handler to the logger
logger.addHandler(handler)
# -------------------------------------------
# Models
# -------------------------------------------
class User(db.Model):
__tablename__ = 'users'
