def __init__(self, name: str, app: str, pod: str, container: str,
namespace: str):
threading.Thread.__init__(self)
self.name = name
self.pod = pod
self.namespace = namespace
self.container = container
self.app = app
handler = None
try:
handler = logging_loki.LokiHandler(
url="%s/loki/api/v1/push" % os.environ['LOKI_URL'],
tags={"app": "%s-%s" % (self.namespace, self.app)},
version="1",
)
def handleError(er):
return
handler.handleError = handleError
except: #noqa
sys.stderr.write('Error in creating LokiHandler %s-%s-%s\n' %
(self.namespace, self.app, self.container))
self.logger = logging.getLogger("%s-%s" % (self.namespace, self.app))
if (not handler):
sys.stderr.write('Handler not added for %s-%s-%s\n' %
(self.namespace, self.app, self.container))
else:
self.logger.addHandler(handler)
def _enable_loki_logging(self):
loki_username = self.config.get("username", None)
loki_password = self.config.get("password", None)
auth = None
if loki_username and loki_password:
auth = (loki_username, loki_password)
if self.config.get("async", False):
mode = "async"
handler = logging_loki.LokiQueueHandler(
Queue(-1),
url=self.config.get("url"),
tags={"project": self.context.get_meta("project_name", "Unnamed Project")},
auth=auth,
)
else:
mode = "sync"
handler = logging_loki.LokiHandler(
url=self.config.get("url"),
tags={"project": self.context.get_meta("project_name", "Unnamed Project")},
auth=auth,
)
logging.getLogger("").addHandler(handler)
log.info(
"Enabled Loki logging in %s mode for Dusty {}".format(
pkg_resources.require("dusty")[0].version
),
mode
)
def init_app(cls, app):
Config.init_app(app)
import logging
import urllib
# Log to docker logs
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.INFO)
# Error log to slack
if cls.slack_response == 403:
app.logger.info("****** Slack webhook is not correct ***********")
else:
if cls.SLACK_LOGGING:
try:
from slack_log_handler import SlackLogHandler
slack_handler = SlackLogHandler(
webhook_url=cls.SLACK_WEBHOOK_URL,
channel=cls.SLACK_CHANNEL,
format=
'%(levelname)s - %(asctime)s - %(name)s - %(message)s')
slack_handler.setLevel(logging.ERROR)
app.logger.addHandler(slack_handler)
except urllib.error.HTTPError:
app.logger.info(
"******************** Slack webhook is not working ******************"
)
# Log to loki server
if cls.LOKI_LOGGING:
import logging_loki
tag = os.environ.get('TAGS')
try:
handler = logging_loki.LokiHandler(
url="{0}/loki/api/v1/push".format(cls.LOKI_URL),
tags={"application": tag},
auth=(cls.LOKI_USERNAME, cls.LOKI_PASSWORD),
version="1",
)
app.logger.setLevel(logging.INFO)
app.logger.addHandler(handler)
except:
pass
def enable_loki_logging(default_config):
""" Start logging to Loki """
if not default_config.get("loki", None):
return
loki_url = default_config["loki"].get("url", None)
if not loki_url:
logging.warning("No Loki URL in config. Skipping Loki logging")
return
loki_username = default_config["loki"].get("username", None)
loki_password = default_config["loki"].get("password", None)
auth = None
if loki_username and loki_password:
auth = (loki_username, loki_password)
if default_config["loki"].get("async", False):
mode = "async"
handler = logging_loki.LokiQueueHandler(
Queue(-1),
url=loki_url,
tags={"project": default_config.get("project_name", "unknown")},
auth=auth,
)
else:
mode = "sync"
handler = logging_loki.LokiHandler(
url=loki_url,
tags={"project": default_config.get("project_name", "unknown")},
auth=auth,
)
logging.getLogger("").addHandler(handler)
logging.info("Enabled Loki logging in %s mode", mode)
import logging
import logging_loki
import os
handler = logging_loki.LokiHandler(
url="http://localhost:3100/loki/api/v1/push",
tags={"movie_database": "top_revenue_movies"},
# auth=("username", "password"),
version="1",
)
logger = logging.getLogger("top_revenue_movies_python_script")
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.info("testing!")
top = 1000
min_budget_limit = 1000
wiki_df_columns = ['title', 'link', 'abstract']
raw_csv_columns = [
'title', 'budget', 'release_date', 'revenue', 'vote_average',
'vote_average', 'production_companies'
]
csv_filtered_columns = {
'budget': 'int',
'genres': 'string',
'title': 'string',
import logging
import logging_loki
handler = logging_loki.LokiHandler(
url="http://127.0.0.1:3100/loki/api/v1/push",
# 方便 Query
tags={"application": "Demo"},
version="1",
)
logger = logging.getLogger("logger")
logger.addHandler(handler)
for i in range(1, 10):
logger.error(
f"ERROR Test - {i}",
extra={"tags": {
"service": "tunnel"
}},
)
"""
export LOKI_ADDR=http://localhost:3100
logcli query '{application="Demo"}'
"""
def report_errors(aggregated_errors, errors, args):
report_types = []
with open(PATH_TO_CONFIG, "rb") as f:
config = yaml.load(f.read())
if config:
report_types = list(config.keys())
if report_types.__contains__('loki'):
loki_host = config['loki'].get("host")
loki_port = config['loki'].get("port")
if not all([loki_host, loki_port]):
print("Loki configuration values missing, proceeding "
"without Loki")
else:
loki_url = "{}:{}/api/prom/push".format(loki_host, loki_port)
handler = logging_loki.LokiHandler(
url=loki_url,
tags={"Test": args['simulation']},
)
error_message = "Error key: {};; UTC Time: {};; Request name: {};; Method: {};; Response code: {};;" \
" URL: {};; Error message: {};; Request params: {};; Headers: {};; Response body: {};;"
logger = logging.getLogger("error-logger")
logger.addHandler(handler)
for error in errors:
logger.error(
error_message.format(str(error['error_key']),
str(error['Time']),
str(error['Request name']),
str(error['Method']),
str(error['Response code']),
str(error['Request URL']),
str(error['Error_message']),
str(error['Request_params']),
str(error['Request headers']),
str(error['Response'])), )
rp_service = None
if report_types.__contains__('reportportal'):
rp_project = config['reportportal'].get("rp_project_name")
rp_url = config['reportportal'].get("rp_host")
rp_token = config['reportportal'].get("rp_token")
rp_launch_name = config['reportportal'].get("rp_launch_name")
if not (rp_project and rp_url and rp_token and rp_launch_name):
print("ReportPortal configuration values missing, proceeding "
"without report portal integration ")
else:
rp_service = ReportPortal(aggregated_errors, args, rp_url,
rp_token, rp_project, rp_launch_name)
if rp_service:
rp_service.my_error_handler(sys.exc_info())
rp_service.report_errors()
jira_service = None
if report_types.__contains__('jira'):
jira_url = config['jira'].get("url", None)
jira_user = config['jira'].get("username", None)
jira_pwd = config['jira'].get("password", None)
jira_project = config['jira'].get("jira_project", None)
jira_assignee = config['jira'].get("assignee", None)
jira_issue_type = config['jira'].get("issue_type", 'Bug')
jira_lables = config['jira'].get("labels", '')
jira_watchers = config['jira'].get("watchers", '')
jira_epic_key = config['jira'].get("epic_link", None)
if not (jira_url and jira_user and jira_pwd and jira_project
and jira_assignee):
print(
"Jira integration configuration is messed up, proceeding without Jira"
)
else:
jira_service = JiraWrapper(jira_url, jira_user, jira_pwd,
jira_project, jira_assignee,
jira_issue_type, jira_lables,
jira_watchers, jira_epic_key)
if jira_service:
jira_service.connect()
if jira_service.valid:
for error in aggregated_errors:
issue_hash = get_hash_code(aggregated_errors[error], args)
description = create_description(aggregated_errors[error],
args)
jira_service.create_issue(
aggregated_errors[error]['Request name'], 'Major',
description, issue_hash)
else:
print("Failed connection to Jira or project does not exist")
import logging
import logging_loki
import sys
from classes.config import Config
config = Config()
logger = logging.getLogger('hh.bumper')
logger.setLevel(logging.INFO)
# STDOUT
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
# LOKI
if config.get_loki_host() and config.get_loki_pass() and config.get_loki_user(
):
handler = logging_loki.LokiHandler(
url=config.get_loki_host(),
tags={"job": "hh.bumper"},
auth=(config.get_loki_user(), config.get_loki_pass()),
version="1",
)
logger.addHandler(handler)