def metrics_failure_summary(self, df, **kwargs):
tablefmt = kwargs.get('tablefmt', 'html')
self.mdLogger.
print("\n## {} metrics: last {} records from {} to {} , now is {} in {}\n\n"
.format(kwargs.get('name'),
kwargs.get('size'),
kwargs.get('beginTime'),
kwargs.get('endTime'),
FileLogger.getCurrentTimeStr('%Y-%m-%d %H:%M:%S'),
self.env))
if (kwargs.get('columns')):
mdTable = tabulate(df, headers=kwargs.get('columns'), tablefmt=tablefmt)
else:
mdTable = tabulate(df, headers='keys', tablefmt=tablefmt)
self.mdLogger.append(mdTable)
return mdTable
def triggerAlerts(category, env, interval, emailList):
alertItems = tool.alertConfig.get_alert_config_items()
for key, value in alertItems:
print(key, value)
metricsChecker = MetricsChecker(category, env, key, value)
if not metricsChecker.isEnabled:
print("{} is not enabled".format(key))
continue
if metricsChecker.isSnoozed():
print("{} is not snoozed".format(key))
continue
df, totalErrCnt = metricsChecker.queryErrorRecords(interval, size=10)
queryErrCnt = df.index.size
alertFlag = metricsChecker.needTriggeredByCount(totalErrCnt)
if (metricsChecker.totalQuery):
totalCount = metricsChecker.queryTotalCount('1d')
alertFlag = alertFlag or metricsChecker.needTriggeredByRatio(totalErrCnt, totalCount)
print("queryErrCnt={}, totalErrCnt={}, totalCount={}, alertFlag={}".format(queryErrCnt, totalErrCnt,
totalCount, alertFlag))
if (alertFlag):
print("trigger alert for {}".format(key))
timeStr = FileLogger.getCurrentTimeStr('%Y-%m-%dT%H:%M:%S')
metricsChecker.reporter.title = 'Metrics_Alert_Report_at_%s' % timeStr
metricsChecker.makeFailureSummary(df, interval)
metricsChecker.reporter.mdLogger.
print("\n* Query string: {}".format(metricsChecker.failureQuery))
metricsChecker.reporter.mdLogger.
print("* Queried Metrics Error Count={}, Total Metrics Error Count={}, Total Metrics Count={}".format(
queryErrCnt, totalErrCnt, totalCount))
metricsChecker.reporter.sendEmail(emailList)
def __init__(self,
filename,
title=None,
isShow=True,
isBlock=False,
needChangePath=False,
**kwargs):
self.filename = filename
self.title = title
self.isShow = isShow
self.isBlock = isBlock
self.defaultStyle = ['bs-', 'ro-', 'y^-']
plt.style.use('seaborn-whitegrid')
self.fig = plt.figure()
if (needChangePath):
self.pngFile = "./logs/{}/{}.png".format(
FileLogger.getCurrentTimeStr(),
os.path.basename(self.filename))
else:
self.pngFile = filename
def make_report_files(self, title):
baseFileName = "./logs/{}_{}".format(self.title, FileLogger.getCurrentTimeStr('%Y%m%d%H%M%S'))
self.mdLogger = FileLogger.MarkdownLogger(baseFileName + '.md')
self.csvLogger = FileLogger.CsvLogger(baseFileName + '.csv', None, False)
self.csvFiles.append(self.csvLogger.getFilePath())