def calculatePerformanceReference(connDbMaster,
start,
end,
resetReference=False):
cursor = connDbMaster.cursor()
if resetReference:
masterdao.resetLogPathDurationData(cursor)
connDbMaster.commit()
hosts = masterdao.findHosts(cursor)
for host in hosts:
components = masterdao.findComponentsByHostId(cursor, host['id'])
for component in components:
logger.debug('Process:' + str(component))
rowsPaths = masterdao.findLogPathsByComponentId(
cursor, component['id'])
for rowpath in rowsPaths:
pathId = rowpath[0]
(count, avg, std, maxv) = masterdao.findReferencePathMeasures(
cursor, pathId, start, end, True)
masterdao.updateLogPathDurationData(cursor, pathId, count, avg,
std, maxv, start, end)
connDbMaster.commit()
def findComponentsByHostId(self, hostId):
"""
***********************************************************************
***********************************************************************
"""
#TODO AGENTiD O hOSTiD
return masterdao.findComponentsByHostId(self.cursor, hostId)
def getPerformanceData(self, start, end):
"""
***********************************************************************
***********************************************************************
"""
performanceData = []
hosts = masterdao.findHosts(self.cursor)
for host in hosts:
components = masterdao.findComponentsByHostId(
self.cursor, host['id'])
for component in components:
results = masterdao.findPerformanceMeasures(
self.cursor, component['id'], start, end)
performanceData.append({
'instance': component['name'],
'results': results
})
return {'start': start, 'end': end, 'performanceData': performanceData}
def processOutliers(connDbMaster, start, end):
cursor = connDbMaster.cursor()
hosts = masterdao.findHosts(cursor)
# hosts = [ masterdao.findHostByKey(cursor, "device") ]
for host in hosts:
components = masterdao.findComponentsByHostId(cursor, host['id'])
for component in components:
logger.debug('Process:' + str(component))
rowsPaths = masterdao.findLogPathsByComponentId(
cursor, component['id'])
for rowpath in rowsPaths:
pathId = rowpath[0]
(count, avg, std, maxv) = masterdao.findReferencePathMeasures(
cursor, pathId, start, end, True)
if avg == None or std == None: continue
if std == 0: std = 0.1
vmax = avg + 3 * std
masterdao.updatePathMeasureRefFlagByAvgRange(
cursor, pathId, start, end, vmax, False)
connDbMaster.commit()
# hosts = [ masterdao.findHostByKey(cursor, "device") ]
componentList = []
colnamesAnom = ['component', 'error', 'description']
errorTestList = []
errorTrainList = []
descriptionAnomList = []
colnamesRegr = ['component', 'mse', 'r2', 'description']
mseList = []
r2List = []
descriptionRegrList = []
for host in hosts:
components = masterdao.findComponentsByHostId(cursor, host['id'])
for component in components:
logger.debug('Process:' + str(component))
mTypes = masterdao.findMeasuresTypesByComponent(
cursor, component['id'])
df = pd.read_csv('/logmapper/data/arrange_' + component['key'] +
'.csv')
df.set_index('date', inplace=True)
componentList.append(component['name'])
dft = transform.selectAndTransformMetricsAndEvents(df, mTypes)
dfsvm, nuBest, gammaBest, errorMin = findBestSvmParameters(dft)
if nuBest:
logger.debug("Best: nu={:f} gamma={:f} errorMin={:f}".format(
def measureAndPredict(connDbMaster, start, end):
performance.calcPathsPerformance(connDbMaster, start, end)
cursor = connDbMaster.cursor()
hosts = masterdao.findHosts(cursor)
for host in hosts:
components = masterdao.findComponentsByHostId(cursor, host['id'])
for component in components:
logger.debug('Process:' + str(component))
mTypes = masterdao.findMeasuresTypesByComponent(
cursor, component['id'])
performance.calcPerformanceByComponent(connDbMaster,
component['id'],
component['hostId'], start,
end)
df = arrange.arrangeComponentData(cursor,
component['id'],
component['hostId'],
start,
end,
ref=None)
logger.debug("len(df)=" + str(len(df)))
anomalyPredicted = predictAnomalies(component['key'], df, mTypes)
for index, row in anomalyPredicted.iterrows():
resultId = masterdao.findPerfomanceMeasure(
cursor, index, component['id'])
if resultId:
masterdao.updateAnomalyPredicted(cursor, resultId['id'],
row['anomaly'])
else:
period = lmutil.calcTimeGroup(index)
masterdao.createPerfomanceMeasureWithAnomaly(
cursor, index, period, component['hostId'],
component['id'], row['anomaly'])
connDbMaster.commit()
perfPredicted = predictPerformance(component['key'], df, mTypes)
for index, row in perfPredicted.iterrows():
resultId = masterdao.findPerfomanceMeasure(
cursor, index, component['id'])
if resultId:
masterdao.updatePerformancePredicted(
cursor, resultId['id'], row['predicted'])
connDbMaster.commit()
# if len(df) > 5:
# anomalyPredictedDf = predictAnomalies(component['key'], df, mTypes)
# if anomalyPredictedDf == None or len(anomalyPredictedDf) == 0:
# logger.debug("No data, Continue")
# continue
# for i in range(len(df)):
# dfs = df.iloc[i:i+1]
# anomalyPredicted = predictAnomalies(component['key'], dfs, mTypes)
# perfPredicted = predictPerformance(component['key'], dfs, mTypes)
# if anomalyPredicted != None:
# masterdao.updatePerfomanceMeasure(cursor, dfs.index[0], component['id'], perfPredicted, anomalyPredicted)
connDbMaster.commit()