def __init__(self):
basicApi.d_print('TFIDF_analyzer: __init__()')
#initial analyzer
self.stopWordsFile = '%s/stopword.txt' % Env().getPath('TFIDF_HOME_PATH')
self.stopWordDic = {}
self.featureDic = {}
self.err = []
def deleteCR(self, cr):
basicApi.d_print('ManualRemoveItem:deleteCR()')
rtl = self.task.db.cateSummary.searchItem(cr)
itemList = rtl.getList()
for itemId in itemList:
self.udt.deleteTRD(itemId, cr)
self.task.db.cateSummary.removeItem([itemId])
def callProc(self, proc, args=()):
basicApi.d_print('common.DbCategory:callProc()')
sProc = '"%s"."%s.%s.procs::%s"' % (self.db.schema, self.db.basePath,
self.name, proc)
#print('%s'%(sProc))
#print(tuple(args))
return self.db.cur.callproc(sProc, tuple(args))
def parpareTaskData(self):
#1, Split whole jnl file into case blocks
#2, Save each blocks into local disk
#3, Write caseName to jnl maps into task object
#caseList = [(itemId, caseName), ...]
basicApi.d_print('AseSetup: parpareTaskData()')
for jnlPath, execPath in self.jnlPair:
self._fillExecOutDic(execPath)
for jnlCaseNum, block in self._jnlGenerator(jnlPath):
caseName = self._getCaseName(jnlCaseNum)
#Insert current failed test case information into DB
rtl = self.task.db.cateCycleInfo.insertItems([
self.task.tag, self.task.tagTime, self.task.branch,
self.task.platform, self.task.testSet, caseName,
self.task.workDir, self.task.user, '?'
])
#Get return itemId from DB
itemId = rtl.getProcRtId()
#Wrtie block data into disk with itemId
basicApi.writeFile(self.task.getJnlStoreName(itemId), block)
#Append caseList for current task
self.task.caseList.append((str(itemId), caseName))
#Log jnl paser info
basicApi.writeFile(os.path.join(self.aaLogDir, 'autoAnalysis.log'),
self.err)
pass
def run(self, task):
basicApi.d_print('common.Analyzer:run()')
#self.task = task
self.setTask(task)
self.preProcess()
self.analysing()
self.record()
def execute(self, assignDict):
basicApi.d_print('ReAssignUsr:execute()')
for testSet, userName in assignDict.items():
print('test_set: %s to user: %s' % (testSet, userName))
self.task.db.cateCycleInfo.updateTestSetAssignment(
[testSet, usrName])
pass
def __init__(self, clsList):
basicApi.d_print('common.Framework:__init__()')
self.clsNameList = []
for cls in clsList:
setattr(self, cls.name, cls())
self.clsNameList.append(cls.name)
pass
def execute(self, itemId, cr):
basicApi.d_print('ManualUpdateItem:execute()')
#Update user specified CR into database
self.updateDB(itemId, cr)
#We still need update this CR in TRD
self.updateTRD(itemId, cr)
pass
def preProcess(self):
basicApi.d_print('TFIDF_analyzer: preProcess()')
#Get feature from db
rtl = self.task.db.cateTFIDF.searchFeatureKey_tb()
self.featureDic = rtl.getDict()
#Fill the stop words dict
self._fillStopWordDic()
pass
def __init__(self):
basicApi.d_print('CNN_analyzer: __init__()')
#initial analyzer
self.checkpoint_dir = '%s/runs/checkpoints' % Env().getPath(
'CNN_HOME_PATH')
self.err = []
self.prediction_list = []
self.train_cr_order = '%s/runs/train_cr_order.txt' % Env().getPath(
'CNN_HOME_PATH')
def __init__(self, task):
basicApi.d_print('CycleCleanUp:__init__()')
self.task = task
#initial update and remove actions
self.mri = ManualRemoveItem(task)
self.mui = ManualUpdateItem(task)
self.upt = UpdateTRD(task)
self.qtsdb = AseAccess()
# Setup Quasr_trd executing PATH
self._setEnv()
def analysing(self):
basicApi.d_print('CNN_analyzer: analysing()')
if self.task.caseList == []:
#That means all test case pass
pass
else:
storeJnlList = self.task.getStoreJnlList()
input_list = self._filter_data(storeJnlList)
x_list = self._transform_vector(input_list)
self.prediction_list = self._prediction(x_list)
pass
def _loadConfig(self, fileName):
basicApi.d_print('common.Env:_loadConfig()')
fd = file(fileName, 'r')
cfgDict = yaml.load(fd)
#Process absolute path
for key, value in cfgDict['ABSOLUTE_PATH'].items():
self.configDict[key] = value
#Process relative path
for key, value in cfgDict['RELATIVE_PATH'].items():
self.configDict[key] = '%s/%s' % (self.homeDir, value)
pass
def __init__(self):
basicApi.d_print('AseSetup: __init__()')
self.err = []
self.tcfList = []
self.jnlPair = []
self.testSetWorkDir = ''
self.resultDir = ''
#Used to store execOut data
self.execResultDict = {}
#Used to store current Tcf full name
self.tcfFullName = ''
def execute(self, itemId, cr):
basicApi.d_print('ManualRemoveItem:execute()')
#Remove fake error message items from db
#self.task.db.cateSummary.removeItem([itemId])
#Do we need to check whether this itemId has CR attached.
#If yes, then we need to remove it from TRD when we remove it.
#after discussed with wanghao, we need to remove it from TRD
if cr != 'new' and cr != 'manual':
self.udt.deleteTRD(itemId, cr)
#Remove fake error message items from db
self.task.db.cateSummary.removeItem([itemId])
pass
def deleteTRD(self, itemId, cr):
basicApi.d_print('UpdateTRD:deleteTRD()')
rtl = self.task.db.cateCycleInfo.searchItemInfo(itemId)
(tag, platformBit, fullCaseName) = rtl.getTuple()
#sync test case attached CR from our data base to TRD
pathList = fullCaseName.split('/')
caseName = pathList[-1]
cmd = '/remote/quasr1/quasr/nightly/build/linux/bin/quasr_trd delete cr -case_name "%s" \
-tag %s -number %s -plat_and_bit %s' % (caseName, tag, cr,
platformBit)
return basicApi.quasrCmd(cmd)
pass
def execute(self):
basicApi.d_print('CycleCleanUp:execute()')
#Infact there are 2 jobs:
#1, Check all not processed items in current cycle, sync the data with TRD.
#2, Check all CR status, remove closed CR related items.
print('****************************************')
print(' Sync analysis result from TRD: ')
print('****************************************')
self.syncFromTRD()
print('\n')
print('****************************************')
print(' Check CR status on current branch: ')
print('****************************************')
self.removeCloseCR()
def updateModel(self, task):
if Env(
).homeDir == '/remote/asepw_archive2/grid/autoAnalyzer/gitAutoAnalyze':
#This is used for re-calculation
basicApi.d_print('AseAnalysis: updateModel()')
#Record recalculation action into db
task.db.cateSummary.insertRecalculateLog()
#Begin update
self.cnn.update(task)
self.tfidf.update(task)
else:
print('Re-calculate must be run under:')
print(
' /remote/asepw_archive2/grid/autoAnalyzer/gitAutoAnalyze')
pass
def __init__(self):
basicApi.d_print('HanaDbAccess: __init__()')
#Initial connection to database
self.conn = hanadbapi.connect(address="lsvxc0035.sjc.sap.corp",
port=30015,
user="******",
password="******")
self.cur = self.conn.cursor()
#Specify schema name and db path
self.schema = 'AUTO_ANALYZER'
self.basePath = 'aseAutoAnalyzer.db'
#Initial call db access method
self.cateCycleInfo = CycleInfo(self, 'cycleInfo')
self.cateTFIDF = TFIDF(self, 'tfidf')
self.cateCNN = CNN(self, 'cnn')
self.cateSummary = Summary(self, 'summary')
def analysing(self):
basicApi.d_print('TFIDF_analyzer: analysing()')
for itemId,CaseName in self.task.caseList:
jnlPath = self.task.getJnlStoreName(itemId)
block = basicApi.openFile(jnlPath, "r")
#Filter jnl block
filteBlock = _filteJnl(block)
#Simplify jnl block
extractedJnl = tfidf.stemming(filteBlock, self.stopWordDic)
msgVector = tfidf.featureGenerator(extractedJnl, self.featureDic)
#Check msgVector
if self._isZeroVec(msgVector):
text = 'ERROR: Extracted journal file is empty, please manually analysis it.'
self.err.append('%s\n%s\n'%(CaseName, text))
#Write log into database
self.task.db.cateTFIDF.insertNotAnalysis([itemId])
continue
#Find out all vectors
rtl = self.task.db.cateTFIDF.searchAllVecs(itemId)
allVecs = rtl.getDict()
#Translate vector into string
vecStr = basicApi.vec2Str(msgVector)
if allVecs == {}:
#Initial database
self.task.db.cateTFIDF.insertTfidfResult([itemId, vecStr, '?'])
continue
#Insert current vector into database and attach it to exist CR if it has
rtl = self.task.db.cateTFIDF.insertTfidfResult([itemId, vecStr, '?'])
if rtl.getProcRtId() == 1:
#If return value equal to 1 then means it is a new issue
#First get all data from database
if allVecs:
#Fuzzy search process begin
matched_tb = tfidf.fuzzySearch(msgVector, allVecs)
if matched_tb == {}:
#Should not run to here
text = 'ERROR: Not find any similarity vector.'
self.err.append('%s\n%s\n'%(CaseName, text))
else:
#Insert similarty table into db
for simVal,msgId in matched_tb.items():
self.task.db.cateTFIDF.insertSimilarity([itemId, str(msgId), str(simVal)])
else:
#Database is empty, what we need to do, is just insert current message into databse.
text = 'WARNING: Database is empty now, we just directly insert current message as first itme.'
self.err.append('%s\n%s\n'%(CaseName, text))
pass
def preProcess(self):
basicApi.d_print('CNN_analyzer: preProcess()')
FLAGS = tf.flags.FLAGS
self.threshhold_min = -1
self.threshhold_max = 7
tf.flags.DEFINE_boolean("allow_soft_placement", True,
"Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
self.vocab_path = os.path.join(self.checkpoint_dir, "..", "vocab")
self.vocab_processor = learn.preprocessing.VocabularyProcessor.restore(
self.vocab_path)
self.cr_list_dic = self._generate_category_vec()
# self.ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
# self.ckpt_model = self.ckpt.model_checkpoint_path
self.checkpoint_file = tf.train.latest_checkpoint(self.checkpoint_dir)
#pdb.set_trace()
self.graph = tf.Graph()
with self.graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
self.sess = tf.Session(config=session_conf)
with self.sess.as_default():
# Load the saved meta graph and restore variables
self.saver = tf.train.import_meta_graph("{}.meta".format(
self.checkpoint_file))
#self.saver.restore(self.sess, self.ckpt_model)
self.saver.restore(self.sess, self.checkpoint_file)
# Get the placeholders from the graph by name
self.input_x = self.graph.get_operation_by_name(
"input_x").outputs[0]
# input_y = graph.get_operation_by_name("input_y").outputs[0]
self.dropout_keep_prob = self.graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
self.predictions = self.graph.get_operation_by_name(
"output/predictions").outputs[0]
self.scores = self.graph.get_operation_by_name(
"output/scores").outputs[0]
pass
def setupTask(self):
#1, Collect test case related information, like testTag, testPlatform, testBranch, testSet, etc
#2, The jnl/log path which we need to read
#3, Path/file availability check
basicApi.d_print('AseSetup: setupTask()')
argList = self.task.argList
workDir = argList[0]
testSet = argList[1]
tag = argList[2]
user = argList[3]
platform = '%s,%s' % (argList[4], argList[5])
lockSch = argList[6]
pageSize = argList[7]
branch = argList[8]
tagTime = basicApi.parseDate(argList[9])
#init task
self.task.workDir = workDir
self.task.testSet = testSet
self.task.tag = tag
self.task.user = user
self.task.platform = platform
self.task.branch = branch
self.task.tagTime = tagTime
#Get each tcf storage path
argList = [workDir, testSet, lockSch, pageSize]
self.tcfList = self._findTcfsBySet(argList)
#Generate auto-analyzer log directory
self.aaLogDir = os.path.join(self.testSetWorkDir, 'autoAnalyzer')
if os.path.exists(self.aaLogDir):
basicApi.deleteDir(self.aaLogDir)
time.sleep(3)
os.mkdir(self.aaLogDir)
#Stroe this path into task object
self.task.aaLogDir = self.aaLogDir
#Collect *.tcf and *.exec file pairs
if self.tcfList:
for tcf in self.tcfList:
execOutFileList = glob.glob('%s/exec.out.*' % tcf)
jnlFileList = glob.glob('%s/journal.exec.*' % tcf)
for index in range(len(execOutFileList)):
execOut = execOutFileList[index]
jnl = jnlFileList[index]
self.jnlPair.append((jnl, execOut))
def checkTaskEnv(self):
#1, Does testing framework in normal status
#2, Does test case running currectly
basicApi.d_print('AseSetup: checkTaskEnv()')
if not self.tcfList:
#Check whether each test case has been finished normally
#Check whether quasr setup successfully
#os.chmod(autoAnalyzerDir, 0766)
#Write setup failed data into database
self.task.db.cateSummary.insertSetupFail([
self.task.tag, self.task.tagTime, self.task.branch,
self.task.platform, self.task.testSet, self.task.workDir,
self.task.user, '?'
])
basicApi.writeFile(os.path.join(self.aaLogDir, 'autoAnalysis.err'),
self.err)
#Send mail to BianHong for Quasr setup fail
sfr = SetupFailReport(self.task)
sfr.execute(self.resultDir)
#Raise exception
raise basicApi.PROCESS_ERROR('Quasr test case setup failed!')
pass
def __init__(self):
basicApi.d_print('AseAnalysis: __init__()')
self.tfidf = TFIDF()
self.cnn = CNN()
pass
def run(self, task):
basicApi.d_print('AseAnalysis: run()')
self.cnn.run(task)
self.tfidf.run(task)
pass
def __init__(self):
basicApi.d_print('AseSummary: __init__()')
def sum(self):
# Read prediction from each DB and merge them together.
basicApi.d_print('AseSummary: sum()')
if self.task.caseList == []:
#If the claseList is empty, then that means test case all pass.
print('AseSummary: Test case all pass, Skip summary')
return True
#1, search TFIDF result DB, find out all simulate testcases and group them together.
#Just update failed test case for current test set.
#self.task.db.cateSummary.updateCluster()
self.task.db.cateSummary.updateCluster([self.task.testSet])
#2, Search CNN result DB, get all data.
rtl = self.task.db.cateSummary.searchCNNResult([
self.task.tag, self.task.platform, self.task.branch,
self.task.testSet, self.task.tagTime
])
#rtl = self.task.db.cateSummary.searchCNNResult_x()
CNNResultDict = rtl.getDict()
#3, Check CNN prediction result with TFIDF group by decision tree.
for itemId in CNNResultDict.keys():
cnnCR = CNNResultDict[itemId]
rtl = self.task.db.cateSummary.searchClusterIdByItemId(itemId)
if rtl.isEmpty():
#Can not find current itemId in cluster.
#That means TFIDF do not process this itemId.
#In this case, we directly trust CNN result.
self._updateCR(itemId, cnnCR)
else:
clusterId = rtl.getStr()
rtl = self.task.db.cateSummary.searchItemsByClusterId(
clusterId)
itemIdList = rtl.getList()
#Get related CR list by item list
cnnCrList = self._getCNNPredictResult(itemIdList)
#for item in itemIdList:
if self._isAllSame(cnnCrList):
#All same situation no need further analysis
self._updateCR(itemId, cnnCrList[0])
else:
#Partily same
if self._isNewMoreThanHalf(cnnCrList):
#New issue more than 50%
#Attach all issue as new
self._updateCR(itemId, 'new')
else:
#new less equal than 50%
maxCr = self._isSameCRMoreThanHalf(cnnCrList)
if maxCr:
#Attach to same CR
self._updateCR(itemId, maxCr)
else:
#If CNN partly same with most TFIDF result
maxCr = self._ispartlySameWithMostTFIDF(
itemIdList, cnnCrList)
if maxCr:
#Auto attach to most possibility
self._updateCR(itemId, maxCr)
else:
#manual analysis
self._updateCR(itemId, 'new')
#Send mail to test set owner
tsr = TestsetReport(self.task)
tsr.execute()
def run(self, task):
basicApi.d_print('common.Framework:run()')
for insName in self.clsNameList:
instance = getattr(self, insName)
instance.run(task)
pass
def run(self, task):
basicApi.d_print('common.Setup:run()')
self.task = task
self.setupTask()
self.checkTaskEnv()
self.parpareTaskData()
def __init__(self):
basicApi.d_print('common.Setup:__init__()')
pass