def prepareEnv(self, clean=False):
print('Preparing train db... ', end='')
self.prepareDb(clean)
print('done')
print('Loading commit list... ', end='')
self.loadCommitList(clean)
print('done')
if clean:
print('Checking out to root... ', end='')
self.checkoutToNextVersion()
print('done')
print('Initial analysis... ', end='')
self.codeChecker.check(True)
print('done')
print('Storing initial results... ', end='')
self.codeChecker.store(self.commits[self.currentCommitIndex])
print('done')
print('Storing version information... ', end='')
self.db.storeLastCommit(self.commits[self.currentCommitIndex])
print('done')
print('Cleaning up tmp directory... ', end='')
shutil.rmtree(config.getTmpDir())
print('done')
print('Cleaning up working directory... ', end='')
self.codeChecker.clean()
print('done')
def getDiffResolvedIds(self):
resolved = self.codeChecker.diffResolved(config.getCcRunName(),
config.getTmpDir(), self.ccdb)
ids = []
for bug in resolved:
ids.append(bug['reportId'])
return ids
def check(self, clean=False):
if clean:
self.clean()
jobs = ''
if config.ccNoOfJobs > 1:
jobs = ' -j {0}'.format(config.ccNoOfJobs)
checkers = '-e all'
#if len(config.customCheckers) > 0:
# checkers = '-d all -e {0}'.format(' -e '.join(config.customCheckers.split(',')))
cmd = 'CodeChecker check {0} -e all -b "cd {1} && make {0}" -o {2}'.format(
jobs, self.repo, config.getTmpDir())
return self.runCmd(cmd).decode(sys.stdout.encoding)
def findAndStoreFixDataForVersion(self):
print('Analyzing version',
self.commits[self.currentCommitIndex],
'... ',
end='')
self.codeChecker.check(True)
print('done')
print('Getting list of resolved bugs for version',
self.commits[self.currentCommitIndex],
'... ',
end='')
ids = self.getDiffResolvedIds()
print('done')
bugNo = 1
allBugs = len(ids)
anyStored = False
for id in ids:
print('Parsing data for bug ({0}/{1}, #{2})...'.format(
bugNo, allBugs, id),
sep='',
end='')
fixData = self.extractCode(id)
bugNo = bugNo + 1
print('done')
if fixData is not None:
print('Storing fixData... ', end='')
self.db.store(fixData.getBugCode(), fixData.getFixCode(),
fixData.getChecker(), fixData.getMessage(),
fixData.getLine())
anyStored = True
print('done')
if bugNo % 100 == 0 and anyStored:
self.db.commit()
anyStored = False
if anyStored:
self.db.commit()
print('Storing CodeChecker results for this version... ', end='')
self.codeChecker.store(self.commits[self.currentCommitIndex])
print('done')
print('Storing version information... ', end='')
self.db.storeLastCommit(self.commits[self.currentCommitIndex])
print('done')
print('Cleaning up tmp directory... ', end='')
shutil.rmtree(config.getTmpDir())
print('done')
print('Cleaning up working directory... ', end='')
self.codeChecker.clean()
print('done')
def main(self):
# Do analysis
shutil.rmtree(config.getTmpDir())
self.codeChecker.check(True)
# Diff new
newBugs = self.getDiffNew()
if len(newBugs) < 1:
print('No new bugs introduced, commit is accepted!')
return
print("New bugs found! Count: {0}. Attempting repairs...".format(len(newBugs)))
# Load models
models = {}
for checker in globals.availableCheckers:
models[checker] = load_model(config.cfModelFilenameFormat.format(checker))
# Load all content from files having new
files = set([self.convertFilePathToRepoRelativePath(x.getFile()) for x in newBugs])
fileContents = {}
for f in files:
fn = config.getRepoDir() + f
with open(fn, 'r') as fh:
fileContents[f] = ''.join(fh.readlines())
# For each file sort by bug line desc
suggestions = []
validSuggestions = 0
for f in files:
bugs = [x for x in newBugs if self.convertFilePathToRepoRelativePath(x.getFile()) == f]
bugs.sort(key=lambda x: x.getLine(), reverse=True)
print("=== File: {0} ===".format(f))
# For each bug get a suggestion and test it
for b in bugs:
print("L{0}, Type: {1}".format(b.getLine(), b.getChecker()))
# Prepare useful data
dictionary = Dictionary(b.getChecker())
coder = Coder(dictionary)
totalDictionaryLength = dictionary.length()
# Prepare and extract bug fragment
checkerInfo = self.checkers.extractTokensForChecker(b.getChecker(), b.getMessage())
extractor = CodeExtractor(b)
extractor.loadCodeFromText(fileContents[f])
extractor.extractBugCode()
bugCodeFragment = extractor.getBugCodeFragment()
fixCodeFragment = ''
# Encode it
encodedBugData, initialUnkList = coder.encode(bugCodeFragment, checkerData = checkerInfo)
# Convert to one-hot
MODEL_X_MAX_LEN = models[b.getChecker()].get_layer(index = 0).input_shape[1]
if len(encodedBugData) > MODEL_X_MAX_LEN:
print("Ignored: Code too big for model")
continue
noZerosToPad = MODEL_X_MAX_LEN - len(encodedBugData)
if noZerosToPad > 0:
encodedBugData = coder.applyPadding(encodedBugData, noZerosToPad)
X = np.zeros((1, MODEL_X_MAX_LEN, totalDictionaryLength))
X[0] = coder.convertToOneHot(encodedBugData, np.zeros((MODEL_X_MAX_LEN, totalDictionaryLength)))
# Predict and convert from one-hot
Y = coder.convertFromOneHot(models[b.getChecker()].predict(X)[0])
Y = coder.removePadding(Y)
# Decode
fixCodeFragment = coder.decode(Y, initialUnkList)[:-1]
#Verify?
vStatus = 2
if config.cfVerifyPrediction:
# Apply fix in source code file
extractor.applyFix(fixCodeFragment)
extractor.saveToFile(b.getFile())
# Run CodeChecker and analyze code
shutil.rmtree(config.getTmpDir())
compilationLog = self.codeChecker.check(True)
newBugsAfterFix = self.getDiffNew()
# Check if ID is resolved in tmp folder
isFixed = 'Build failed' not in compilationLog
for nb in newBugsAfterFix:
if self.isBugDataEqual(b, nb):
isFixed = False
# Set vStatus accordingly
if isFixed:
vStatus = 1
else:
vStatus = 0
# Revert file
extractor.loadCodeFromText(fileContents[f])
extractor.saveToFile(b.getFile())
if vStatus == 0:
print("Verification: Negative, cannot be applied")
elif vStatus == 1:
print("Verification: Positive, can be applied")
validSuggestions += 1
elif vStatus == 2:
print("Verification: Skipped")
validSuggestions += 1
sugg = SuggestionData(f, b, bugCodeFragment, fixCodeFragment, vStatus)
suggestions.append(sugg)
print("Valid suggestions prepared for {0} / {1} bugs.".format(validSuggestions, len(newBugs)))
if validSuggestions > 0:
print("Apply valid suggestions (a), display them (d), ignore them (i) or abort commit (q)?")
apply = False
choice = True
while choice:
c = sys.stdin.read(1)
if c == 'a':
apply = True
choice = False
print("Applying fixes...")
elif c == 'i':
choice = False
print("Fixes ignored...")
elif c == 'd':
self.displaySuggestions(suggestions)
print("Apply valid suggestions (a), ignore them (i) or abort commit (q)?")
elif c == 'q':
print("Aborting commit...")
sys.exit(1)
if apply:
self.applyValidFixes(suggestions, files)
print("Fixes applied!")
if validSuggestions != len(newBugs):
print("Unable to fix all bugs, continue with commit (c) or abort (q)?")
choice = True
while choice:
c = sys.stdin.read(1)
if c == 'c':
choice = False
print("Continuing...")
elif c == 'q':
print("Aborting commit...")
sys.exit(1)
else:
print("Bugs corrected, commit is good to go!")
def getDiffNew(self):
new = self.codeChecker.diffNew(config.getCcRunName(), config.getTmpDir(), self.ccdb)
ids = []
for bug in new:
ids.append(self.getBugDataFromDiff(bug))
return ids
def store(self, tag):
cmd = 'CodeChecker store {0} -n {1} --tag {2}'.format(
config.getTmpDir(), config.getCcRunName(), tag)
self.runCmd(cmd)
