def __init__(self):
self.vcs = GitProvider(config.getRepoDir())
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.checkers = Checkers()
self.loadCommitList()
def testDatabaseOpenNonExistingFailure(self):
dbPath = config.getRepoDir() + "/testtmp.sqlite"
with self.assertRaises(FileNotFoundError):
db = CCDatabase(dbPath)
class Predictor():
def __init__(self):
self.vcs = GitProvider(config.getRepoDir())
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.checkers = Checkers()
self.loadCommitList()
def loadCommitList(self):
self.commits = self.vcs.getAllVersions(config.getBranch())
self.currentCommitIndex = 0
def convertFilePathToRepoRelativePath(self, path):
return os.path.relpath(path, config.getRepoDir())
def getDiffResolvedIds(self):
resolved = self.codeChecker.diffResolved(config.getCcRunName(),
config.getTmpDir(), self.ccdb)
ids = []
for bug in resolved:
ids.append(bug['reportId'])
return ids
def predict(self, id, checker):
# Load all bugs
print("Loading bug data...")
ids = []
if id == -1:
bugs = self.ccdb.getAllBugsForChecker(checker)
ids = [x[0] for x in bugs]
else:
ids.append(id)
# Loading model
print("Loading model...")
model = load_model(config.cfModelFilenameFormat.format(checker))
model.summary()
vLabels = ['NOT OK', 'OK', 'Skipped']
# Initialize coder
print("Initializing coder...")
self.dictionary = Dictionary(checker)
self.coder = Coder(self.dictionary)
self.totalDictionaryLength = self.dictionary.length()
# Predicting
print("Starting predictions...")
for i in ids:
allData = self.ccdb.getBugData(i)
if allData.getChecker(
) not in globals.availableCheckers or allData.getChecker(
) != checker:
print("Bug #{0} - checker not supported".format(i))
else:
# Load extra tokens from checker message
checkerInfo = self.checkers.extractTokensForChecker(
allData.getChecker(), allData.getMessage())
# Retrieve code fragment with bug
fileRelativePath = self.convertFilePathToRepoRelativePath(
allData.getFile())
fullCodeWithBug = self.vcs.getFileContents(
fileRelativePath, self.commits[self.currentCommitIndex])
extractor = CodeExtractor(allData)
extractor.loadCodeFromText(fullCodeWithBug)
extractor.extractBugCode()
bugCodeFragment = extractor.getBugCodeFragment()
fixCodeFragment = ''
# Encode it
encodedBugData, initialUnkList = self.coder.encode(
bugCodeFragment, checkerData=checkerInfo)
# Convert to one-hot
MODEL_X_MAX_LEN = model.get_layer(index=0).input_shape[1]
if len(encodedBugData) > MODEL_X_MAX_LEN:
print(
"Bug #{0} - Code too big for model, ignored".format(i))
continue
elif id == -1:
print("Bug #{0} - Good to go".format(i))
continue
noZerosToPad = MODEL_X_MAX_LEN - len(encodedBugData)
if noZerosToPad > 0:
encodedBugData = self.coder.applyPadding(
encodedBugData, noZerosToPad)
X = np.zeros((1, MODEL_X_MAX_LEN, self.totalDictionaryLength))
X[0] = self.coder.convertToOneHot(
encodedBugData,
np.zeros((MODEL_X_MAX_LEN, self.totalDictionaryLength)))
# Predict and convert from one-hot
Y = self.coder.convertFromOneHot(model.predict(X)[0])
print(Y)
# Decode
Y = self.coder.removePadding(Y)
fixCodeFragment = self.coder.decode(Y, initialUnkList)
#Verify?
vStatus = 2
if config.cfVerifyPrediction:
# Apply fix in source code file
extractor.applyFix(fixCodeFragment)
extractor.saveToFile(allData.getFile())
# Run CodeChecker and analyze code
self.codeChecker.check(True)
resolvedIds = self.getDiffResolvedIds()
# Check if ID is resolved in tmp folder
isFixed = i in resolvedIds
# Set vStatus accordingly
if isFixed:
vStatus = 1
else:
vStatus = 0
#Print
print("Bug #{0} - summary".format(i))
print("== Code fragment with bug ==")
print(bugCodeFragment)
print("== Suggested fix ==")
print(fixCodeFragment)
print("Verification: {0}".format(vLabels[vStatus]))
a = ' '
while a != 'y' and a != 'n':
a = input("Apply fix? (y/n): ")
if a == 'y':
if not config.cfVerifyPrediction:
# Apply fix in source code file
extractor.applyFix(fixCodeFragment)
extractor.saveToFile(allData.getFile())
elif config.cfVerifyPrediction:
# Revert file contents
self.vcs.checkout(self.commits[self.currentCommitIndex])
print('Done')
print("All done, exiting...")
def __init__(self):
self.vcs = GitProvider(config.getRepoDir())
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
def testDatabaseOpenExistingSuccess(self):
dbPath = config.getCcDbFile()
db = CCDatabase(dbPath)
self.assertEqual(len(db.getAllReports()), 0)
def fix(code, bugData):
if bugData.getChecker() == 'deadcode.DeadStores':
return fixDeadStore(code, bugData)
if bugData.getChecker(
) == 'clang-diagnostic-tautological-constant-out-of-range-compare':
return findTautOORCmp(code, bugData)
if bugData.getChecker() == 'clang-diagnostic-unused-parameter':
return fixUnusedParam(code, bugData)
if bugData.getChecker() == 'clang-diagnostic-constant-conversion':
return fixConstConv(code, bugData)
return None
ccdb = CCDatabase(config.getCcDbFile())
db = sqlite3.connect('../Results/db.sqlite')
cursor = db.cursor()
cursor.execute('SELECT * FROM bugs')
dataFromDb = cursor.fetchall()
bugs = []
bugsPerFile = {}
BUG_NOT_PROCESSED = 0
vcs = GitProvider(config.getRepoDir())
checkers = Checkers()
currentCommit = vcs.getAllVersions(config.getBranch())[0]
bugDataList = {}
fileContents = {}
codechecker = CodeChecker(config.getRepoDir())
if len(dataFromDb) > 0:
class TestDbBuilder():
def __init__(self):
self.vcs = GitProvider(config.getRepoDir())
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
def loadCommitList(self, clean=False):
self.commits = self.vcs.getAllVersions(config.getBranch())
if not clean:
lastCommit = self.db.getLastCommit()
lastIndex = self.commits.index(lastCommit) + 1
if lastIndex < len(self.commits) - 1:
self.commits = self.commits[0:lastIndex]
self.currentCommitIndex = lastIndex - 1
else:
self.currentCommitIndex = len(self.commits)
def prepareDb(self, clean=False):
self.db = CFDatabase(config.getCfDbFile())
if clean:
self.db.clean()
def checkoutToNextVersion(self):
self.currentCommitIndex = self.currentCommitIndex - 1
if (self.currentCommitIndex < 0):
return False
self.vcs.checkout(self.commits[self.currentCommitIndex])
return True
def getDiffResolvedIds(self):
resolved = self.codeChecker.diffResolved(config.getCcRunName(),
config.getTmpDir(), self.ccdb)
ids = []
for bug in resolved:
ids.append(bug['reportId'])
return ids
def convertFilePathToRepoRelativePath(self, path):
return os.path.relpath(path, config.getRepoDir())
def extractCode(self, id):
bugData = self.ccdb.getNotResolvedBugData(id)
#TODO: Possible improvement for bugData
if bugData is None:
#TODO: Implement custom errors
return None
fileRelativePath = self.convertFilePathToRepoRelativePath(
bugData.getFile())
try:
fullCodeWithBug = self.vcs.getFileContents(
fileRelativePath, self.commits[self.currentCommitIndex + 1])
fullCodeWithoutBug = self.vcs.getFileContents(
fileRelativePath, self.commits[self.currentCommitIndex])
except KeyError as extractError:
return None
diff = POSIXDiffer().diff(fullCodeWithBug, fullCodeWithoutBug)
extractor = CodeExtractor(bugData)
try:
extractor.extractAll(fullCodeWithBug, diff)
except ValueError as extractError:
return None
bugCodeFragment = extractor.getBugCodeFragment()
fixCodeFragment = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
#Easy version - ignore bug if none or more than one diff used to fix
#TODO: Possible improvement here
if len(usedDiffs) != 1:
return None
return entities.FixData(bugCodeFragment, fixCodeFragment,
bugData.getChecker(), bugData.getMessage(),
bugData.getLine() - bugData.getStartLine())
def prepareEnv(self, clean=False):
print('Preparing train db... ', end='')
self.prepareDb(clean)
print('done')
print('Loading commit list... ', end='')
self.loadCommitList(clean)
print('done')
if clean:
print('Checking out to root... ', end='')
self.checkoutToNextVersion()
print('done')
print('Initial analysis... ', end='')
self.codeChecker.check(True)
print('done')
print('Storing initial results... ', end='')
self.codeChecker.store(self.commits[self.currentCommitIndex])
print('done')
print('Storing version information... ', end='')
self.db.storeLastCommit(self.commits[self.currentCommitIndex])
print('done')
print('Cleaning up tmp directory... ', end='')
shutil.rmtree(config.getTmpDir())
print('done')
print('Cleaning up working directory... ', end='')
self.codeChecker.clean()
print('done')
def findAndStoreFixDataForVersion(self):
print('Analyzing version',
self.commits[self.currentCommitIndex],
'... ',
end='')
self.codeChecker.check(True)
print('done')
print('Getting list of resolved bugs for version',
self.commits[self.currentCommitIndex],
'... ',
end='')
ids = self.getDiffResolvedIds()
print('done')
bugNo = 1
allBugs = len(ids)
anyStored = False
for id in ids:
print('Parsing data for bug ({0}/{1}, #{2})...'.format(
bugNo, allBugs, id),
sep='',
end='')
fixData = self.extractCode(id)
bugNo = bugNo + 1
print('done')
if fixData is not None:
print('Storing fixData... ', end='')
self.db.store(fixData.getBugCode(), fixData.getFixCode(),
fixData.getChecker(), fixData.getMessage(),
fixData.getLine())
anyStored = True
print('done')
if bugNo % 100 == 0 and anyStored:
self.db.commit()
anyStored = False
if anyStored:
self.db.commit()
print('Storing CodeChecker results for this version... ', end='')
self.codeChecker.store(self.commits[self.currentCommitIndex])
print('done')
print('Storing version information... ', end='')
self.db.storeLastCommit(self.commits[self.currentCommitIndex])
print('done')
print('Cleaning up tmp directory... ', end='')
shutil.rmtree(config.getTmpDir())
print('done')
print('Cleaning up working directory... ', end='')
self.codeChecker.clean()
print('done')
def iterateThroughVcsHistory(self):
while self.checkoutToNextVersion():
self.findAndStoreFixDataForVersion()
def checkoutToTop(self):
self.vcs.checkout(config.getBranch())
def build(self, clean=False):
self.prepareEnv(clean)
self.iterateThroughVcsHistory()
self.checkoutToTop()
class FixUnusedParam():
def __init__(self):
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.code = []
self.checkers = Checkers()
def loadCodeFromFile(self):
with open(self.bugData.getFile(), 'r') as file:
code = file.readlines()
self.code = []
for line in code:
self.code.append(line.replace("\r\n", "\n"))
"""
code = file.read()
noCrLf = code.count("\r\n")
noCr = code.count("\r") - noCrLf
noLf = code.count("\n") - noCrLf
if noCrLf > noCr:
if noCrLf > noLf:
self.lineEnding = "\r\n"
else:
self.lineEnding = "\n"
else:
if noCr > noLf:
self.lineEnding = "\r"
else:
self.lineEnding = "\n"
self.code = code.split(self.lineEnding)
"""
def saveCodeToFile(self):
with open(self.bugData.getFile(), 'w') as file:
#file.write(self.lineEnding.join(self.code))
file.writelines(self.code)
def fix(self, file):
fileData = self.ccdb.getFileData(file)
print(fileData)
bugs = self.ccdb.getAllBugsForChecker(
'clang-diagnostic-unused-parameter')
lines = []
inserts = 0
for bug in bugs:
if bug[2] == fileData[-1][0]:
self.bugData = self.ccdb.getBugData(bug[0])
if self.bugData.getLine() not in lines:
self.loadCodeFromFile()
newLine = '// codechecker_intentional [clang-diagnostic-unused-parameter] Suppress\n'
self.code.insert(self.bugData.getLine() - 1 + inserts,
newLine)
inserts = inserts + 1
self.saveCodeToFile()
lines = lines + list(
range((self.bugData.getLine() - 11),
(self.bugData.getLine() + 10)))
print("Passed")
print(self.bugData.getLine())
pass
else:
print("Dismissed")
print(self.bugData.getLine())
pass
def __init__(self):
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.code = []
self.checkers = Checkers()
class FixDeadStore():
def __init__(self):
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.code = []
def loadCodeFromFile(self):
with open(self.bugData.getFile(), 'r') as file:
code = file.readlines()
self.code = []
for line in code:
self.code.append(line.replace("\r\n", "\n"))
"""
code = file.read()
noCrLf = code.count("\r\n")
noCr = code.count("\r") - noCrLf
noLf = code.count("\n") - noCrLf
if noCrLf > noCr:
if noCrLf > noLf:
self.lineEnding = "\r\n"
else:
self.lineEnding = "\n"
else:
if noCr > noLf:
self.lineEnding = "\r"
else:
self.lineEnding = "\n"
self.code = code.split(self.lineEnding)
"""
def saveCodeToFile(self):
with open(self.bugData.getFile(), 'w') as file:
#file.write(self.lineEnding.join(self.code))
file.writelines(self.code)
def fix(self, file):
fileData = self.ccdb.getFileData(file)
print(fileData)
bugs = self.ccdb.getAllBugsForChecker('deadcode.DeadStores')
lines = []
for bug in bugs:
if bug[2] == fileData[-1][0]:
self.bugData = self.ccdb.getBugData(bug[0])
if self.bugData.getLine() not in lines:
self.loadCodeFromFile()
#lineEnding = self.getLineEnding()
if self.bugData.getLine() == 1:
self.code = ["\n"] + self.code[1:]
elif self.bugData.getLine() == len(self.code):
self.code = self.code[:-1] + ["\n"]
else:
self.code = self.code[0:(self.bugData.getLine() - 1)] + ["\n"] + self.code[self.bugData.getLine():]
self.saveCodeToFile()
lines = lines + list(range((self.bugData.getLine() - 11), (self.bugData.getLine() + 10)))
print("Passed")
print(self.bugData.getLine())
pass
else:
print("Dismissed")
print(self.bugData.getLine())
pass
class FixTautOORCmp():
def __init__(self):
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.code = []
self.checkers = Checkers()
def loadCodeFromFile(self):
with open(self.bugData.getFile(), 'r') as file:
code = file.readlines()
self.code = []
for line in code:
self.code.append(line.replace("\r\n", "\n"))
"""
code = file.read()
noCrLf = code.count("\r\n")
noCr = code.count("\r") - noCrLf
noLf = code.count("\n") - noCrLf
if noCrLf > noCr:
if noCrLf > noLf:
self.lineEnding = "\r\n"
else:
self.lineEnding = "\n"
else:
if noCr > noLf:
self.lineEnding = "\r"
else:
self.lineEnding = "\n"
self.code = code.split(self.lineEnding)
"""
def saveCodeToFile(self):
with open(self.bugData.getFile(), 'w') as file:
#file.write(self.lineEnding.join(self.code))
file.writelines(self.code)
def findMatchingDefine(self, value):
patValue = '({0})'
patName = '[ \t]([0-9a-zA-Z_]+)[ \t]'
for line in self.code:
if line[:7] == '#define':
if re.search(patValue.format(value), line):
return re.search(patName, line).group(1)
return None
def fix(self, file):
fileData = self.ccdb.getFileData(file)
print(fileData)
bugs = self.ccdb.getAllBugsForChecker(
'clang-diagnostic-tautological-constant-out-of-range-compare')
lines = []
for bug in bugs:
if bug[2] == fileData[-1][0]:
self.bugData = self.ccdb.getBugData(bug[0])
if self.bugData.getLine() not in lines:
self.loadCodeFromFile()
tokens = self.checkers.extractTokensForChecker(
'clang-diagnostic-tautological-constant-out-of-range-compare',
self.bugData.getMessage())
line = self.code[self.bugData.getLine() - 1]
pat = "\([^\(]*{0}\)".format(tokens[0]['value'])
match = re.search(pat, line)
if match is None:
pat = "\([^\(]*{0}\)".format(
self.findMatchingDefine(tokens[0]['value']))
match = re.search(pat, line)
match = match.group(0)
newLine = line.replace(match,
'({0})'.format(tokens[1]['value']))
self.code[self.bugData.getLine() - 1] = newLine
self.saveCodeToFile()
lines = lines + list(
range((self.bugData.getLine() - 11),
(self.bugData.getLine() + 10)))
print("Passed")
print(self.bugData.getLine())
pass
else:
print("Dismissed")
print(self.bugData.getLine())
pass
class FixTautOORCmp():
def __init__(self):
self.ccdb = CCDatabase(config.getCcDbFile())
self.codeChecker = CodeChecker(config.getRepoDir())
self.code = []
self.checkers = Checkers()
def loadCodeFromFile(self):
with open(self.bugData.getFile(), 'r') as file:
code = file.readlines()
self.code = []
for line in code:
self.code.append(line.replace("\r\n", "\n"))
def saveCodeToFile(self):
with open(self.bugData.getFile(), 'w') as file:
#file.write(self.lineEnding.join(self.code))
file.writelines(self.code)
def findMatchingDefine(self, value):
patValue = '({0})'
patName = '[ \t]([0-9a-zA-Z_]+)[ \t]'
for line in self.code:
if line[:7] == '#define':
if re.search(patValue.format(value), line):
return re.search(patName, line).group(1)
return None
def fix(self, file):
fileData = self.ccdb.getFileData(file)
print(fileData)
bugs = self.ccdb.getAllBugsForChecker('clang-diagnostic-constant-conversion')
lines = []
linesSingle = {}
linesToSuppress = []
for bug in bugs:
if bug[2] == fileData[-1][0]:
self.bugData = self.ccdb.getBugData(bug[0])
if self.bugData.getLine() in linesSingle:
linesSingle[self.bugData.getLine()] += 1
else:
linesSingle[self.bugData.getLine()] = 1
for k in linesSingle:
if linesSingle[k] > 5:
linesToSuppress.append(k)
inserts = 0
for bug in bugs:
if bug[2] == fileData[-1][0]:
self.bugData = self.ccdb.getBugData(bug[0])
if self.bugData.getLine() not in lines:
self.loadCodeFromFile()
if self.bugData.getLine() not in linesToSuppress:
tokens = self.checkers.extractTokensForChecker('clang-diagnostic-constant-conversion', self.bugData.getMessage())
test = tokens
line = self.code[self.bugData.getLine() - 1 + inserts]
newLine = line.replace(tokens[0]['value'], '({0}){1}'.format(tokens[1]['value'], tokens[0]['value']))
if line == newLine:
tokens[0]['value'] = self.findMatchingDefine(tokens[0]['value'])
if tokens[0]['value'] is not None:
newLine = line.replace(tokens[0]['value'], '({0}){1}'.format(tokens[1]['value'], tokens[0]['value']))
if line == newLine:
s = re.search('= (.*);', line)
expr = s.group(1)
newLine = line.replace(expr, '({0})({1})'.format(tokens[1]['value'], expr))
self.code[self.bugData.getLine() - 1 + inserts] = newLine
else:
line = self.code[self.bugData.getLine() - 1 + inserts]
self.code[self.bugData.getLine() - 1 + inserts] = '// codechecker_intentional [clang-diagnostic-constant-conversion] Suppress\n{0}'.format(line)
inserts += 1
self.saveCodeToFile()
lines = lines + list(range((self.bugData.getLine() - 11), (self.bugData.getLine() + 10)))
print("Passed, suppressed: {0}".format(self.bugData.getLine() in linesToSuppress))
print(self.bugData.getLine())
pass
else:
print("Dismissed")
print(self.bugData.getLine())
pass