def testCodeFixExtractSuccessAllChanges(self):
bugData = self.getCorrectBugDataFileMiddle()
expectedFixedCode = """int main(void)
{
int a, b;
b = 2;
a = 3;
a = 0;
if (a != 0)
{
"""
fileDiff = self.getFileDiff(bugData.getStartLine() - 1, [],
[' int a;'])
fileDiff += "\n" + self.getFileDiff(
bugData.getStartLine(), [' int a, b;', ' b = 2;'], [])
fileDiff += "\n" + self.getFileDiff(bugData.getStartLine() + 2,
[' if (a != 0)'],
[' if (a == 0)'])
extractor = CodeExtractor(bugData)
extractor.loadCodeFromFile()
extractor.extractBugCode()
extractor.loadDiff(fileDiff)
extractor.extractFixCode()
fixedCode = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
self.assertEqual(expectedFixedCode, fixedCode)
self.assertEqual(3, len(usedDiffs))
self.assertEqual('7d7', usedDiffs[0])
self.assertEqual('7a7', usedDiffs[1])
self.assertEqual('10c10', usedDiffs[2])
def testExtractCodeWithEmptyDiffBetweenTwoCommits(self):
gp = gitprovider.GitProvider(config.getRepoDir())
commits = gp.getAllVersions('trainDbScriptTest')
commit1 = commits[-2]
commit2 = commits[-2]
file1 = gp.getFileContents('bugcode2.cpp', commit1)
file2 = gp.getFileContents('bugcode2.cpp', commit2)
diff = LinuxDiffer().diff(file1, file2)
usedDiffs = []
bugData = self.getBugData()
extractor = CodeExtractor(bugData)
extractor.loadCodeFromText(file1, '\r\n', '\n')
extractor.extractBugCode()
extractor.loadDiff(diff)
with self.assertRaises(ValueError):
extractor.extractFixCode()
bugCode = extractor.getBugCodeFragment()
fixCode = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
expectedOutputFix = ''
expectedOutputBug = """int main(void)
{
int a;
a = 3;
a = 0;
if (a == 0)
{
"""
self.assertEqual(expectedOutputBug, bugCode)
self.assertEqual(expectedOutputFix, fixCode)
self.assertEqual(0, len(usedDiffs))
def testCodeFixExtractFailureNoDiff(self):
bugData = self.getCorrectBugDataFileMiddle()
bugCode = self.getBugCodeFileMiddle()
expectedFixedCode = ''
fileDiff = self.getFileDiff(bugData.getEndLine() + 20, [],
[' a = 3;'])
extractor = CodeExtractor(bugData)
extractor.loadCodeFromFile()
extractor.extractBugCode()
extractor.loadDiff(fileDiff)
with self.assertRaises(ValueError):
extractor.extractFixCode()
fixedCode = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
self.assertEqual(expectedFixedCode, fixedCode)
self.assertEqual(0, len(usedDiffs))
def testCodeFixExtractSuccessRemove(self):
bugData = self.getCorrectBugDataFileMiddle()
expectedFixedCode = """int main(void)
{
int a;
a = 0;
if (a == 0)
{
"""
fileDiff = self.getFileDiff(bugData.getStartLine(), [], [' a = 3;'])
extractor = CodeExtractor(bugData)
extractor.loadCodeFromFile()
extractor.extractBugCode()
extractor.loadDiff(fileDiff)
extractor.extractFixCode()
fixedCode = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
self.assertEqual(expectedFixedCode, fixedCode)
self.assertEqual(1, len(usedDiffs))
self.assertEqual('8d8', usedDiffs[0])
def testExtractCodeWithDiffBetweenTwoCommitsMultiDiffInFragment(self):
gp = gitprovider.GitProvider(config.getRepoDir())
commits = gp.getAllVersions('trainDbScriptTest')
commit1 = commits[-6]
commit2 = commits[-7]
file1 = gp.getFileContents('bugcode3.cpp', commit1)
file2 = gp.getFileContents('bugcode3.cpp', commit2)
diff = LinuxDiffer().diff(file1, file2)
usedDiffs = []
bugData = self.getBugData3()
extractor = CodeExtractor(bugData)
extractor.loadCodeFromText(file1, '\r\n', '\n')
extractor.extractBugCode()
extractor.loadDiff(diff)
extractor.extractFixCode()
bugCode = extractor.getBugCodeFragment()
fixCode = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
expectedOutputFix = """{;
int a;
a = 0;
a = 2;
if (a != 0)
{;
int b = 1 / a;
"""
expectedOutputBug = """{
int a;
a = 0;
a = 2;
if (a != 0)
{
int b = 1 / a;
"""
self.assertEqual(expectedOutputBug, bugCode)
self.assertEqual(expectedOutputFix, fixCode)
self.assertEqual(2, len(usedDiffs))
def extractCode(self, id):
bugData = self.ccdb.getNotResolvedBugData(id)
#TODO: Possible improvement for bugData
if bugData is None:
#TODO: Implement custom errors
return None
fileRelativePath = self.convertFilePathToRepoRelativePath(
bugData.getFile())
try:
fullCodeWithBug = self.vcs.getFileContents(
fileRelativePath, self.commits[self.currentCommitIndex + 1])
fullCodeWithoutBug = self.vcs.getFileContents(
fileRelativePath, self.commits[self.currentCommitIndex])
except KeyError as extractError:
return None
diff = POSIXDiffer().diff(fullCodeWithBug, fullCodeWithoutBug)
extractor = CodeExtractor(bugData)
try:
extractor.extractAll(fullCodeWithBug, diff)
except ValueError as extractError:
return None
bugCodeFragment = extractor.getBugCodeFragment()
fixCodeFragment = extractor.getFixCodeFragment()
usedDiffs = extractor.getUsedDiffs()
#Easy version - ignore bug if none or more than one diff used to fix
#TODO: Possible improvement here
if len(usedDiffs) != 1:
return None
return entities.FixData(bugCodeFragment, fixCodeFragment,
bugData.getChecker(), bugData.getMessage(),
bugData.getLine() - bugData.getStartLine())
def ProcessBugsInFile(fileName):
# 5.1.
# 5.2.
for bug in bugsPerFile[fileName]:
bugData = bugDataList[bug]
cleanFn = fileName[:-4]
fn = '../Results/Analysis/{0}_{1}_{2}.txt'.format(
cleanFn, bug, bugData.getChecker())
repoFn = os.fsdecode(
os.path.join(os.fsencode(config.getRepoDir()),
os.fsencode(fileName)))
if os.path.isfile(fn):
continue
model = None
if bugData.getChecker() == 'deadcode.DeadStores':
model = model1
coder = coder1
totalDictionaryLength = totalDictionaryLength1
if bugData.getChecker(
) == 'clang-diagnostic-tautological-constant-out-of-range-compare':
model = model2
coder = coder2
totalDictionaryLength = totalDictionaryLength2
if bugData.getChecker() == 'clang-diagnostic-unused-parameter':
model = model3
coder = coder3
totalDictionaryLength = totalDictionaryLength3
if bugData.getChecker() == 'clang-diagnostic-constant-conversion':
model = model4
coder = coder4
totalDictionaryLength = totalDictionaryLength4
MODEL_X_MAX_LEN = model.get_layer(index=0).input_shape[1]
# 5.2.1.
fullCodeWithBug = fileContents[fileName]
extractor = CodeExtractor(bugData)
extractor.loadCodeFromText(fullCodeWithBug)
extractor.extractBugCode()
bugCodeFragment = extractor.getBugCodeFragment()
# 5.2.2.
fullCodeWithoutBug = ''.join(fix(extractor.code, bugData))
diff = POSIXDiffer().diff(fullCodeWithBug, fullCodeWithoutBug)
extractor.loadDiff(diff)
try:
extractor.extractFixCode()
except ValueError as ve:
print(
"Unable to generate expected fix for bug #{0} ({1}), checker = {2}"
.format(bug, fileName, bugData.getChecker()))
continue
expectedFixCodeFragment = extractor.getFixCodeFragment()
# 5.2.3.
checkerInfo = checkers.extractTokensForChecker(bugData.getChecker(),
bugData.getMessage())
encodedBugData, initialUnkList = coder.encode(bugCodeFragment,
checkerData=checkerInfo)
noZerosToPad = MODEL_X_MAX_LEN - len(encodedBugData)
if noZerosToPad > 0:
encodedBugData = coder.applyPadding(encodedBugData, noZerosToPad)
X = np.zeros((1, MODEL_X_MAX_LEN, totalDictionaryLength))
X[0] = coder.convertToOneHot(
encodedBugData, np.zeros((MODEL_X_MAX_LEN, totalDictionaryLength)))
Y = coder.convertFromOneHot(model.predict(X)[0])
while (Y[-1] == 0):
Y.pop()
fixCodeFragment = coder.decode(Y, initialUnkList)
# 5.2.4.
isCompiling = False
f1 = os.fsdecode(
os.path.join(os.fsencode(config.getRepoDir()),
os.fsencode(cleanFn)))
f2 = os.fsdecode(
os.path.join(os.fsencode(config.getRepoDir()),
os.fsencode('{0}.o'.format(cleanFn))))
if os.path.isfile(f1):
os.remove(f1)
if os.path.isfile(f2):
os.remove(f2)
extractor.loadCodeFromText(fullCodeWithBug)
extractor.applyFix(fixCodeFragment)
with open(repoFn, 'wt') as f:
f.writelines(extractor.code)
log = codechecker.runCmd(
'CodeChecker check -e all -b "cd {0} && make {1}" -o /tmp/codefixer_{1}'
.format(config.getRepoDir(), cleanFn))
if os.path.isfile(f1):
isCompiling = True
# 5.2.5.
isFixed = False
ids = []
if isCompiling:
resolved = codechecker.diffResolved(
config.getCcRunName(), '/tmp/codefixer_{0}'.format(cleanFn),
ccdb)
for bugInfo in resolved:
ids.append(bugInfo['reportId'])
if bug in ids:
isFixed = True
# 5.2.6.
isExpected = False
if fixCodeFragment == expectedFixCodeFragment:
isExpected = True
# 5.2.7.
encodedExpFix, finalUnkList = coder.encode(expectedFixCodeFragment,
unkList=initialUnkList,
reverse=False)
noXTokens = len(encodedExpFix)
noYTokens = len(Y)
noAllTokens = max(noXTokens, noYTokens)
noCorrectTokens = 0
#print(encodedExpFix)
#print(Y)
for i in range(min(noXTokens, noYTokens)):
if encodedExpFix[i] == Y[i]:
noCorrectTokens += 1
# 5.2.8.
with open(fn, 'wt') as f:
f.write(
'#BUG#\n{0}\n#EXP#\n{1}\n#FIX#\n{2}\n#STATS#\n{3},{4},{5},{6},{7},{8},{9}\n{10}\n{11}'
.format(bugCodeFragment, expectedFixCodeFragment,
fixCodeFragment, isCompiling, isFixed, isExpected,
noXTokens, noYTokens, noAllTokens, noCorrectTokens,
log, ids))
# 5.2.9.
with open(repoFn, 'wt') as f:
f.write(fileContents[fileName])
# 5.2.10.
# Not used due to multithreading issues, will be done on file-by-file basis after closing pool
# 5.3.
return fileName