def testCodeFixExtractSuccessAllChanges(self): bugData = self.getCorrectBugDataFileMiddle() expectedFixedCode = """int main(void) { int a, b; b = 2; a = 3; a = 0; if (a != 0) { """ fileDiff = self.getFileDiff(bugData.getStartLine() - 1, [], [' int a;']) fileDiff += "\n" + self.getFileDiff( bugData.getStartLine(), [' int a, b;', ' b = 2;'], []) fileDiff += "\n" + self.getFileDiff(bugData.getStartLine() + 2, [' if (a != 0)'], [' if (a == 0)']) extractor = CodeExtractor(bugData) extractor.loadCodeFromFile() extractor.extractBugCode() extractor.loadDiff(fileDiff) extractor.extractFixCode() fixedCode = extractor.getFixCodeFragment() usedDiffs = extractor.getUsedDiffs() self.assertEqual(expectedFixedCode, fixedCode) self.assertEqual(3, len(usedDiffs)) self.assertEqual('7d7', usedDiffs[0]) self.assertEqual('7a7', usedDiffs[1]) self.assertEqual('10c10', usedDiffs[2])
def testExtractCodeWithEmptyDiffBetweenTwoCommits(self): gp = gitprovider.GitProvider(config.getRepoDir()) commits = gp.getAllVersions('trainDbScriptTest') commit1 = commits[-2] commit2 = commits[-2] file1 = gp.getFileContents('bugcode2.cpp', commit1) file2 = gp.getFileContents('bugcode2.cpp', commit2) diff = LinuxDiffer().diff(file1, file2) usedDiffs = [] bugData = self.getBugData() extractor = CodeExtractor(bugData) extractor.loadCodeFromText(file1, '\r\n', '\n') extractor.extractBugCode() extractor.loadDiff(diff) with self.assertRaises(ValueError): extractor.extractFixCode() bugCode = extractor.getBugCodeFragment() fixCode = extractor.getFixCodeFragment() usedDiffs = extractor.getUsedDiffs() expectedOutputFix = '' expectedOutputBug = """int main(void) { int a; a = 3; a = 0; if (a == 0) { """ self.assertEqual(expectedOutputBug, bugCode) self.assertEqual(expectedOutputFix, fixCode) self.assertEqual(0, len(usedDiffs))
def testCodeFixExtractFailureNoDiff(self): bugData = self.getCorrectBugDataFileMiddle() bugCode = self.getBugCodeFileMiddle() expectedFixedCode = '' fileDiff = self.getFileDiff(bugData.getEndLine() + 20, [], [' a = 3;']) extractor = CodeExtractor(bugData) extractor.loadCodeFromFile() extractor.extractBugCode() extractor.loadDiff(fileDiff) with self.assertRaises(ValueError): extractor.extractFixCode() fixedCode = extractor.getFixCodeFragment() usedDiffs = extractor.getUsedDiffs() self.assertEqual(expectedFixedCode, fixedCode) self.assertEqual(0, len(usedDiffs))
def testCodeFixExtractSuccessRemove(self): bugData = self.getCorrectBugDataFileMiddle() expectedFixedCode = """int main(void) { int a; a = 0; if (a == 0) { """ fileDiff = self.getFileDiff(bugData.getStartLine(), [], [' a = 3;']) extractor = CodeExtractor(bugData) extractor.loadCodeFromFile() extractor.extractBugCode() extractor.loadDiff(fileDiff) extractor.extractFixCode() fixedCode = extractor.getFixCodeFragment() usedDiffs = extractor.getUsedDiffs() self.assertEqual(expectedFixedCode, fixedCode) self.assertEqual(1, len(usedDiffs)) self.assertEqual('8d8', usedDiffs[0])
def testExtractCodeWithDiffBetweenTwoCommitsMultiDiffInFragment(self): gp = gitprovider.GitProvider(config.getRepoDir()) commits = gp.getAllVersions('trainDbScriptTest') commit1 = commits[-6] commit2 = commits[-7] file1 = gp.getFileContents('bugcode3.cpp', commit1) file2 = gp.getFileContents('bugcode3.cpp', commit2) diff = LinuxDiffer().diff(file1, file2) usedDiffs = [] bugData = self.getBugData3() extractor = CodeExtractor(bugData) extractor.loadCodeFromText(file1, '\r\n', '\n') extractor.extractBugCode() extractor.loadDiff(diff) extractor.extractFixCode() bugCode = extractor.getBugCodeFragment() fixCode = extractor.getFixCodeFragment() usedDiffs = extractor.getUsedDiffs() expectedOutputFix = """{; int a; a = 0; a = 2; if (a != 0) {; int b = 1 / a; """ expectedOutputBug = """{ int a; a = 0; a = 2; if (a != 0) { int b = 1 / a; """ self.assertEqual(expectedOutputBug, bugCode) self.assertEqual(expectedOutputFix, fixCode) self.assertEqual(2, len(usedDiffs))
def extractCode(self, id): bugData = self.ccdb.getNotResolvedBugData(id) #TODO: Possible improvement for bugData if bugData is None: #TODO: Implement custom errors return None fileRelativePath = self.convertFilePathToRepoRelativePath( bugData.getFile()) try: fullCodeWithBug = self.vcs.getFileContents( fileRelativePath, self.commits[self.currentCommitIndex + 1]) fullCodeWithoutBug = self.vcs.getFileContents( fileRelativePath, self.commits[self.currentCommitIndex]) except KeyError as extractError: return None diff = POSIXDiffer().diff(fullCodeWithBug, fullCodeWithoutBug) extractor = CodeExtractor(bugData) try: extractor.extractAll(fullCodeWithBug, diff) except ValueError as extractError: return None bugCodeFragment = extractor.getBugCodeFragment() fixCodeFragment = extractor.getFixCodeFragment() usedDiffs = extractor.getUsedDiffs() #Easy version - ignore bug if none or more than one diff used to fix #TODO: Possible improvement here if len(usedDiffs) != 1: return None return entities.FixData(bugCodeFragment, fixCodeFragment, bugData.getChecker(), bugData.getMessage(), bugData.getLine() - bugData.getStartLine())
def ProcessBugsInFile(fileName): # 5.1. # 5.2. for bug in bugsPerFile[fileName]: bugData = bugDataList[bug] cleanFn = fileName[:-4] fn = '../Results/Analysis/{0}_{1}_{2}.txt'.format( cleanFn, bug, bugData.getChecker()) repoFn = os.fsdecode( os.path.join(os.fsencode(config.getRepoDir()), os.fsencode(fileName))) if os.path.isfile(fn): continue model = None if bugData.getChecker() == 'deadcode.DeadStores': model = model1 coder = coder1 totalDictionaryLength = totalDictionaryLength1 if bugData.getChecker( ) == 'clang-diagnostic-tautological-constant-out-of-range-compare': model = model2 coder = coder2 totalDictionaryLength = totalDictionaryLength2 if bugData.getChecker() == 'clang-diagnostic-unused-parameter': model = model3 coder = coder3 totalDictionaryLength = totalDictionaryLength3 if bugData.getChecker() == 'clang-diagnostic-constant-conversion': model = model4 coder = coder4 totalDictionaryLength = totalDictionaryLength4 MODEL_X_MAX_LEN = model.get_layer(index=0).input_shape[1] # 5.2.1. fullCodeWithBug = fileContents[fileName] extractor = CodeExtractor(bugData) extractor.loadCodeFromText(fullCodeWithBug) extractor.extractBugCode() bugCodeFragment = extractor.getBugCodeFragment() # 5.2.2. fullCodeWithoutBug = ''.join(fix(extractor.code, bugData)) diff = POSIXDiffer().diff(fullCodeWithBug, fullCodeWithoutBug) extractor.loadDiff(diff) try: extractor.extractFixCode() except ValueError as ve: print( "Unable to generate expected fix for bug #{0} ({1}), checker = {2}" .format(bug, fileName, bugData.getChecker())) continue expectedFixCodeFragment = extractor.getFixCodeFragment() # 5.2.3. checkerInfo = checkers.extractTokensForChecker(bugData.getChecker(), bugData.getMessage()) encodedBugData, initialUnkList = coder.encode(bugCodeFragment, checkerData=checkerInfo) noZerosToPad = MODEL_X_MAX_LEN - len(encodedBugData) if noZerosToPad > 0: encodedBugData = coder.applyPadding(encodedBugData, noZerosToPad) X = np.zeros((1, MODEL_X_MAX_LEN, totalDictionaryLength)) X[0] = coder.convertToOneHot( encodedBugData, np.zeros((MODEL_X_MAX_LEN, totalDictionaryLength))) Y = coder.convertFromOneHot(model.predict(X)[0]) while (Y[-1] == 0): Y.pop() fixCodeFragment = coder.decode(Y, initialUnkList) # 5.2.4. isCompiling = False f1 = os.fsdecode( os.path.join(os.fsencode(config.getRepoDir()), os.fsencode(cleanFn))) f2 = os.fsdecode( os.path.join(os.fsencode(config.getRepoDir()), os.fsencode('{0}.o'.format(cleanFn)))) if os.path.isfile(f1): os.remove(f1) if os.path.isfile(f2): os.remove(f2) extractor.loadCodeFromText(fullCodeWithBug) extractor.applyFix(fixCodeFragment) with open(repoFn, 'wt') as f: f.writelines(extractor.code) log = codechecker.runCmd( 'CodeChecker check -e all -b "cd {0} && make {1}" -o /tmp/codefixer_{1}' .format(config.getRepoDir(), cleanFn)) if os.path.isfile(f1): isCompiling = True # 5.2.5. isFixed = False ids = [] if isCompiling: resolved = codechecker.diffResolved( config.getCcRunName(), '/tmp/codefixer_{0}'.format(cleanFn), ccdb) for bugInfo in resolved: ids.append(bugInfo['reportId']) if bug in ids: isFixed = True # 5.2.6. isExpected = False if fixCodeFragment == expectedFixCodeFragment: isExpected = True # 5.2.7. encodedExpFix, finalUnkList = coder.encode(expectedFixCodeFragment, unkList=initialUnkList, reverse=False) noXTokens = len(encodedExpFix) noYTokens = len(Y) noAllTokens = max(noXTokens, noYTokens) noCorrectTokens = 0 #print(encodedExpFix) #print(Y) for i in range(min(noXTokens, noYTokens)): if encodedExpFix[i] == Y[i]: noCorrectTokens += 1 # 5.2.8. with open(fn, 'wt') as f: f.write( '#BUG#\n{0}\n#EXP#\n{1}\n#FIX#\n{2}\n#STATS#\n{3},{4},{5},{6},{7},{8},{9}\n{10}\n{11}' .format(bugCodeFragment, expectedFixCodeFragment, fixCodeFragment, isCompiling, isFixed, isExpected, noXTokens, noYTokens, noAllTokens, noCorrectTokens, log, ids)) # 5.2.9. with open(repoFn, 'wt') as f: f.write(fileContents[fileName]) # 5.2.10. # Not used due to multithreading issues, will be done on file-by-file basis after closing pool # 5.3. return fileName