Example #1
0
    def setUp(self):

        self.testChunk = logChunk.logChunk("")
        #Read in the full tests
        self.chunk1 = logChunk.logChunk(self.readHelper("testChunk1.txt"))
        self.chunk2 = logChunk.logChunk(self.readHelper("testChunk2.txt"))
        self.chunk3 = logChunk.logChunk(self.readHelper("testChunk3.txt"))
        self.chunk4 = logChunk.logChunk(self.readHelper("testChunk4.txt"))
        self.chunk5 = logChunk.logChunk(self.readHelper("testChunk5.txt"))
        self.chunk6 = logChunk.logChunk(self.readHelper("testChunk6.txt"))
        self.chunk7 = logChunk.logChunk(self.readHelper("testChunk7.txt"))
        self.chunk8 = logChunk.logChunk(self.readHelper("testChunk8.txt"))
        self.chunk9 = logChunk.logChunk(self.readHelper("testChunk9.txt"))
Example #2
0
def main():
    print("Utility to BULK process github logs")

    if len(sys.argv) < 2:
        print("!!! Usage: python allRun.py top_project directory")
        sys.exit()
    if not os.path.isdir("../Results"):
        os.mkdir("../Results")

    fPtrChangeSummary = open("../Results/" + "ChangeSummary.csv", 'w')
    fPtrChangeSummary.write(
        "project,sha,author,author_email,commit_date,is_bug\n")
    fPtrPatchSummary = open("../Results/" + "PatchSummary.csv", 'w')
    fPtrMisMatchSummary = open("../Results/" + "MisMatchSummary.csv", 'w')
    fPtrMisMatchSummary.write(
        "project,Total,Match,MisMatch,Exception,matchException,misMatchException\n"
    )
    lst = []
    listToDict = {}
    mockChunk = logChunk("")
    mockChunk.readKeywords(lst)
    keywords = [sub_list[0] for sub_list in lst]
    for keyword in keywords:
        listToDict[str(keyword) + " Adds"] = 0
        listToDict[str(keyword) + " Dels"] = 0
    #fPtrPatchSummary.write("project, sha, language, file_name, is_test,bracket_diff,isExceptionPatch, method_name,total_add,total_del,uniqueExcepAdd,uniqueExcepDel,%s\n"%",".join(listToDict.keys()))
    fPtrPatchSummary.write(
        "project, sha, language, file_name, is_test, method_name,total_add,total_del,%s\n"
        % ",".join(sorted(listToDict.keys())))
    fPtrChangeSummary.close()
    fPtrPatchSummary.close()
    fPtrMisMatchSummary.close()

    rootdir = str(sys.argv[1])

    for dir in os.listdir(rootdir):
        path = os.path.join(rootdir, dir)
        print(path)
        os.system('python ghProc.py %s' % path)
Example #3
0
def main():
  print "Utility to BULK process github logs"

  if len(sys.argv) < 2:
  	print "!!! Usage: python allRun.py top_project directory"
  	sys.exit()
  if not os.path.isdir("../Results"):
            os.mkdir("../Results")

  fPtrChangeSummary=open("../Results/"+"ChangeSummary.csv",'w')
  fPtrChangeSummary.write("project,sha,author,commit_date,is_bug\n")
  fPtrPatchSummary=open("../Results/"+"PatchSummary.csv",'w')
  fPtrMisMatchSummary=open("../Results/"+"MisMatchSummary.csv",'w')
  fPtrMisMatchSummary.write("project,Total,Match,MisMatch,Exception,matchException,misMatchException\n")
  lst=[]
  listToDict={}
  mockChunk=logChunk("")
  mockChunk.readKeywords(lst)
  keywords= [sub_list[0] for sub_list in lst]
  for keyword in keywords:
      listToDict[str(keyword)+" Adds"]=0
      listToDict[str(keyword)+" Dels"]=0
  #fPtrPatchSummary.write("project, sha, language, file_name, is_test,bracket_diff,isExceptionPatch, method_name,total_add,total_del,uniqueExcepAdd,uniqueExcepDel,%s\n"%",".join(listToDict.keys()))
  fPtrPatchSummary.write("project, sha, language, file_name, is_test, method_name,total_add,total_del,%s\n"%",".join(sorted(listToDict.keys())))
  fPtrChangeSummary.close()
  fPtrPatchSummary.close()
  fPtrMisMatchSummary.close()





  rootdir = str(sys.argv[1])

  for dir in os.listdir(rootdir):
    path= os.path.join(rootdir,dir)
    print path
    os.system('python ghProc.py %s'%path)
Example #4
0
    def processLog(self, config = ""):
        if(config == ""):
            config = self.config_info.CONFIG

        signal.signal(signal.SIGALRM, timeout)

        project1 = os.path.split(self.log_file)[0]
        project1 = project1.rstrip(os.sep)
        self.project_name = os.path.basename(project1)
        print("---------- %s ------------\n" % (self.project_name))

        if(self.config_info.DATABASE):
            dl = dumpLogs(self.dbPass, self.config_info)

        if(self.config_info.CSV):
            if not os.path.isdir("../Results"):
                os.mkdir("../Results")
            inf1=open("../Results/"+str(self.project_name)+"ChangeSummary.csv",'w')
            fPtrChangeSummary=open("../Results/"+"ChangeSummary.csv",'w')

            inf1.write("project,sha,author,commit_date,is_bug\n")

            inf2=open("../Results/"+str(self.project_name)+"PatchSummary.csv",'w')
            fPtrPatchSummary=open("../Results/"+"PatchSummary.csv",'w')

            lst=[]
            listToDict={}
            mockChunk=logChunk("", "C", self.config_info) #TODO: This is C specific,  Why is this C specific?
            lst = mockChunk.readKeywords(lst)
            keywords= [k[0] for k in lst if k[1] == INCLUDED]
            for keyword in keywords:
                listToDict[str(keyword)+" Adds"]=0
                listToDict[str(keyword)+" Dels"]=0

            inf2.write("project, sha, language, file_name, is_test, method_name,total_add,total_del,%s\n"%",".join(sorted(listToDict.keys())))

        inf = codecs.open(self.log_file, "r", "iso-8859-1")

        shaObj   = None
        patchObj = None
        is_diff  = False
        log_mssg = ""
        is_no_prev_ver = False
        is_no_next_ver = False
        curLogChunk = logChunk("", "C", self.config_info)
        linenum = 0

        for l in inf:

            try:
                signal.alarm(0)

                sha  = self.isSha(l)
                line = l


                #if(self.config_info.DEBUGLITE):
                #    try:
                #        print(line)
                #    except:
                #        pass

                if sha:
                    #Reverting back to version that outputs at the end...
                    #if(shaObj != None):
                    #    if(self.config_info.DEBUGLITE):
                    #        print("Writing Sha:" + sha)

                    #    if(self.config_info.DATABASE):            
                    #        shaObj.dumpSha(dl)
                    #    elif(self.config_info.CSV):
                    #        shaObj.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary)
                    #    else:
                    #        shaObj.printSha()
          
                    shaObj = Sha(self.project_name, sha)
                    #if(self.config_info.DEBUGLITE): #Save for testing.
                    self.shas.append(shaObj) #This will become very memory intensive in large git logs.
                    
                    is_diff = False
                    log_mssg = ""
                    
                    continue

                elif self.isAuthor(line,shaObj):
                    continue

                elif self.isDate(line,shaObj):
                    continue

                fullLine=line
                line=line.rstrip()

                if line.startswith('diff --git '):
                    shaObj.setLog(log_mssg)
                    is_diff = True
                    is_no_prev_ver = False
                    is_no_next_ver = False
                    continue

                    if patchObj != None:
                        shaObj.patches.append(patchObj)

                elif is_diff == False:
                    if not line.strip():
                        continue
                    log_mssg += line + "\t"


                if is_diff:
                    if line.startswith("--- a/"):
                        #Finish the changes to the old patch object
                        if(patchObj != None):
                            #If there is an existing chunk to parse, process it
                            if(curLogChunk.header != ""):
                                if(self.config_info.DEBUG): 
                                    print("New diff with previous version: " + line)
                                    print("HEADER: " + curLogChunk.header)
                                self.processLastChunk(patchObj, curLogChunk)
                            
                            #Reset the current chunk obj
                            if (self.config_info.DEBUG):
                                print("Resetting.")
                            curLogChunk.reset()
                            curLogChunk.setLang("." + self.cur_lang) #DOUBLE CHECK ME!

                        patchObj = self.createPatch(line)
                        shaObj.patches.append(patchObj)
                        #print patchObj
                        #print shaObj.patches
                    elif (line == '--- /dev/null'): #earlier file was empty
                        is_no_prev_ver = True
                    elif (line == '+++ /dev/null'): #next file version was empty
                        is_no_next_ver = True
                        continue
                    elif (is_no_prev_ver == True) and line.startswith("+++ b/"):
                        #Finish the changes to the old patch object
                        if(patchObj != None):
                            if(curLogChunk.header != ""): #If there is an existing chunk
                                if (self.config_info.DEBUG): 
                                    print("New diff with no previous version: " + line)
                                    print("HEADER: " + curLogChunk.header)
                                self.processLastChunk(patchObj, curLogChunk)

                                if (self.config_info.DEBUG):
                                    print("Resetting.")
                                curLogChunk.reset()
                                curLogChunk.setLang("." + self.cur_lang) #DOUBLE CHECK ME!

                        patchObj = self.createPatchWithNoPrevVersion(line)
                        shaObj.patches.append(patchObj)
                    else: #Then we reached a content line.
                        self.processPatch(fullLine, patchObj, curLogChunk)

            except TimeExceededError.TimeExceededError:
                print("Line Timed out, moving to next.")
                continue

        #Clear timeouts.
        signal.alarm(0)

        #Make sure to get the last patch in the file!
        if(curLogChunk.header != ""): #If there is an existing chunk to parse
            if(self.config_info.DEBUG): 
                print("Last Patch: " + line)
                print("HEADER: " + curLogChunk.header)
            self.processLastChunk(patchObj, curLogChunk)

        #if shaObj != None:
        #    shaObj.patches.append(patchObj)

        parseFinish = datetime.now()

        if(self.shas != []): #If the log wasn't empty...
            #Create the change summary table and the method change table now if necessary
            if(self.config_info.DATABASE):
                cfg = Config(self.config_info.CONFIG)
                keywordFile = cfg.ConfigSectionMap("Keywords")
                full_title = dumpLogs.getFullTitleString(curLogChunk.getEmptyKeywordDict())

                dl.createSummaryTable()

                if(full_title != ""): #Check if the changes table exists and create it if we have a title.
                    dl.createMethodChangesTable(full_title)

            for s in self.shas:
                #s.printSha()
                if s != None:
                   if(self.config_info.DATABASE):            
                       s.dumpSha(dl)
                   elif(self.config_info.CSV):
                       s.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary)
                   else:
                       s.printSha()


        #Write out last sha.
        #if(shaObj != None and self.config_info.DATABASE):
        #    if(self.config_info.DEBUGLITE):
        #        print("Writing to db.")
        #    shaObj.dumpSha(dl)

        if(self.config_info.DATABASE):
            print("Closing Time.")
            dl.close()
        
        if(self.config_info.CSV):
            inf1.close()
            inf2.close()
            fPtrChangeSummary.close()
            fPtrPatchSummary.close()

        print("Sha's processed:")
        print(len(self.shas))

        return parseFinish
Example #5
0
    def setUp(self):
        self.method1 = "def debugFunctions(self, funcList):"
        self.method2 = "def _Funct1():"
        self.method3 = " def Another_function(arg1, arg2 = 5):    "
        self.method4 = "def test( _args ): "
        self.method5 = "def 1Bad():"
        self.method6 = "def functionname([formal_args,] *var_args_tuple ):"
        self.method7 = "def alsoBad() "
        self.method8 = "noFunct():"
        self.method9 = " def        okay(args = 4):"

        c_info = ConfigInfo("../util/pytest.ini")

        self.testChunk = logChunk.logChunk("", "Python")

        self.chunk1 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk1.txt"), "Python",
            c_info)
        self.chunk2 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk2.txt"), "Python",
            c_info)
        self.chunk3 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk3.txt"), "Python",
            c_info)
        self.chunk4 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk4.txt"), "Python",
            c_info)
        self.chunk5 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk5.txt"), "Python",
            c_info)
        self.chunk6 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk6.txt"), "Python",
            c_info)
        self.chunk7 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk7.txt"), "Python",
            c_info)
        self.chunk8 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk8.txt"), "Python",
            c_info)
        self.chunk9 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk9.txt"), "Python",
            c_info)
        self.chunk10 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk10.txt"), "Python",
            c_info)
        self.chunk11 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk11.txt"), "Python",
            c_info)
        self.chunk12 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk12.txt"), "Python",
            c_info)
        self.chunk13 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk13.txt"), "Python",
            c_info)
        self.chunk14 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk14.txt"), "Python",
            c_info)
        self.chunk15 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk15.txt"), "Python",
            c_info)
        self.chunk16 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk16.txt"), "Python",
            c_info)
        self.chunk17 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk17.txt"), "Python",
            c_info)
        self.chunk18 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk18.txt"), "Python",
            c_info)
        self.chunk19 = logChunk.logChunk(
            self.readHelper("testfiles/Python/testChunk19.txt"), "Python",
            c_info)
Example #6
0
    def processLog(self, config=""):
        if (config == ""):
            config = self.config_info.CONFIG

        signal.signal(signal.SIGALRM, timeout)

        project1 = os.path.split(self.log_file)[0]
        project1 = project1.rstrip(os.sep)
        self.project_name = os.path.basename(project1)
        print(("---------- %s ------------\n" % (self.project_name)))

        if (self.config_info.DATABASE):
            dl = dumpLogs(self.dbPass, self.config_info)

        if (self.config_info.CSV):
            if not os.path.isdir("../Results"):
                os.mkdir("../Results")
            inf1 = open(
                "../Results/" + str(self.project_name) + "ChangeSummary.csv",
                'w')
            fPtrChangeSummary = open("../Results/" + "ChangeSummary.csv", 'w')

            inf1.write("project,sha,author,author_email,commit_date,is_bug\n")

            inf2 = open(
                "../Results/" + str(self.project_name) + "PatchSummary.csv",
                'w')
            fPtrPatchSummary = open("../Results/" + "PatchSummary.csv", 'w')

            lst = []
            listToDict = {}
            mockChunk = logChunk(
                "", "C", self.config_info
            )  #TODO: This is C specific,  Why is this C specific?
            lst = mockChunk.readKeywords(lst)
            keywords = [k[0] for k in lst if k[1] == INCLUDED]
            for keyword in keywords:
                listToDict["\"" + str(keyword) + "\" adds"] = 0
                listToDict["\"" + str(keyword) + "\" dels"] = 0

            inf2.write(
                "project, sha, language, file_name, is_test, method_name,total_add,total_del,%s\n"
                % ",".join(sorted(listToDict.keys())))

        inf = codecs.open(self.log_file, "r", "iso-8859-1")

        shaObj = None
        patchObj = None
        is_diff = False
        log_mssg = ""
        is_no_prev_ver = False
        is_no_next_ver = False
        curLogChunk = logChunk("", "C", self.config_info)
        linenum = 0

        for l in inf:

            try:
                signal.alarm(0)

                sha = self.isSha(l)
                line = l

                #if(self.config_info.DEBUGLITE):
                #    try:
                #        print(line)
                #    except:
                #        pass

                if sha:
                    #Reverting back to version that outputs at the end...
                    #if(shaObj != None):
                    #    if(self.config_info.DEBUGLITE):
                    #        print("Writing Sha:" + sha)

                    #    if(self.config_info.DATABASE):
                    #        shaObj.dumpSha(dl)
                    #    elif(self.config_info.CSV):
                    #        shaObj.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary)
                    #    else:
                    #        shaObj.printSha()

                    shaObj = Sha(self.project_name, sha)
                    #if(self.config_info.DEBUGLITE): #Save for testing.
                    self.shas.append(
                        shaObj
                    )  #This will become very memory intensive in large git logs.

                    is_diff = False
                    log_mssg = ""

                    continue

                elif self.isAuthor(line, shaObj):
                    continue

                elif self.isDate(line, shaObj):
                    continue

                fullLine = line
                line = line.rstrip()

                if line.startswith('diff --git '):
                    shaObj.setLog(log_mssg)
                    is_diff = True
                    is_no_prev_ver = False
                    is_no_next_ver = False
                    continue

                    if patchObj != None:
                        shaObj.patches.append(patchObj)

                elif is_diff == False:
                    if not line.strip():
                        continue
                    log_mssg += line + "\t"

                if is_diff:
                    if line.startswith("--- a/"):
                        #Finish the changes to the old patch object
                        if (patchObj != None):
                            #If there is an existing chunk to parse, process it
                            if (curLogChunk.header != ""):
                                if (self.config_info.DEBUG):
                                    print(("New diff with previous version: " +
                                           line))
                                    print(("HEADER: " + curLogChunk.header))
                                self.processLastChunk(patchObj, curLogChunk)

                            #Reset the current chunk obj
                            if (self.config_info.DEBUG):
                                print("Resetting.")
                            curLogChunk.reset()
                            curLogChunk.setLang(
                                "." + self.cur_lang)  #DOUBLE CHECK ME!

                        patchObj = self.createPatch(line)
                        shaObj.patches.append(patchObj)
                        #print patchObj
                        #print shaObj.patches
                    elif (line == '--- /dev/null'):  #earlier file was empty
                        is_no_prev_ver = True
                    elif (line == '+++ /dev/null'
                          ):  #next file version was empty
                        is_no_next_ver = True
                        continue
                    elif (is_no_prev_ver
                          == True) and line.startswith("+++ b/"):
                        #Finish the changes to the old patch object
                        if (patchObj != None):
                            if (curLogChunk.header !=
                                    ""):  #If there is an existing chunk
                                if (self.config_info.DEBUG):
                                    print((
                                        "New diff with no previous version: " +
                                        line))
                                    print(("HEADER: " + curLogChunk.header))
                                self.processLastChunk(patchObj, curLogChunk)

                                if (self.config_info.DEBUG):
                                    print("Resetting.")
                                curLogChunk.reset()
                                curLogChunk.setLang(
                                    "." + self.cur_lang)  #DOUBLE CHECK ME!

                        patchObj = self.createPatchWithNoPrevVersion(line)
                        shaObj.patches.append(patchObj)
                    else:  #Then we reached a content line.
                        self.processPatch(fullLine, patchObj, curLogChunk)

            except TimeExceededError.TimeExceededError:
                print("Line Timed out, moving to next.")
                continue

        #Clear timeouts.
        signal.alarm(0)

        #Make sure to get the last patch in the file!
        if (curLogChunk.header != ""):  #If there is an existing chunk to parse
            if (self.config_info.DEBUG):
                print(("Last Patch: " + line))
                print(("HEADER: " + curLogChunk.header))
            self.processLastChunk(patchObj, curLogChunk)

        #if shaObj != None:
        #    shaObj.patches.append(patchObj)

        parseFinish = datetime.now()

        if (self.shas != []):  #If the log wasn't empty...
            #Create the change summary table and the method change table now if necessary
            if (self.config_info.DATABASE):
                cfg = Config(self.config_info.CONFIG)
                keywordFile = cfg.ConfigSectionMap("Keywords")
                full_title = dumpLogs.getFullTitleString(
                    curLogChunk.getEmptyKeywordDict())

                dl.createSummaryTable()

                if (
                        full_title != ""
                ):  #Check if the changes table exists and create it if we have a title.
                    dl.createMethodChangesTable(full_title)

            for s in self.shas:
                #s.printSha()
                if s != None:
                    if (self.config_info.DATABASE):
                        s.dumpSha(dl)
                    elif (self.config_info.CSV):
                        s.shaToCsv(inf1, inf2, fPtrChangeSummary,
                                   fPtrPatchSummary)
                    else:
                        s.printSha()

        #Write out last sha.
        #if(shaObj != None and self.config_info.DATABASE):
        #    if(self.config_info.DEBUGLITE):
        #        print("Writing to db.")
        #    shaObj.dumpSha(dl)

        if (self.config_info.DATABASE):
            print("Closing Time.")
            dl.close()

        if (self.config_info.CSV):
            inf1.close()
            inf2.close()
            fPtrChangeSummary.close()
            fPtrPatchSummary.close()

        print("Sha's processed:")
        print((len(self.shas)))

        return parseFinish
    def setUp(self):
        self.javaMethod1 = "public static Intent createIntent(Context context, String username, String password) {"
        self.javaMethod2 = " public <V> V post(final String uri, final Object params, final Type type) \n throws IOException {"
        self.javaMethod3 = "public static Intent createIntent(final Collection<? extends Issue> issues,\n final Repository repository, final int position) {"
        self.javaMethod4 = "@Override \n public List<User> run(Account account) throws Exception {"
        self.javaMethod5 = "private JClass typeBoundsToJClass(GeneratedClassHolder holder, List<? extends TypeMirror> bounds, Map<String, TypeMirror> actualTypes) {"
        self.javaMethod6 = " public JMethod implementMethod(GeneratedClassHolder holder, List<ExecutableElement> methods, String methodName, String returnType, String... parameterTypes) {"
        self.javaMethod7 = "ProgrammerInterview pInstance = new    ProgrammerInterview() {\npublic void read() {"

        self.testChunk2 = logChunk.logChunk("", "Java")

        #Read in the block tests
        self.chunkb1 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk1.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb2 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk2.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb3 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk3.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb4 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk4.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb5 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk5.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb6 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk6.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb7 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk7.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb8 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk8.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb9 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk9.txt"), "Java", "../util/sample_conf2.ini")
        self.chunkb10 = logChunk.logChunk(self.readHelper("testfiles/Block/testChunk10.txt"), "Java", "../util/sample_conf2.ini")
Example #8
0
    def setUp(self):
        self.keyword1 = ["wtc/Assertions.h","excluded","single"]
        self.keyword2 = ["ut_ad","included","single"]
        self.keyword3 = ["try","Included","BLOCK"]
        self.keyword4 = ["for","excludeD","block"]
        self.keyword5 = ["ut_ad included single"]
        self.keyword6 = ["printf(format","str)","included","single"]
        self.keyword7 = ["printf(format, str)","included","single"]
        self.keyword8 = ["assert","incuded","single"]
        self.keyword9 = ["assert","included","lock"]


        self.method1 = "static void blarg() {"
        self.method2 = "int more(int stuff) {"
        self.method3 = "ccv_string * getStuff (int[] stuffToGet) {"
        self.method4 = "int add2to3(int (*functionPtr)(int, int)) {"
        self.method5 = "public static void other(int one, int (*functionPtr)(int, int)) {"
        self.method6 = "static void\n multiline(\n int arg1, string arg2\n) {"
        self.method7 = "int lotsOfSpace     (int stuff) {"
        self.method8 = "                .matrix = {"
        self.method9 = "ccv_string* getStuff (int[] stuffToGet) {"
        self.method10 = "ccv_string *getStuff (int[] stuffToGet) {"
        self.method11 = "void NdbBlob::getBlobEvent(NdbEventImpl& be, const NdbEventImpl* e, const NdbColumnImpl* c) {"
        self.method12 = "bool Repair_mrg_table_error_handler::handle_condition(THD *,uint sql_errno,const char*, MYSQL_ERROR::enum_warning_level level,const char*,MYSQL_ERROR ** cond_hdl) {"
        self.method13 = "(_log2) += 1;  static int CeilingLog2(unsigned int i) {"
        #self.method14 = "(STATEMENT_CONSTRUCTOR_BASE_PARAMETERS, ExpressionPtr exp)   : Statement(STATEMENT_CONSTRUCTOR_BASE_PARAMETER_VALUES),     m_exp(exp) {"
        self.method15 = "static JSC::UString& globalExceptionString(){"
        self.method16 = "(jint) AWT_WINDOW_LOST_FOCUS, (jint) AWT_WINDOW_DEACTIVATED,  static gboolean window_focus_in_cb (GtkWidget * widget, GdkEventFocus *event, jobject peer) {"
        self.method17 = "LinuxPtraceDumper dumper(getpid()); }  TEST(LinuxPtraceDumperTest, FindMappings) {"
        self.method18 = "endif  defined(HAVE_ALTIVEC)   include  elif defined(HAVE_SSE2)   include  endif   ONLY64 inline static int idxof(int i) {"
        self.method19 = "#define UNUSED_VARIABLE(x) (void)(x) const char *sfmt_get_idstring(sfmt_t * sfmt) {"
        self.method20 = "static av_always_inline void hl_motion_420(H264Context *h, uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,            qpel_mc_func (*qpix_put)[16], h264_chroma_mc_func (*chroma_put),       qpel_mc_func (*qpix_avg)[16], h264_chroma_mc_func (*chroma_avg),            h264_weight_func *weight_op, h264_biweight_func *weight_avg,             int pixel_shift) {"
        self.method21 = "        for (j = 0; j < i; j++)             if (!memcmp(h->pps.scaling_matrix8[j], h->pps.scaling_matrix8[i],                         64 * sizeof(uint8_t))) {"
        self.method22 = " c_type jl_unbox_##j_type(jl_value_t *v)  {"

        self.testChunk = logChunk.logChunk("", "C++")
        self.testChunk_C = logChunk.logChunk("", "C")

        self.javaMethod1 = "public static Intent createIntent(Context context, String username, String password) {"
        self.javaMethod2 = " public <V> V post(final String uri, final Object params, final Type type) \n throws IOException {"
        self.javaMethod3 = "public static Intent createIntent(final Collection<? extends Issue> issues,\n final Repository repository, final int position) {"
        self.javaMethod4 = "@Override \n public List<User> run(Account account) throws Exception {"
        self.javaMethod5 = "private JClass typeBoundsToJClass(GeneratedClassHolder holder, List<? extends TypeMirror> bounds, Map<String, TypeMirror> actualTypes) {"
        self.javaMethod6 = " public JMethod implementMethod(GeneratedClassHolder holder, List<ExecutableElement> methods, String methodName, String returnType, String... parameterTypes) {"


        #Read in the single tests
        self.chunk1 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk1.txt"), "C++") #Check C++
        self.chunk2 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk2.txt"), "C") #Check C
        self.chunk3 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk3.txt"), "C++") #Check C++
        #self.chunk4 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk4.txt")) #Nope
        #self.chunk5 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk5.txt")) #Nope
        self.chunk6 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk6.txt"), "C++") #Check C++
        self.chunk7 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk7.txt"), "C++") #Check C++
        self.chunk8 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk8.txt"), "C++") #Check C
        self.chunk9 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk9.txt"), "C++") #Check C++
        self.chunk10 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk10.txt"), "C++") #Check C++
        self.chunk11 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk11.txt"), "C++") #Check C++
        self.chunk12 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk12.txt"), "C++") #Check C++
        self.chunk13 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk13.txt"), "C++") #Check C++
        self.chunk14 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk14.txt"), "C++") #Check C++
        self.chunk15 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk15.txt"), "C++") #Check C++
        #self.chunk16 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk16.txt")) #Nope
        #self.chunk17 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk17.txt")) #Nope
        #self.chunk18 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk18.txt")) #Nope
        #self.chunk19 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk19.txt")) #Nope
        #self.chunk20 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk20.txt")) #Nope
        self.chunk21 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk21.txt"), "C++") #Check C++ 
        self.chunk22 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk22.txt"), "C++") #Check C++
        self.chunk23 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk23.txt"), "C++") #Check C++
        self.chunk24 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk24.txt"), "C++") #Check C++
        self.chunk25 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk25.txt"), "C") #Check C
        #self.chunk26 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk26.txt")) #Nope
        self.chunk27 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk27.txt"), "C") #Check C
        #self.chunk28 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk28.txt")) #Nope
        self.chunk29 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk29.txt"), "C") #Check C
        #self.chunk30 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk30.txt")) #Maybe?
        self.chunk31 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk31.txt"), "C++") #Check C++
        self.chunk32 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk32.txt"), "C++") #Check C++
        self.chunk33 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk33.txt"), "C++") #Check C++
        #self.chunk34 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk34.txt")) #Nope
        self.chunk35 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk35.txt"), "C") #Check C
        self.chunk36 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk36.txt"), "C++") #Check C++
        self.chunk37 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk37.txt"), "C++") #Check C++
        self.chunk38 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk38.txt"), "C++") #Check C++
        self.chunk39 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk39.txt"), "C") #C
        self.chunk40 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk40.txt"), "C") #Check C
        self.chunk41 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk41.txt"), "C++") #Check C++
        self.chunk42 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk42.txt"), "C++") # C++
        self.chunk43 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk43.txt"), "C") #Check C
        self.chunk44 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk44.txt"), "C++") # C++
        self.chunk45 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk45.txt"), "C++") # C++ 
        self.chunk46 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk46.txt"), "C++") # C++
        self.chunk47 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk47.txt"), "C++") # C++
        self.chunk48 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk48.txt"), "C++") # C++
Example #9
0
import sys
Example #10
0
import sys
Example #11
0
    def setUp(self):
        self.method1 = "def debugFunctions(self, funcList):"
        self.method2 = "def _Funct1():"
        self.method3 = " def Another_function(arg1, arg2 = 5):    "
        self.method4 = "def test( _args ): "
        self.method5 = "def 1Bad():"
        self.method6 = "def functionname([formal_args,] *var_args_tuple ):"
        self.method7 = "def alsoBad() "
        self.method8 = "noFunct():"
        self.method9 = " def        okay(args = 4):"


        c_info = ConfigInfo("../util/pytest.ini")

        self.testChunk = logChunk.logChunk("", "Python")

        self.chunk1 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk1.txt"), "Python", c_info)
        self.chunk2 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk2.txt"), "Python", c_info)
        self.chunk3 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk3.txt"), "Python", c_info)
        self.chunk4 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk4.txt"), "Python", c_info)
        self.chunk5 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk5.txt"), "Python", c_info)
        self.chunk6 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk6.txt"), "Python", c_info)
        self.chunk7 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk7.txt"), "Python", c_info)
        self.chunk8 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk8.txt"), "Python", c_info)
        self.chunk9 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk9.txt"), "Python", c_info)
        self.chunk10 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk10.txt"), "Python", c_info)
        self.chunk11 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk11.txt"), "Python", c_info)
        self.chunk12 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk12.txt"), "Python", c_info)
        self.chunk13 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk13.txt"), "Python", c_info)
        self.chunk14 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk14.txt"), "Python", c_info)
        self.chunk15 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk15.txt"), "Python", c_info)
        self.chunk16 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk16.txt"), "Python", c_info)
        self.chunk17 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk17.txt"), "Python", c_info)
        self.chunk18 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk18.txt"), "Python", c_info)
        self.chunk19 = logChunk.logChunk(self.readHelper("testfiles/Python/testChunk19.txt"), "Python", c_info)
Example #12
0
    def processLog(self, config = Util.CONFIG):

        project1 = os.path.split(self.log_file)[0]
        project1 = project1.rstrip(os.sep)
        self.project_name = os.path.basename(project1)
        print("---------- %s ------------\n" % (self.project_name))

        if(Util.DATABASE == 1):
            dl = dumpLogs()

        if(Util.CSV==1):
            if not os.path.isdir("../Results"):
                os.mkdir("../Results")
            inf1=open("../Results/"+str(self.project_name)+"ChangeSummary.csv",'w')
            fPtrChangeSummary=open("../Results/"+"ChangeSummary.csv",'a')

            inf1.write("project,sha,author,commit_date,is_bug\n")

            inf2=open("../Results/"+str(self.project_name)+"PatchSummary.csv",'w')
            fPtrPatchSummary=open("../Results/"+"PatchSummary.csv",'a')

            lst=[]
            listToDict={}
            mockChunk=logChunk("", "C")
            mockChunk.readKeywords(lst)
            keywords= [sub_list[0] for sub_list in lst]
            for keyword in keywords:
                listToDict[str(keyword)+" Adds"]=0
                listToDict[str(keyword)+" Dels"]=0

            inf2.write("project, sha, language, file_name, is_test,isExceptionPatch, method_name,total_add,total_del,%s\n"%",".join(listToDict.keys()))

        inf = codecs.open(self.log_file, "r", "iso-8859-1")

        shaObj   = None
        patchObj = None
        is_diff  = False
        log_mssg = ""
        is_no_prev_ver = False
        is_no_next_ver = False
        curLogChunk = logChunk("", "C", config)
        linenum = 0

        for l in inf:
            sha  = self.isSha(l)
            line = l

            if sha:
                if(shaObj != None):
                    if(Util.DATABASE):            
                        shaObj.dumpSha(dl)
                    else:
                        shaObj.printSha()
                        if(Util.CSV):
                            shaObj.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary)

                shaObj = Sha(self.project_name, sha)
                if(Util.DEBUGLITE): #Save for testing.
                    self.shas.append(shaObj) #This will become very memory intensive in large git logs.
                is_diff = False
                log_mssg = ""
                
                continue

            elif self.isAuthor(line,shaObj):
                continue

            elif self.isDate(line,shaObj):
                continue

            fullLine=line
            line=line.strip()

            if line.startswith('diff --git '):
                shaObj.setLog(log_mssg)
                is_diff = True
                is_no_prev_ver = False
                is_no_next_ver = False
                continue

                if patchObj != None:
                    shaObj.patches.append(patchObj)

            elif is_diff == False:
                if not line.strip():
                    continue
                log_mssg += line + "\t"


            if is_diff:
                if line.startswith("--- a/"):
                    #Finish the changes to the old patch object
                    if(patchObj != None):
                        #If there is an existing chunk to parse, process it
                        if(curLogChunk.header != ""):
                            if Util.DEBUG == 1: 
                                print("New diff with previous version: " + line)
                                print("HEADER: " + curLogChunk.header)
                            self.processLastChunk(patchObj, curLogChunk)
                        
                        #Reset the current chunk obj
                        if Util.DEBUG == 1:
                            print("Resetting.")
                        curLogChunk.reset()
                        curLogChunk.setLang("." + self.cur_lang) #DOUBLE CHECK ME!

                    patchObj = self.createPatch(line)
                    shaObj.patches.append(patchObj)
                    #print patchObj
                    #print shaObj.patches
                elif (line == '--- /dev/null'): #earlier file was empty
                    is_no_prev_ver = True
                elif (line == '+++ /dev/null'): #next file version was empty
                    is_no_next_ver = True
                    continue
                elif (is_no_prev_ver == True) and line.startswith("+++ b/"):
                    #Finish the changes to the old patch object
                    if(patchObj != None):
                        if(curLogChunk.header != ""): #If there is an existing chunk
                            if Util.DEBUG == 1: 
                                print("New diff with no previous version: " + line)
                                print("HEADER: " + curLogChunk.header)
                            self.processLastChunk(patchObj, curLogChunk)

                            if Util.DEBUG == 1:
                                print("Resetting.")
                            curLogChunk.reset()
                            curLogChunk.setLang("." + self.cur_lang) #DOUBLE CHECK ME!

                    patchObj = self.createPatchWithNoPrevVersion(line)
                    shaObj.patches.append(patchObj)
                else: #Then we reached a content line.
                    self.processPatch(fullLine, patchObj, curLogChunk)


        #if shaObj != None:
        #    shaObj.patches.append(patchObj)


        #Make sure to get the last patch in the file!
        if(curLogChunk.header != ""): #If there is an existing chunk to parse
            if Util.DEBUG == 1: 
                print("Last Patch: " + line)
                print("HEADER: " + curLogChunk.header)
            self.processLastChunk(patchObj, curLogChunk)

        #Write out last sha.
        if(shaObj != None and Util.DATABASE):
            if(Util.DEBUGLITE):
                print("Writing to db.")
            shaObj.dumpSha(dl)

        if(Util.DATABASE == 1):
            print("Closing Time.")
            dl.close()
        
        if(Util.CSV == 1):
            shaObj.printSha();
            shaObj.shaToCsv(inf1,inf2,fPtrChangeSummary,fPtrPatchSummary)
            inf1.close()
            inf2.close()
            fPtrChangeSummary.close()
            fPtrPatchSummary.close()