def processDiffs(self,proj,path): #self.got_some = {'java':True, 'cxx':True, 'hxx':True} self.got_some = {'java':False, 'cxx':True, 'hxx':False} self.pb = PathBuilder(self.tmpPath, force_clean = True) if os.path.isdir(path) is True: self.filterDiffProj(proj) elif os.path.isfile(path) is True: self.filterDiffFile(proj) else: return ('Invalid path : ' + path, False) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() progress = (self.operations_so_far.incr() / float(self.num_operations))*100 callback = lambda: self.progress('Converting to ccfx input format') converter.convert(proj, self.pb, callback) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) return ("Converting diffs to ccFinder compatible format is done",True)
def processDiffs(self, proj, path): #self.got_some = {'java':True, 'cxx':True, 'hxx':True} self.got_some = {'java': False, 'cxx': True, 'hxx': False} self.pb = PathBuilder(self.tmpPath, force_clean=True) if os.path.isdir(path) is True: self.filterDiffProj(proj) elif os.path.isfile(path) is True: self.filterDiffFile(proj) else: return ('Invalid path : ' + path, False) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() progress = (self.operations_so_far.incr() / float(self.num_operations)) * 100 callback = lambda: self.progress('Converting to ccfx input format') converter.convert(proj, self.pb, callback) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) return ("Converting diffs to ccFinder compatible format is done", True)
def processImpl(self, model): proj0 = model.getProj(PathBuilder.Proj0) proj1 = model.getProj(PathBuilder.Proj1) path_builder = model.getPathBuilder() converter = CCFXInputConverter() ccfx = CCFXEntryPoint(path_builder, model.getCcfxPath(), model.getCcfxTokenSize()) step = 0 total_steps = 20.0 final_status = False while not self.sync.stopRequested(): if step == 0: self.progress("Loading version histories for first project", step / total_steps) step += 1 proj0.load() elif step == 1: self.progress("Loading version histories for second project", step / total_steps) step += 1 proj1.load() elif step == 2: self.progress("Dumping commits for first project", step / total_steps) step += 1 proj0.dumpCommits() elif step == 3: self.progress("Dumping commits for second project", step / total_steps) step += 1 proj1.dumpCommits() elif step == 4: self.progress("Converting diffs to ccfx compatible format for first project", step / total_steps) step += 1 converter.convert(path_builder) elif step == 5: self.progress("Converting diffs to ccfx compatible format for second project", step / total_steps) step += 1 elif step == 6: self.progress("Running ccFinder for old C, this will take quite some time...", step / total_steps) step += 1 have_old_c = ccfx.processPairs(LangDecider.CXX, False) elif step == 7: self.progress("Running ccFinder for new C, this will take quite some time...", step / total_steps) step += 1 have_new_c = ccfx.processPairs(LangDecider.CXX, True) elif step == 8: self.progress("Running ccFinder for old headers, this will take quite some time...", step / total_steps) step += 1 have_old_h = ccfx.processPairs(LangDecider.HXX, False) elif step == 9: self.progress("Running ccFinder for new headers, this will take quite some time...", step / total_steps) step += 1 have_new_h = ccfx.processPairs(LangDecider.HXX, True) elif step == 10: self.progress("Running ccFinder for old Java, this will take quite some time...", step / total_steps) step += 1 have_old_j = ccfx.processPairs(LangDecider.JAVA, False) elif step == 11: self.progress("Running ccFinder for new Java, this will take quite some time...", step / total_steps) step += 1 have_new_j = ccfx.processPairs(LangDecider.JAVA, True) elif step == 12: self.progress("Filtering ccFinder old C output based on operation...", step / total_steps) step += 1 if not have_old_c: continue is_new = False lang = LangDecider.CXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) elif step == 13: self.progress("Filtering ccFinder new C output based on operation...", step / total_steps) step += 1 if not have_new_c: continue is_new = True lang = LangDecider.CXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) elif step == 14: self.progress("Filtering ccFinder old header output based on operation...", step / total_steps) step += 1 if not have_old_h: continue is_new = False lang = LangDecider.HXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) elif step == 15: self.progress("Filtering ccFinder new header output based on operation...", step / total_steps) step += 1 if not have_new_h: continue is_new = True lang = LangDecider.HXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) elif step == 16: self.progress("Filtering ccFinder old java output based on operation...", step / total_steps) step += 1 if not have_old_j: continue is_new = False lang = LangDecider.JAVA output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) elif step == 17: self.progress("Filtering ccFinder new java output based on operation...", step / total_steps) step += 1 if not have_new_j: continue is_new = True lang = LangDecider.JAVA output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) elif step == 18: self.progress( "Combining ccFinder output into a unified database...", step / total_steps) step += 1 pickle.dump(model, open(path_builder.getModelPathAndName(), 'w')) rep_populator = RepDBPopulator(path_builder) db = rep_populator.generateDB(proj0, proj1) db_file = open(path_builder.getDBPathAndName(), 'w') pickle.dump(db, db_file) db_file.close() else: final_status = True break if final_status: return 'Success!', final_status return 'Aborting', final_status
def filterDiffs(self, interface): self.got_some = {'java':True, 'cxx':True, 'hxx':True} # self.haveJava = haveC = haveH = False self.pb = PathBuilder(self.tmpPath, force_clean = True) # First, filter the input diffs by file type, so that all c diffs # are in one set of files, and similarly for java/headers if self.processDirectory is True: self.filterDiffProjs(interface) else: self.filterDiffFiles(interface) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() callback = lambda: interface.progress( 'Converting to ccfx input format', self.operations_so_far.incr() / float(self.num_operations)) converter.convert(self.pb, callback) #new and old for 3 langs self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) clone_path = self.pb.getCCFXOutputPath() # Third, call ccfx for each directory worked = True for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) continue old_path0 = self.pb.getCCFXInputPath(PathBuilder.PROJ0, lang, False) old_path1 = self.pb.getCCFXInputPath(PathBuilder.PROJ1, lang, False) new_path0 = self.pb.getCCFXInputPath(PathBuilder.PROJ0, lang, True) new_path1 = self.pb.getCCFXInputPath(PathBuilder.PROJ1, lang, True) tmp_old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = True) tmp_new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = True) old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = False) new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = False) if self.paths['proj0'] == self.paths['proj1']: old_path1 = old_path0 new_path1 = new_path0 worked = worked and self.ccfx.processPair( old_path0, old_path1, tmp_old_out, old_out, lang) interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) worked = worked and self.ccfx.processPair( new_path0, new_path1, tmp_new_out, new_out, lang) interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) if not worked: return ('ccFinderX execution failed', False) # Fourth, build up our database of clones print "Repertoire filtering...." #new and old for 3 langs self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: interface.progress('Repertoire filtering based on operation', self.operations_so_far.incr() / float(self.num_operations)) continue for is_new in [True, False]: output = convert_ccfx_output(self.pb, lang, is_new) rep_out_path = self.pb.getRepertoireOutputPath(lang, is_new) suffix = '_old.txt' if is_new: suffix = '_new.txt' output.writeToFile(rep_out_path + lang + suffix) interface.progress('Repertoire filtering based on operation', self.operations_so_far.incr() / float(self.num_operations)) print "Processing successful!!" return ('Processing successful', True)
def processImpl(self, model): proj0 = model.getProj(PathBuilder.Proj0) proj1 = model.getProj(PathBuilder.Proj1) path_builder = model.getPathBuilder() converter = CCFXInputConverter() ccfx = CCFXEntryPoint(path_builder, model.getCcfxPath(), model.getCcfxTokenSize()) step = 0 total_steps = 20.0 final_status = False # step == 0: print "Loading version histories for first project" + str(step / total_steps) step += 1 proj0.load() print "Loading version histories for second project" + str(step / total_steps) step += 1 proj1.load() print "Dumping commits for first project" + str(step / total_steps) step += 1 proj0.dumpCommits() print "Dumping commits for second project" + str(step / total_steps) step += 1 proj1.dumpCommits() print "Converting diffs to ccfx compatible format" + str(step / total_steps) step += 1 converter.convert(path_builder) print "Running ccFinder for old C, this will take quite some time..." + str(step / total_steps) step += 1 have_old_c = ccfx.processPairs(LangDecider.CXX, False) print "Running ccFinder for new C, this will take quite some time..." + str(step / total_steps) step += 1 have_new_c = ccfx.processPairs(LangDecider.CXX, True) print "Running ccFinder for old headers, this will take quite some time..." + str(step / total_steps) step += 1 have_old_h = ccfx.processPairs(LangDecider.HXX, False) print "Running ccFinder for new headers, this will take quite some time..." + str(step / total_steps) step += 1 have_new_h = ccfx.processPairs(LangDecider.HXX, True) print "Running ccFinder for old Java, this will take quite some time..." + str(step / total_steps) step += 1 have_old_j = ccfx.processPairs(LangDecider.JAVA, False) print "Running ccFinder for new Java, this will take quite some time..." + str(step / total_steps) step += 1 have_new_j = ccfx.processPairs(LangDecider.JAVA, True) print "Filtering ccFinder old C output based on operation..." + str(step / total_steps) step += 1 if have_old_c: is_new = False lang = LangDecider.CXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder new C output based on operation..." + str(step / total_steps) step += 1 if have_new_c: is_new = True lang = LangDecider.CXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder old header output based on operation..." + str(step / total_steps) step += 1 if have_old_h: is_new = False lang = LangDecider.HXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder new header output based on operation..." + str(step / total_steps) step += 1 if have_new_h: is_new = True lang = LangDecider.HXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder old java output based on operation..." + str(step / total_steps) step += 1 if not have_old_j: is_new = False lang = LangDecider.JAVA output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder new java output based on operation..." + str(step / total_steps) step += 1 if not have_new_j: is_new = True lang = LangDecider.JAVA output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Combining ccFinder output into a unified database..." + str(step / total_steps) step += 1 pickle.dump(model, open(path_builder.getModelPathAndName(), "w")) rep_populator = RepDBPopulator(path_builder) db = rep_populator.generateDB(proj0, proj1) db_file = open(path_builder.getDBPathAndName(), "w") pickle.dump(db, db_file) db_file.close() final_status = True if final_status: return "Success!", final_status return "Aborting", final_status
def __init__(self, model): self.proj0 = model.getProj(PathBuilder.Proj0) self.proj1 = model.getProj(PathBuilder.Proj1) self.path_builder = model.getPathBuilder() self.converter = CCFXInputConverter() self.ccfx = CCFXEntryPoint(self.path_builder, model.getCcfxPath(), model.getCcfxTokenSize())
class RepDriver: def __init__(self, model): self.proj0 = model.getProj(PathBuilder.Proj0) self.proj1 = model.getProj(PathBuilder.Proj1) self.path_builder = model.getPathBuilder() self.converter = CCFXInputConverter() self.ccfx = CCFXEntryPoint(self.path_builder, model.getCcfxPath(), model.getCcfxTokenSize()) def ccfxConvert(self): print "Converting diffs to ccfx compatible format" if self.proj0: proj0_repo = self.proj0.getRepoRoot() + os.sep print proj0_repo self.path_builder.setExtDiffPath(0, proj0_repo) if self.proj1: proj1_repo = self.proj1.getRepoRoot() + os.sep print proj1_repo self.path_builder.setExtDiffPath(1, proj1_repo) self.converter.convertExtDiffs(self.path_builder) def runCCFX_old(self, lang): print "Running ccFinder for old files, this will take quite some time..." have_old_lang = self.ccfx.processPairs(lang, False) print "Filtering ccFinder old output based on operation..." if have_old_lang: is_new = False output = convert_ccfx_output(self.path_builder, lang, is_new) rep_out_path = self.path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = self.path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) def runCCFX_new(self, lang): print "Running ccFinder for new files, this will take quite some time..." have_new_lang = self.ccfx.processPairs(lang, True) print "Filtering ccFinder new output based on operation..." if have_new_lang: is_new = True # lang = have_new_lang output = convert_ccfx_output(self.path_builder, lang, is_new) rep_out_path = self.path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = self.path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) def runCCFX(self): # self.runCCFX_old() for lang in [LangDecider.CXX, LangDecider.HXX, LangDecider.JAVA]: self.runCCFX_new(lang) self.runCCFX_old(lang) def process(self, rep_model): msg, success = self.processImpl(rep_model) def processImpl(self, model): proj0 = model.getProj(PathBuilder.Proj0) proj1 = model.getProj(PathBuilder.Proj1) path_builder = model.getPathBuilder() converter = CCFXInputConverter() ccfx = CCFXEntryPoint(path_builder, model.getCcfxPath(), model.getCcfxTokenSize()) step = 0 total_steps = 20.0 final_status = False # step == 0: print "Loading version histories for first project" + str(step / total_steps) step += 1 proj0.load() print "Loading version histories for second project" + str(step / total_steps) step += 1 proj1.load() print "Dumping commits for first project" + str(step / total_steps) step += 1 proj0.dumpCommits() print "Dumping commits for second project" + str(step / total_steps) step += 1 proj1.dumpCommits() print "Converting diffs to ccfx compatible format" + str(step / total_steps) step += 1 converter.convert(path_builder) print "Running ccFinder for old C, this will take quite some time..." + str(step / total_steps) step += 1 have_old_c = ccfx.processPairs(LangDecider.CXX, False) print "Running ccFinder for new C, this will take quite some time..." + str(step / total_steps) step += 1 have_new_c = ccfx.processPairs(LangDecider.CXX, True) print "Running ccFinder for old headers, this will take quite some time..." + str(step / total_steps) step += 1 have_old_h = ccfx.processPairs(LangDecider.HXX, False) print "Running ccFinder for new headers, this will take quite some time..." + str(step / total_steps) step += 1 have_new_h = ccfx.processPairs(LangDecider.HXX, True) print "Running ccFinder for old Java, this will take quite some time..." + str(step / total_steps) step += 1 have_old_j = ccfx.processPairs(LangDecider.JAVA, False) print "Running ccFinder for new Java, this will take quite some time..." + str(step / total_steps) step += 1 have_new_j = ccfx.processPairs(LangDecider.JAVA, True) print "Filtering ccFinder old C output based on operation..." + str(step / total_steps) step += 1 if have_old_c: is_new = False lang = LangDecider.CXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder new C output based on operation..." + str(step / total_steps) step += 1 if have_new_c: is_new = True lang = LangDecider.CXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder old header output based on operation..." + str(step / total_steps) step += 1 if have_old_h: is_new = False lang = LangDecider.HXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder new header output based on operation..." + str(step / total_steps) step += 1 if have_new_h: is_new = True lang = LangDecider.HXX output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder old java output based on operation..." + str(step / total_steps) step += 1 if not have_old_j: is_new = False lang = LangDecider.JAVA output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Filtering ccFinder new java output based on operation..." + str(step / total_steps) step += 1 if not have_new_j: is_new = True lang = LangDecider.JAVA output = convert_ccfx_output(path_builder, lang, is_new) rep_out_path = path_builder.getRepertoireOutputPath(lang, is_new) rep_out_file = path_builder.getRepertoireOutputFileName(lang, is_new) output.writeToFile(rep_out_path + rep_out_file) print "Combining ccFinder output into a unified database..." + str(step / total_steps) step += 1 pickle.dump(model, open(path_builder.getModelPathAndName(), "w")) rep_populator = RepDBPopulator(path_builder) db = rep_populator.generateDB(proj0, proj1) db_file = open(path_builder.getDBPathAndName(), "w") pickle.dump(db, db_file) db_file.close() final_status = True if final_status: return "Success!", final_status return "Aborting", final_status