def __init__(self, cur): PathBuilder.__init__(self, cur, int(get_option('path_cache_high_mark', "2000")), int(get_option('path_cache_low_mark', "1000"))) print("resetting yields...") self.cur.execute("""update nodes set yield=0""")
def processDiffs(self, proj, path): #self.got_some = {'java':True, 'cxx':True, 'hxx':True} self.got_some = {'java': False, 'cxx': True, 'hxx': False} self.pb = PathBuilder(self.tmpPath, force_clean=True) if os.path.isdir(path) is True: self.filterDiffProj(proj) elif os.path.isfile(path) is True: self.filterDiffFile(proj) else: return ('Invalid path : ' + path, False) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() progress = (self.operations_so_far.incr() / float(self.num_operations)) * 100 callback = lambda: self.progress('Converting to ccfx input format') converter.convert(proj, self.pb, callback) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) return ("Converting diffs to ccFinder compatible format is done", True)
def processDiffs(self,proj,path): #self.got_some = {'java':True, 'cxx':True, 'hxx':True} self.got_some = {'java':False, 'cxx':True, 'hxx':False} self.pb = PathBuilder(self.tmpPath, force_clean = True) if os.path.isdir(path) is True: self.filterDiffProj(proj) elif os.path.isfile(path) is True: self.filterDiffFile(proj) else: return ('Invalid path : ' + path, False) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() progress = (self.operations_so_far.incr() / float(self.num_operations))*100 callback = lambda: self.progress('Converting to ccfx input format') converter.convert(proj, self.pb, callback) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) return ("Converting diffs to ccFinder compatible format is done",True)
class RepertoireModel: def __init__(self): self.processDirectory = True self.num_operations = 0 self.operations_so_far = IntegerWrapper(0) self.ccfx = CCFXEntryPoint('../ccFinder/ccfx',40,True,True) self.flags = {"No":True, "Yes":False} def setDiffPaths(self, path0 = None, path1 = None): path0 = str(path0) path1 = str(path1) if (not os.path.isdir(path0) or not os.path.isdir(path1)): return False self.paths = {'proj0':path0, 'proj1':path1} return True def setDiffPaths(self, path0 = None, path1 = None, isDirectory = True): path0 = str(path0) path1 = str(path1) self.ccfx.isDirectory = self.processDirectory = isDirectory if (isDirectory is True) and (not os.path.isdir(path0) or not os.path.isdir(path1)): return False elif (isDirectory is False) and (not os.path.isfile(path0) or not os.path.isfile(path1)): return False self.paths = {'proj0':path0, 'proj1':path1} return True def setTmpDirectory(self, path): path = str(path) if not os.path.isdir(path): return False # great, we have a scratch space, lets put our own directory there # so we know we probably aren't going to fight someone else for names uniq = 'repertoire_tmp_' + str(int(os.times()[4] * 100)) tmpPath = path + os.sep + uniq os.mkdir(tmpPath) self.tmpPath = tmpPath return True def setSuffixes(self, jSuff = '', cSuff = '', hSuff = ''): jSuff = str(jSuff) cSuff = str(cSuff) hSuff = str(hSuff) if jSuff.startswith('.'): jSuff = jSuff[1:] if cSuff.startswith('.'): cSuff = cSuff[1:] if hSuff.startswith('.'): hSuff = hSuff[1:] self.suffixes = { 'java':jSuff, 'cxx':cSuff, 'hxx':hSuff, } self.filters = { 'java' : DiffFilter(jSuff), 'cxx' : DiffFilter(cSuff), 'hxx' : DiffFilter(hSuff) } def setCcfxDirectory(self, path): path = str(path) if not os.path.isdir(path): return False ccfx_binary = path + "/ccfx" if os.path.exists(ccfx_binary): self.ccfx.ccfxPath = ccfx_binary return True return False def setCcfxToken(self, token_size): self.ccfx.tokenSize = token_size print "setting ccFinder token size = " + token_size return True def setCcfxFileSeparator(self, flag): self.ccfx.fileSep = self.flags[str(flag)] print "setting ccFinder file separator flag to %d" % (self.ccfx.fileSep) return True def setCcfxGroupSeparator(self, flag): self.ccfx.grpSep = self.flags[str(flag)] print "setting ccFinder group separator flag to %d" % (self.ccfx.grpSep) return True def filterDiffProjs(self, interface): # 3 different file formats, 2 operations each (filter/convert) self.num_operations = len(os.listdir(self.paths['proj0'])) * 3 * 2 if self.paths['proj0'] != self.paths['proj1']: self.num_operations += len(os.listdir(self.paths['proj1'])) * 3 * 2 self.num_operations += 2*6 #2 ccFinder call for all 6 output files self.operations_so_far = IntegerWrapper(0) for proj in ['proj0', 'proj1']: for lang in ['java', 'cxx', 'hxx']: the_filter = self.filters[lang] for i, file_name in enumerate(os.listdir(self.paths[proj])): if interface.cancelled(): return ('User cancelled processing', False) interface.progress('Filtering {0} files'.format(lang), self.operations_so_far.value / float(self.num_operations)) input_path = self.paths[proj] + os.sep + file_name # out_path = (self.pb.getFilterOutputPath(proj, lang) + # ('%04d' % i) + '.' + self.suffixes[lang]) out_path = (self.pb.getFilterOutputPath(proj, lang) + file_name + '.' + self.suffixes[lang]) (ok, gotsome) = the_filter.filterDiff(input_path, out_path) # this is actually tricky, if we got some output for java # in one project but not the other, then we know that # there can't be any clones self.got_some[lang] = self.got_some[lang] and gotsome if not ok: return ('Error processing: ' + file_name, False) self.operations_so_far.incr() if self.paths['proj0'] == self.paths['proj1']: print "filterDiffProjs: two paths same, breaking!!" break def filterDiffFiles(self, interface): # 3 different file formats, 2 operations each (filter/convert) self.num_operations = 3 * 2 if self.paths['proj0'] != self.paths['proj1']: self.num_operations += 3 * 2 self.operations_so_far = IntegerWrapper(0) input_file1 = self.paths['proj0'] input_file2 = self.paths['proj1'] lang1 = os.path.splitext(input_file1)[1] #extension lang2 = os.path.splitext(input_file2)[1] #extension if lang1 != lang2 : print "!!the two files have different extension" print "lang1 = " + lang1 print "lang2 = " + lang2 return False for proj in ['proj0', 'proj1']: for lang in ['java', 'cxx', 'hxx']: the_filter = self.filters[lang] if interface.cancelled(): return ('User cancelled processing', False) interface.progress('Filtering {0} files'.format(lang), self.operations_so_far.value / float(self.num_operations)) input_path = self.paths[proj] out_path = (self.pb.getFilterOutputPath(proj, lang) + os.path.basename(input_path) + '.' + self.suffixes[lang]) (ok, gotsome) = the_filter.filterDiff(input_path, out_path) # this is actually tricky, if we got some output for java # in one project but not the other, then we know that # there can't be any clones self.got_some[lang] = self.got_some[lang] and gotsome if not ok: return ('Error processing: ' + file_name, False) self.operations_so_far.incr() if self.paths['proj0'] == self.paths['proj1']: print "filterDiffFiles: two paths same, breaking!!" break def filterDiffs(self, interface): self.got_some = {'java':True, 'cxx':True, 'hxx':True} # self.haveJava = haveC = haveH = False self.pb = PathBuilder(self.tmpPath, force_clean = True) # First, filter the input diffs by file type, so that all c diffs # are in one set of files, and similarly for java/headers if self.processDirectory is True: self.filterDiffProjs(interface) else: self.filterDiffFiles(interface) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() callback = lambda: interface.progress( 'Converting to ccfx input format', self.operations_so_far.incr() / float(self.num_operations)) converter.convert(self.pb, callback) #new and old for 3 langs self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) clone_path = self.pb.getCCFXOutputPath() # Third, call ccfx for each directory worked = True for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) continue old_path0 = self.pb.getCCFXInputPath(PathBuilder.PROJ0, lang, False) old_path1 = self.pb.getCCFXInputPath(PathBuilder.PROJ1, lang, False) new_path0 = self.pb.getCCFXInputPath(PathBuilder.PROJ0, lang, True) new_path1 = self.pb.getCCFXInputPath(PathBuilder.PROJ1, lang, True) tmp_old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = True) tmp_new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = True) old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = False) new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = False) if self.paths['proj0'] == self.paths['proj1']: old_path1 = old_path0 new_path1 = new_path0 worked = worked and self.ccfx.processPair( old_path0, old_path1, tmp_old_out, old_out, lang) interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) worked = worked and self.ccfx.processPair( new_path0, new_path1, tmp_new_out, new_out, lang) interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) if not worked: return ('ccFinderX execution failed', False) # Fourth, build up our database of clones print "Repertoire filtering...." #new and old for 3 langs self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: interface.progress('Repertoire filtering based on operation', self.operations_so_far.incr() / float(self.num_operations)) continue for is_new in [True, False]: output = convert_ccfx_output(self.pb, lang, is_new) rep_out_path = self.pb.getRepertoireOutputPath(lang, is_new) suffix = '_old.txt' if is_new: suffix = '_new.txt' output.writeToFile(rep_out_path + lang + suffix) interface.progress('Repertoire filtering based on operation', self.operations_so_far.incr() / float(self.num_operations)) print "Processing successful!!" return ('Processing successful', True)
def filterDiffs(self, interface): self.got_some = {'java':True, 'cxx':True, 'hxx':True} # self.haveJava = haveC = haveH = False self.pb = PathBuilder(self.tmpPath, force_clean = True) # First, filter the input diffs by file type, so that all c diffs # are in one set of files, and similarly for java/headers if self.processDirectory is True: self.filterDiffProjs(interface) else: self.filterDiffFiles(interface) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() callback = lambda: interface.progress( 'Converting to ccfx input format', self.operations_so_far.incr() / float(self.num_operations)) converter.convert(self.pb, callback) #new and old for 3 langs self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) clone_path = self.pb.getCCFXOutputPath() # Third, call ccfx for each directory worked = True for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) continue old_path0 = self.pb.getCCFXInputPath(PathBuilder.PROJ0, lang, False) old_path1 = self.pb.getCCFXInputPath(PathBuilder.PROJ1, lang, False) new_path0 = self.pb.getCCFXInputPath(PathBuilder.PROJ0, lang, True) new_path1 = self.pb.getCCFXInputPath(PathBuilder.PROJ1, lang, True) tmp_old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = True) tmp_new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = True) old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = False) new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = False) if self.paths['proj0'] == self.paths['proj1']: old_path1 = old_path0 new_path1 = new_path0 worked = worked and self.ccfx.processPair( old_path0, old_path1, tmp_old_out, old_out, lang) interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) worked = worked and self.ccfx.processPair( new_path0, new_path1, tmp_new_out, new_out, lang) interface.progress('ccFinderX executing', self.operations_so_far.incr() / float(self.num_operations)) if not worked: return ('ccFinderX execution failed', False) # Fourth, build up our database of clones print "Repertoire filtering...." #new and old for 3 langs self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: interface.progress('Repertoire filtering based on operation', self.operations_so_far.incr() / float(self.num_operations)) continue for is_new in [True, False]: output = convert_ccfx_output(self.pb, lang, is_new) rep_out_path = self.pb.getRepertoireOutputPath(lang, is_new) suffix = '_old.txt' if is_new: suffix = '_new.txt' output.writeToFile(rep_out_path + lang + suffix) interface.progress('Repertoire filtering based on operation', self.operations_so_far.incr() / float(self.num_operations)) print "Processing successful!!" return ('Processing successful', True)
class RepertoireModel: def __init__(self): self.processDirectory = True self.num_operations = 0 self.operations_so_far = IntegerWrapper(0) #self.ccfx = CCFXEntryPoint('/home/tang/research/linux/Repertoire/ccFinderx/ccfx', 40, False, True) #self.ccfx = CCFXEntryPoint('/home/tang/nas11/research/ray/project/Repertoire/ccFinderx/ccfx', 40, False, True) self.ccfx = CCFXEntryPoint('/if7/ct4ew/research/ray/project/Repertoire_new/ccFinderx/ccfx', 40, False, True) self.flags = {"No":True, "Yes":False} self.path = {} self.isProcessDiff = False self.tmpPath = None self.outPath = None def setDiffPath(self, path = None): self.isProcessDiff = True path = str(path) if os.path.isdir(path): self.ccfx.isDirectory = True else: self.ccfx.isDirectory = False projNo = len(self.path) proj = 'proj' + str(projNo) self.path[proj] = path # print self.path return True def setOutDirectory(self, path): #Just setting the outer directory path = str(path) if not (path.startswith("/home") or path.startswith("~/")): path = os.getcwd() + os.sep + path if not os.path.isdir(path): os.mkdir(path) self.outPath = path print "output files will be stored at " + self.outPath return True def setTmpDirectory(self, path): path = str(path) if not os.path.isdir(path): os.mkdir(path) self.tmpPath = path print "output files will be stored at " + self.tmpPath return True def setSuffixes(self, jSuff = '', cSuff = '', hSuff = ''): jSuff = str(jSuff) cSuff = str(cSuff) hSuff = str(hSuff) if jSuff.startswith('.'): jSuff = jSuff[1:] if cSuff.startswith('.'): cSuff = cSuff[1:] if hSuff.startswith('.'): hSuff = hSuff[1:] self.suffixes = { 'java':jSuff, 'cxx':cSuff, 'hxx':hSuff, } self.filters = { 'java' : DiffFilter(jSuff), 'cxx' : DiffFilter(cSuff), 'hxx' : DiffFilter(hSuff) } def setCcfxDirectory(self, path): path = str(path) if not os.path.isdir(path): return False ccfx_binary = path + "/ccfx" if os.path.exists(ccfx_binary): self.ccfx.ccfxPath = ccfx_binary return True return False def setCcfxToken(self, token_size): self.ccfx.tokenSize = token_size print "setting ccFinder token size = " + token_size return True def setCcfxFileSeparator(self, flag): self.ccfx.fileSep = self.flags[str(flag)] print "setting ccFinder file separator flag to %d" % (self.ccfx.fileSep) return True def setCcfxGroupSeparator(self, flag): self.ccfx.grpSep = self.flags[str(flag)] print "setting ccFinder group separator flag to %d" % (self.ccfx.grpSep) return True def filterDiffProj(self,proj): path = self.path[proj] self.num_operations = 3 * 2 self.num_operations += len(os.listdir(path)) * 3 self.operations_so_far = IntegerWrapper(0) #for lang in ['java', 'cxx', 'hxx']: for lang in ['cxx']: #for lang in ['c']: the_filter = self.filters[lang] for i, file_name in enumerate(os.listdir(path)): self.progress('Filtering {0} files'.format(lang)) input_path = path + os.sep + file_name print file_name print input_path out_path = (self.pb.getFilterOutputPath(proj, lang) + file_name + '.' + self.suffixes[lang]) print out_path (ok, gotsome) = the_filter.filterDiff(input_path, out_path) self.got_some[lang] = self.got_some[lang] and gotsome if not ok: return ('Error processing: ' + file_name, False) self.operations_so_far.incr() def filterDiffFile(self,diff_file): # 3 different file formats, 2 operations each (filter/convert) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: the_filter = self.filters[lang] self.progress('Filtering {0} files'.format(lang)) input_path = diff_file out_path = (self.pb.getFilterOutputPath(proj, lang) + os.path.basename(input_path) + '.' + self.suffixes[lang]) (ok, gotsome) = the_filter.filterDiff(input_path, out_path) self.got_some[lang] = self.got_some[lang] and gotsome if not ok: return ('Error processing: ' + file_name, False) self.operations_so_far.incr() def progress(self,msg): progressSoFar = (self.operations_so_far.value / float(self.num_operations))*100 print "%s..: %f" % (msg,progressSoFar) def processDiffs(self,proj,path): #self.got_some = {'java':True, 'cxx':True, 'hxx':True} self.got_some = {'java':False, 'cxx':True, 'hxx':False} self.pb = PathBuilder(self.tmpPath, force_clean = True) if os.path.isdir(path) is True: self.filterDiffProj(proj) elif os.path.isfile(path) is True: self.filterDiffFile(proj) else: return ('Invalid path : ' + path, False) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() progress = (self.operations_so_far.incr() / float(self.num_operations))*100 callback = lambda: self.progress('Converting to ccfx input format') converter.convert(proj, self.pb, callback) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) return ("Converting diffs to ccFinder compatible format is done",True) def processDiff(self): for proj,path in self.path.items(): self.processDiffs(proj, path) def runCCFinderSelf(self,proj,path): print 'Chong Tang: In runCCFinderSelf() function ----' clone_path = self.pb.getCCFXOutputPath() # Third, call ccfx for each directory worked = True for lang in ['java', 'cxx', 'hxx']: #print 'Chong Tang: In runCCFinerSelf for loop...' #print 'Chong: got_some of java: ' + str(self.got_some['java']) if not self.got_some[lang]: self.progress('ccFinderX executing') continue #print 'Chong Tang: In runCCFinerSelf for loop, after if statement' old_path = self.pb.getCCFXInputPath(proj, lang, False) new_path = self.pb.getCCFXInputPath(proj, lang, True) tmp_old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = True) tmp_new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = True) old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = False, is_tmp = False) new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new = True, is_tmp = False) print 'Chong Tang: before first processPairSelf calling' worked = worked and self.ccfx.processPairSelf( old_path, tmp_old_out, old_out, lang) print 'Chong Tang: after first processPairSelf calling' self.progress('ccFinderX executing') print 'Chong Tang: before second processPairSelf calling' worked = worked and self.ccfx.processPairSelf( new_path, tmp_new_out, new_out, lang) print 'Chong Tang: after second processPairSelf calling' self.progress('ccFinderX executing') if not worked: return ('ccFinderX execution failed', False) self.runRep(proj) # #new and old for 3 langs def runRep(self,proj): print "Repertoire filtering...." self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: self.progress('Repertoire filtering based on operation') continue for is_new in [True, False]: output = convert_ccfx_output(self.pb,proj,lang, is_new) rep_out_path = self.pb.getRepertoireOutputPath(lang, is_new) suffix = '_old.txt' if is_new: suffix = '_new.txt' output.writeToFile(rep_out_path + lang + suffix) self.progress('Repertoire filtering based on operation') print "Processing successful!!" return ('Processing successful', True)
class RepertoireModel: def __init__(self): self.processDirectory = True self.num_operations = 0 self.operations_so_far = IntegerWrapper(0) #self.ccfx = CCFXEntryPoint('/home/tang/research/linux/Repertoire/ccFinderx/ccfx', 40, False, True) #self.ccfx = CCFXEntryPoint('/home/tang/nas11/research/ray/project/Repertoire/ccFinderx/ccfx', 40, False, True) self.ccfx = CCFXEntryPoint( '/if7/ct4ew/research/ray/project/Repertoire_new/ccFinderx/ccfx', 40, False, True) self.flags = {"No": True, "Yes": False} self.path = {} self.isProcessDiff = False self.tmpPath = None self.outPath = None def setDiffPath(self, path=None): self.isProcessDiff = True path = str(path) if os.path.isdir(path): self.ccfx.isDirectory = True else: self.ccfx.isDirectory = False projNo = len(self.path) proj = 'proj' + str(projNo) self.path[proj] = path # print self.path return True def setOutDirectory(self, path): #Just setting the outer directory path = str(path) if not (path.startswith("/home") or path.startswith("~/")): path = os.getcwd() + os.sep + path if not os.path.isdir(path): os.mkdir(path) self.outPath = path print "output files will be stored at " + self.outPath return True def setTmpDirectory(self, path): path = str(path) if not os.path.isdir(path): os.mkdir(path) self.tmpPath = path print "output files will be stored at " + self.tmpPath return True def setSuffixes(self, jSuff='', cSuff='', hSuff=''): jSuff = str(jSuff) cSuff = str(cSuff) hSuff = str(hSuff) if jSuff.startswith('.'): jSuff = jSuff[1:] if cSuff.startswith('.'): cSuff = cSuff[1:] if hSuff.startswith('.'): hSuff = hSuff[1:] self.suffixes = { 'java': jSuff, 'cxx': cSuff, 'hxx': hSuff, } self.filters = { 'java': DiffFilter(jSuff), 'cxx': DiffFilter(cSuff), 'hxx': DiffFilter(hSuff) } def setCcfxDirectory(self, path): path = str(path) if not os.path.isdir(path): return False ccfx_binary = path + "/ccfx" if os.path.exists(ccfx_binary): self.ccfx.ccfxPath = ccfx_binary return True return False def setCcfxToken(self, token_size): self.ccfx.tokenSize = token_size print "setting ccFinder token size = " + token_size return True def setCcfxFileSeparator(self, flag): self.ccfx.fileSep = self.flags[str(flag)] print "setting ccFinder file separator flag to %d" % ( self.ccfx.fileSep) return True def setCcfxGroupSeparator(self, flag): self.ccfx.grpSep = self.flags[str(flag)] print "setting ccFinder group separator flag to %d" % ( self.ccfx.grpSep) return True def filterDiffProj(self, proj): path = self.path[proj] self.num_operations = 3 * 2 self.num_operations += len(os.listdir(path)) * 3 self.operations_so_far = IntegerWrapper(0) #for lang in ['java', 'cxx', 'hxx']: for lang in ['cxx']: #for lang in ['c']: the_filter = self.filters[lang] for i, file_name in enumerate(os.listdir(path)): self.progress('Filtering {0} files'.format(lang)) input_path = path + os.sep + file_name print file_name print input_path out_path = (self.pb.getFilterOutputPath(proj, lang) + file_name + '.' + self.suffixes[lang]) print out_path (ok, gotsome) = the_filter.filterDiff(input_path, out_path) self.got_some[lang] = self.got_some[lang] and gotsome if not ok: return ('Error processing: ' + file_name, False) self.operations_so_far.incr() def filterDiffFile(self, diff_file): # 3 different file formats, 2 operations each (filter/convert) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: the_filter = self.filters[lang] self.progress('Filtering {0} files'.format(lang)) input_path = diff_file out_path = (self.pb.getFilterOutputPath(proj, lang) + os.path.basename(input_path) + '.' + self.suffixes[lang]) (ok, gotsome) = the_filter.filterDiff(input_path, out_path) self.got_some[lang] = self.got_some[lang] and gotsome if not ok: return ('Error processing: ' + file_name, False) self.operations_so_far.incr() def progress(self, msg): progressSoFar = (self.operations_so_far.value / float(self.num_operations)) * 100 print "%s..: %f" % (msg, progressSoFar) def processDiffs(self, proj, path): #self.got_some = {'java':True, 'cxx':True, 'hxx':True} self.got_some = {'java': False, 'cxx': True, 'hxx': False} self.pb = PathBuilder(self.tmpPath, force_clean=True) if os.path.isdir(path) is True: self.filterDiffProj(proj) elif os.path.isfile(path) is True: self.filterDiffFile(proj) else: return ('Invalid path : ' + path, False) # Second, change each diff into ccFinder input format converter = CCFXInputConverter() progress = (self.operations_so_far.incr() / float(self.num_operations)) * 100 callback = lambda: self.progress('Converting to ccfx input format') converter.convert(proj, self.pb, callback) self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) return ("Converting diffs to ccFinder compatible format is done", True) def processDiff(self): for proj, path in self.path.items(): self.processDiffs(proj, path) def runCCFinderSelf(self, proj, path): print 'Chong Tang: In runCCFinderSelf() function ----' clone_path = self.pb.getCCFXOutputPath() # Third, call ccfx for each directory worked = True for lang in ['java', 'cxx', 'hxx']: #print 'Chong Tang: In runCCFinerSelf for loop...' #print 'Chong: got_some of java: ' + str(self.got_some['java']) if not self.got_some[lang]: self.progress('ccFinderX executing') continue #print 'Chong Tang: In runCCFinerSelf for loop, after if statement' old_path = self.pb.getCCFXInputPath(proj, lang, False) new_path = self.pb.getCCFXInputPath(proj, lang, True) tmp_old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new=False, is_tmp=True) tmp_new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new=True, is_tmp=True) old_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new=False, is_tmp=False) new_out = clone_path + self.pb.getCCFXOutputFileName( lang, is_new=True, is_tmp=False) print 'Chong Tang: before first processPairSelf calling' worked = worked and self.ccfx.processPairSelf( old_path, tmp_old_out, old_out, lang) print 'Chong Tang: after first processPairSelf calling' self.progress('ccFinderX executing') print 'Chong Tang: before second processPairSelf calling' worked = worked and self.ccfx.processPairSelf( new_path, tmp_new_out, new_out, lang) print 'Chong Tang: after second processPairSelf calling' self.progress('ccFinderX executing') if not worked: return ('ccFinderX execution failed', False) self.runRep(proj) # #new and old for 3 langs def runRep(self, proj): print "Repertoire filtering...." self.num_operations = 3 * 2 self.operations_so_far = IntegerWrapper(0) for lang in ['java', 'cxx', 'hxx']: if not self.got_some[lang]: self.progress('Repertoire filtering based on operation') continue for is_new in [True, False]: output = convert_ccfx_output(self.pb, proj, lang, is_new) rep_out_path = self.pb.getRepertoireOutputPath(lang, is_new) suffix = '_old.txt' if is_new: suffix = '_new.txt' output.writeToFile(rep_out_path + lang + suffix) self.progress('Repertoire filtering based on operation') print "Processing successful!!" return ('Processing successful', True)
def build_new_paths(map_dict, num_layers): """Given current source paths, builds new source paths to reflect modified directory structure :type map_dict: Dict :type num_layers: str :rtype: None """ # Instantiate new PathBuilder object that will consume getter.source_paths list builder = PathBuilder() # Grab stored dict of filepaths builder.get_mapped_sources() # For each map for mxd_path, lyr_sources_list in map_dict.items(): # Grab the map object to be modified mxd = lyr_sources_list[0][0] # Split the map filepath into list of directories and filename split_target = builder.split_path(mxd_path) # for each (map object, layer object, old layer source filepath) tuple for lyr in lyr_sources_list: # Grab the filename source_fname = builder.split_path(lyr[2])[-1] # Drop the .shp, .tif extenson source_fname_wo_ext = str(source_fname.split('.')[0]) # Get old workspace path try: old_workspace = lyr[1].workspacePath # Create dict for building new path except: print ("Layer doesn't have workspace path", lyr, type(lyr)) print ("Service layer?", lyr[1].isServiceLayer) print ("Group layer?", lyr[1].isGroupLayer) print("Supports DataSource?", lyr[1].supports("DATASOURCE")) print lyr[1].longName break path_dict = builder.get_path_variables(split_target[split_target.index('Y:'): ]) # Move shared data sources from Z: into special Data Library folder if old_workspace.startswith('Z') or old_workspace.startswith('Y:\_Data_Library'): source_path = builder.split_path(old_workspace)[2:] source_path = '\\'.join(source_path) new_workspace = 'Y:\_DataLibrary' + '\\' + source_path else: # All other sources go into their respective project folders new_workspace = builder.match_new_src(path_dict['Project'], source_fname) if not new_workspace: print "Couldn't make source match: " + source_fname else: builder.build(lyr[1], old_workspace, new_workspace) print 'Saving the mxd...' # If you get an error, make sure all other instances of mxd are closed print '-------------------------------------------------------' try: mxd.save() del mxd except (IOError, AttributeError), e: log_error(mxd.filePath) print "Unable to save mxd " + mxd.title
def __init__(self, cur): PathBuilder.__init__(self, cur) self.path = {} # depth -> [ url_id ]