def loads(self): ''' loads a raw file of bug report :return: ''' fileConnt = self.getFileCounts(self.SourceBugPath) bugitems = [] # show progress progress = Progress(u'[%s] Loading bug reports' % self.__name__, 2, 10, True) progress.set_upperbound(fileConnt) progress.start() for root, dirs, files in os.walk(self.SourceBugPath): for f in files: if f[:f.find(u'-')].strip().lower() != self.ProjectName.lower( ): continue #shutil.copy(os.path.join(root, f), os.path.join(_dest, f)) bugitem = self.get_bugitem(os.path.join(root, f)) if bugitem is not None: bugitems.append(bugitem) progress.check() progress.done() return bugitems
def make_childmap(self): visited = set([]) q = Queue() q.put((self.git.head.commit, None)) # (commit, child_hash) progress = Progress(u'[%s] making git childmap' % self.__name__, 500, 10000, False) progress.set_point(0) progress.start() while q.empty() is False: progress.check() commit, child_hash = q.get() commit_hash = str(commit)[:7] # create child map if commit_hash not in self.childmap: self.childmap[commit_hash] = set([]) if child_hash is not None: self.childmap[commit_hash].add(child_hash) if commit_hash in visited: continue visited.add(commit_hash) # pass itself to parent for parent in commit.parents: q.put((parent, commit_hash)) # add ancestors if this commit has no parents if len(commit.parents) == 0: self.ancestors.add(commit_hash) progress.done() pass
def load(self, _force=False): ''' Load commit info from GitLogPath :return: {bugID:[{'hash':u'', 'author':u'', 'commit_date':u'', 'message':u'', 'fixedFiles':{}}, {}, ...], ...} ''' if os.path.exists(self.GitLogPath) is False or _force is True: self.make() logfile = codecs.open(self.GitLogPath, 'r', 'utf-8') progress = Progress(u'[%s] loading git log data' % self.__name__, 1000, 20000, False) progress.set_point(0) progress.start() for logitem in self.file_loader(logfile, _with_filter=False): # filter unuseful logs if len(logitem['fixedFiles']) == 0: continue # We only use bug report id in log message # mapping bug report ID logitem['linked_bug'] = re.findall( r'%s-[0-9]+' % self.ProjectName.upper(), logitem['message']) logitem['linked_bug'] = set(logitem['linked_bug']) for linked_id in logitem['linked_bug']: if linked_id not in self.logs: self.logs[linked_id] = [logitem] else: self.logs[linked_id].append(logitem) progress.check() progress.done() logfile.close() return self.logs
def fill_SubjectSheet(self, _sheet, _group, _srcCounts, _bugCounts, _dupCounts): projects = _bugCounts.keys() projects.sort() size = sum([len(_bugCounts[project]) for project in projects]) progress = Progress(u'[%s] fill subject' % self.__name__, 2, 10, True) progress.set_point(0).set_upperbound(size) progress.start() styles = [self.base_format, self.base_format, self.base_format, self.number_format, self.number_format] for project in projects: for version in _bugCounts[project].keys(): if version == 'all': continue values = [_group, project, version.upper(), _bugCounts[project][version], _srcCounts[project][version]] self.input_row(_sheet, self.subj_data_row, 6, values, styles) self.subj_data_row += 1 progress.check() progress.done() #summary styles = [self.subtitle_format, self.subtitle_format, self.number_format, self.number_format, self.number_format] for project in projects: values = [_group, project.upper(), _bugCounts[project]['all'], _dupCounts[project], _srcCounts[project]['all']] self.input_row(_sheet, self.subj_summary_row, 0, values, styles) self.subj_summary_row += 1 pass
def source_counting(self, _group, _project): statistics = {} progress = Progress('source counting', 2, 10, True) progress.set_upperbound(len(self.S.versions[_project].keys())) progress.start() for version in self.S.versions[_project].keys(): vname = VersionUtil.get_versionName(version, _project) repo = os.path.join(self.S.getPath_source(_group, _project, vname), ) result = self.getCodeCount(repo) if result is None: continue statistics[vname] = result progress.check() progress.done() maxValue = 0 for vname in statistics: if maxValue < statistics[vname]: maxValue = statistics[vname] statistics['max'] = maxValue pretty = PrettyStringBuilder(_indent_depth=2) text = pretty.get_dicttext({_project: statistics}) f = open( os.path.join(self.S.getPath_base(_group, _project), u'sources.txt'), 'w') f.write(text) f.close()
def unhash_folder(_src, _dest): ''' hashed folder ==> unshed folder example) path/aa/00/filename ==> path/filename :param _src: :param _dest: :return: ''' if os.path.exists(_dest) is False: os.makedirs(_dest) progress = Progress(u'Bug reports is merging', 20, 1000, False) progress.start() for root, dirs, files in os.walk(_src): for f in files: shutil.copy(os.path.join(root, f), os.path.join(_dest, f)) progress.check() progress.done()
def make_tagmap(self, ): q = Queue() visited = set([]) # root node find (queue init) for item in list(self.ancestors): q.put((item, None)) # (commit_hash, tagname) # For each item in queue progress = Progress(u'[%s] making git tagmaps' % self.__name__, 500, 10000, False) progress.set_point(0) progress.start() while q.empty() is False: commit_hash, parent_tag = q.get() # If this commit in tags, map with commit_hash and tag if commit_hash in self.tags: commit_tag = self.tags[commit_hash] self.tagmap[commit_hash] = commit_tag # if this commit not in tags, map with child commit_hash and tag else: if commit_hash not in self.tagmap: self.tagmap[commit_hash] = parent_tag else: # compare time previous_tag and parent_tag previous_tag = self.tagmap[commit_hash] pre_time = self.tagtimes[previous_tag] par_time = self.tagtimes[parent_tag] if par_time > pre_time: self.tagmap[commit_hash] = parent_tag commit_tag = parent_tag if commit_hash not in visited: visited.add(commit_hash) for child_hash in self.childmap[commit_hash]: q.put((child_hash, commit_tag)) progress.check() progress.done() pass
def load_raw(self, _force=False): ''' Load commit info from GitLogPath :return: {bugID:[{'hash':u'', 'author':u'', 'commit_date':u'', 'message':u'', 'fixedFiles':{}}, {}, ...], ...} ''' if os.path.exists(self.GitLogPath) is False or _force is True: self.make() self.logs = [] logfile = codecs.open(self.GitLogPath, 'r', 'utf-8') progress = Progress(u'[%s] loading git log data' % self.__name__, 1000, 20000, False) progress.set_point(0) progress.start() for logitem in self.file_loader(logfile): # filter unuseful logs #if len(logitem['fixedFiles'])==0: continue if logitem['hash'] == '': continue self.logs.insert(0, logitem) progress.check() progress.done() logfile.close() return self.logs