def work(_sGroup=None, _sProject=None, _removeTest=True):
S = Subjects()
for group in (S.groups if _sGroup is None else [_sGroup]):
for project in (S.projects[group]
if _sProject is None else [_sProject]):
obj = BugRepositoryMaker(project,
S.getPath_bugrepo(group, project),
S.getPath_gitrepo(group, project),
S.getPath_bugrepo(group, project))
obj.run(S.versions[project].keys(), _removeTest)
def make(_sGroup=None, _sProject=None):
S = Subjects()
for group in S.groups:
if _sGroup is None or group == _sGroup:
for project in S.projects[group]:
if _sProject is None or project == _sProject:
git = GitInflator(project, S.getPath_base(group, project))
git.inflate(
S.versions[project]
) # The items in versions is git tag name map with each version.
def make(_sGroup=None, _sProject=None):
S = Subjects()
for group in S.groups:
if _sGroup is not None and group != _sGroup: continue
for project in S.projects[group]:
if _sProject is not None and project != _sProject: continue
git = GitInflator(project, urls[project],
S.getPath_base(group, project))
git.inflate(
S.versions[project]
) # The items in versions is git tag name map with each version.
def clear(_sGroup=None, _sProject=None):
S = Subjects()
for group in S.groups:
if _sGroup is not None and group != _sGroup: continue
for project in S.projects[group]:
if _sProject is not None and project != _sProject: continue
target = S.getPath_source(group, project)
try:
shutil.rmtree(target, onerror=del_rw)
print(u'removed: %s' % target)
except Exception as e:
print(u'failed to remove : %s' % target)
def clean(): S = Subjects() for group in S.groups: for project in S.projects[group]: print(u'cleanning %s / %s ' % (group, project)) dirpath = os.path.join(S.getPath_bugrepo(group, project), u'repository_merge') fullrepo = os.path.join(S.getPath_bugrepo(group, project), u'repository_merge.xml') try: shutil.rmtree(dirpath) except Exception as e: print(u'Failed to remove repository folder') try: os.remove(fullrepo) except Exception as e: print(u'Failed to remove full repository file') pass
def run(self, _type, _isUnion=False, _featureFile=None):
'''
create result file
'''
self.TYPE = _type
print(_type)
# XLS preparing
self.summarySheet = self.create_SummarySheet(0)
self.subjectSheet = self.create_SubjectSheet(0)
self.dataSheet = self.create_DataSheet(0)
self.bugSheet = self.create_bugDataSheet(0)
self.S = Subjects()
for group in self.S.groups: #['Commons', 'JBoss', 'Wildfly', 'Spring']
for project in self.S.projects[group]:
#if project not in ['HBASE','HIVE','ROO','SEC', 'SPR']:continue #
print(u'working %s / %s ...' % (group, project), end=u'')
# fill Dup and subjects
self.fill_SubjectSheet(self.subjectSheet, group, project,
self.S.sources[project],
self.S.bugs[project],
len(self.S.duplicates[project]))
for tech in self.S.techniques: #['BLIA']:# ['BugLocator', "BRTracer", 'BLUiR', 'BLIA']:#, 'Locus']:#
print(tech + u' ', end=u'')
self.append_project(group, project, tech, _isUnion)
print(u' Done')
self.finalize()
pass
def make(self, _group, _project):
'''
make repository files
:param _group:
:param _project:
:return:
'''
self.S = Subjects()
print(u'working with %s / %s' % (_group, _project))
filename = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository.xml')
bugs = self.load_bugs(filename)
dups = self.S.duplicates[_project]
merges = self.merge(_group, _project, bugs, dups)
print(u'created %d merged reports.' % len(merges))
versionItems = self.getItemsByVersion(merges,
self.S.versions[_project].keys())
print(u'created %d version repositories from merged reports.' %
len(versionItems))
print(u'storing....', end=u'')
repositoryPath = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository_merge')
if os.path.exists(repositoryPath) is True:
shutil.rmtree(repositoryPath)
if os.path.exists(repositoryPath) is False:
os.makedirs(repositoryPath)
self.outputXML(
_project, merges,
os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository_merge.xml'))
for ver in versionItems.keys():
filename = os.path.join(
repositoryPath,
VersionUtil.get_versionName(ver, _project) + u'.xml')
self.outputXML(_project, versionItems[ver], filename)
print(u'Done')
answers = self.make_answers(_project, merges, versionItems)
self.save_answers({_project: answers},
os.path.join(self.S.getPath_base(_group, _project),
u'answers_merge.txt'))
pass
def __init__(self):
self.S = Subjects()
if not os.path.exists(os.path.join(self.ProgramPATH, 'logs')):
os.makedirs(os.path.join(self.ProgramPATH, 'logs'))
t = datetime.now()
timestr = t.strftime(u'%y%m%d_%H%M')
self.logFileName = u'logs_%s' % timestr + u'_%s.txt'
def run(self, _type, _isUnion=False):
'''
create result file
'''
self.TYPE = _type
print(_type)
self.S = Subjects()
self.OverallResult = {}
for tech in self.S.techniques:
self.OverallResult[tech] = {'master': {'AP': [], 'TP': []}, 'duplicate': {'AP': [], 'TP': []}}
# XLS preparing
shtOverall = self.create_OverallSheet(0)
shtMasterSummary, rowMasterSummary = self.create_SummarySheet(u'Summary (Master)', 0)
shtMasterBugData, rowMasterBugData = self.create_bugDataSheet(u'bugData (Master)', 0)
shtMasterData, rowMasterData = self.create_DataSheet(u'rawData (Master)', 0)
shtDupSummary, rowDupSummary = self.create_SummarySheet(u'Summary (Dup)', 0)
shtDupBugData, rowDupBugData = self.create_bugDataSheet(u'bugData (Dup)', 0)
shtDupData, rowDupData = self.create_DataSheet(u'rawData (Dup)', 0)
for group in self.S.groups:
for project in self.S.projects[group]:
print(u'working %s / %s ...' % (group, project), end=u'')
for tech in self.S.techniques:
print(tech + u' ', end=u'')
evMaster, evDuplicate = self.load_results(group, project, tech, _isUnion)
masterBugs, dupBugs = self.make_IDsets(self.S.duplicates[project])
rowMasterSummary = self.fill_SummarySheet(shtMasterSummary, rowMasterSummary, tech, group, project, 'master', evMaster.projectSummary, self.S.sources[project]['max'], len(masterBugs), len(evMaster.bugSummaries))
rowMasterBugData = self.fill_bugDataSheet(shtMasterBugData, rowMasterBugData, tech, group, project, 'master', evMaster.bugSummaries, self.S.answers[project]['all'])
rowMasterData = self.fill_DataSheet(shtMasterData, rowMasterData, tech, group, project, 'master', evMaster.bugSummaries, evMaster.rawData, self.S.sources[project], self.S.answers[project]['all'])
rowDupSummary= self.fill_SummarySheet(shtDupSummary, rowDupSummary, tech, group, project, 'duplicate', evDuplicate.projectSummary, self.S.sources[project]['max'], len(dupBugs), len(evDuplicate.bugSummaries))
rowDupBugData = self.fill_bugDataSheet(shtDupBugData, rowDupBugData, tech, group, project, 'duplicate', evDuplicate.bugSummaries, self.S.answers[project]['all'])
rowDupData = self.fill_DataSheet(shtDupData, rowDupData, tech, group, project, 'duplicate', evDuplicate.bugSummaries, evDuplicate.rawData, self.S.sources[project], self.S.answers[project]['all'])
for bugID in evMaster.bugSummaries:
self.OverallResult[tech]['master']['AP'].append(evMaster.bugSummaries[bugID].AP)
self.OverallResult[tech]['master']['TP'].append(evMaster.bugSummaries[bugID].TP)
for bugID in evDuplicate.bugSummaries:
self.OverallResult[tech]['duplicate']['AP'].append(evDuplicate.bugSummaries[bugID].AP)
self.OverallResult[tech]['duplicate']['TP'].append(evDuplicate.bugSummaries[bugID].TP)
print(u' Done')
self.fill_OverallSheet(shtOverall)
self.finalize()
pass
def clean(_sGroup=None, _sProject=None):
S = Subjects()
for group in (S.groups if _sGroup is None else [_sGroup]):
for project in (S.projects[group]
if _sProject is None else [_sProject]):
print(u'cleanning %s / %s ' % (group, project))
tempfile1 = os.path.join(S.getPath_bugrepo(group, project),
u'.git.log')
tempfile2 = os.path.join(S.getPath_bugrepo(group, project),
u'.git_version.txt')
dirpath = os.path.join(S.getPath_bugrepo(group, project),
u'repository')
fullrepo = os.path.join(S.getPath_bugrepo(group, project),
u'repository_full.xml')
filteredrepo = os.path.join(S.getPath_bugrepo(group, project),
u'repository.xml')
try:
os.remove(tempfile1)
except Exception as e:
print(u'Failed to remove git log')
try:
os.remove(tempfile2)
except Exception as e:
print(u'Failed to remove git versions')
try:
shutil.rmtree(dirpath)
except Exception as e:
print(u'Failed to remove repository folder')
try:
os.remove(fullrepo)
except Exception as e:
print(u'Failed to remove full repository file')
try:
os.remove(filteredrepo)
except Exception as e:
print(u'Failed to remove filtered repository file')
pass
class Counting(object):
'''
Make bugs.txt, answers.txt, sources.txt each project
The files will be stored in each project folder.
- bugs.txt : bug report count by each version
- answers.txt : This file has the number of answer files each bug report
- sources.txt : This file has the number of source files each version
'''
def __init__(self):
self.S = Subjects()
pass
def getBugs(self, _repo):
'''
:param _repo:
:return:
'''
bugs = []
try:
e = ElementTree.parse(_repo).getroot()
bugtags = e.findall('bug')
for tag in bugtags:
bugs.append(int(tag.attrib['id']))
except Exception as e:
return None
return bugs
def getAnswers(self, _repo):
'''
get answer files from bug repository files
:param _repo: repository file path
:return:
'''
bugAnswers = {}
try:
e = ElementTree.parse(_repo).getroot()
bugtags = e.findall('bug')
for tag in bugtags:
id = int(tag.attrib['id'])
files = tag.findall('fixedFiles')[0]
bugAnswers[id] = len(files.findall('file'))
except Exception as e:
return None
return bugAnswers
def getCodeCount(self, _path):
if os.path.exists(_path) is False: return 0
count = 0
for root, dirs, files in os.walk(_path):
for fileitem in files:
if fileitem.endswith('.java') is False: continue
count += 1
return count
def bug_counting(self, _group, _project):
statistics = {}
repo = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository.xml')
statistics['all'] = self.getBugs(repo)
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository', u'%s.xml' % vname)
result = self.getBugs(repo)
if result is None: continue
statistics[vname] = result
# Check missed items
idset = set(statistics['all'])
for key in statistics:
if key == 'all': continue
idset -= set(statistics[key])
statistics['miss'] = list(idset)
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(
os.path.join(self.S.getPath_base(_group, _project), u'bugs.txt'),
'w')
f.write(text)
f.close()
pass
def answers_counting(self, _group, _project):
statistics = {}
repo = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository.xml')
statistics['all'] = self.getAnswers(repo)
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository', u'%s.xml' % vname)
result = self.getAnswers(repo)
if result is None: continue
statistics[vname] = result
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(
os.path.join(self.S.getPath_base(_group, _project),
u'answers.txt'), 'w')
f.write(text)
f.close()
pass
def source_counting(self, _group, _project):
statistics = {}
progress = Progress('source counting', 2, 10, True)
progress.set_upperbound(len(self.S.versions[_project].keys()))
progress.start()
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_source(_group, _project,
vname), )
result = self.getCodeCount(repo)
if result is None: continue
statistics[vname] = result
progress.check()
progress.done()
maxValue = 0
for vname in statistics:
if maxValue < statistics[vname]:
maxValue = statistics[vname]
statistics['max'] = maxValue
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(
os.path.join(self.S.getPath_base(_group, _project),
u'sources.txt'), 'w')
f.write(text)
f.close()
def run(self):
for group in self.S.groups:
for project in self.S.projects[group]:
print(u'Counting for %s / %s' % (group, project))
self.bug_counting(group, project)
self.answers_counting(group, project)
def compare_single_results(self, _basepath):
'''
for Table 1 : single results
:param _basepath:
:return:
'''
def get_averages(_itemType):
results = {}
for tData in ['Old', 'New_Single']:
filepath = os.path.join(_basepath,
u'%s_%s.txt' % (tData, _itemType))
titles, data = self.load_results_items(filepath, ['str'] * 3 +
['float'] * 6)
for group in data:
if group not in results: results[group] = {}
for project in data[group]:
CNTs = dict(zip(titles, CNTdata[group][project]))
results[group][project] = self.get_technique_averages(
data[group][project], CNTs)
return results
techinques, CNTdata = self.load_results(
os.path.join(_basepath, u'BugCNT.txt'), ['str'] * 2 + ['int'] * 6)
APresults = get_averages('AP')
TPresults = get_averages('TP')
S = Subjects()
# make MAP values for New Subjects
print(u'\n\n')
print(u'Technique Mann-Whitney U Test p-values')
print(u'\t' + u'\t\t'.join(self.techniques))
print(
u'Subject\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR'
)
print(u"\\hline")
print(u"\\multicolumn{13}{c}{\\bf New subjects} \\\\")
print(u"\\hline")
for group in S.groups:
for project in S.projects[group]:
text = u'%s' % project
APmax = self.get_max_technique(APresults[group][project],
self.techniques)
TPmax = self.get_max_technique(TPresults[group][project],
self.techniques)
for tech in self.techniques:
text += self.make_max_text(APresults[group][project][tech],
APmax == tech, u'blue!25')
text += self.make_max_text(TPresults[group][project][tech],
TPmax == tech, u'green!25')
text += u' \\\\'
print(text)
# make average information
avgAPs = dict(zip(self.techniques, [0] * len(self.techniques)))
avgTPs = dict(zip(self.techniques, [0] * len(self.techniques)))
for tech in self.techniques:
count = 0
for group in S.groups:
count += len(S.projects[group])
avgAPs[tech] += sum(APresults[group][project][tech]
for project in S.projects[group])
avgTPs[tech] += sum(TPresults[group][project][tech]
for project in S.projects[group])
avgAPs[tech] /= count
avgTPs[tech] /= count
text = u'Average'
APmax = self.get_max_technique(avgAPs, self.techniques)
TPmax = self.get_max_technique(avgTPs, self.techniques)
for tech in self.techniques:
text += self.make_max_text(avgAPs[tech], APmax == tech, u'blue!25')
text += self.make_max_text(avgTPs[tech], TPmax == tech,
u'green!25')
text += u' \\\\'
print(u'\\hline')
print(text)
# make MAP values for OLD Subjects
print(u"\\hline")
print(u"\\multicolumn{13}{c}{\\bf Old subjects} \\\\")
print(u"\\hline")
group = u'Previous'
projects = [u'AspectJ', u'ZXing', u'PDE', u'JDT', u'SWT']
for project in projects:
text = u'%s' % project
APmax = self.get_max_technique(APresults[group][project],
self.techniques)
TPmax = self.get_max_technique(TPresults[group][project],
self.techniques)
for tech in self.techniques:
text += self.make_max_text(APresults[group][project][tech],
APmax == tech, u'blue!25')
text += self.make_max_text(TPresults[group][project][tech],
TPmax == tech, u'green!25')
text += u' \\\\'
print(text)
# make average information
avgAPs = {}
avgTPs = {}
for tech in self.techniques:
avgAPs[tech] = sum(APresults[group][project][tech]
for project in projects) / len(projects)
avgTPs[tech] = sum(TPresults[group][project][tech]
for project in projects) / len(projects)
text = u'Average'
APmax = self.get_max_technique(avgAPs, self.techniques)
TPmax = self.get_max_technique(avgTPs, self.techniques)
for tech in self.techniques:
text += self.make_max_text(avgAPs[tech], APmax == tech, u'blue!25')
text += self.make_max_text(avgTPs[tech], TPmax == tech,
u'green!25')
text += u' \\\\'
print(u'\\hline')
print(text)
pass
def compare_multi_results(self, _basepath, _withoutTest=False):
'''
:param _basepath:
:return:
'''
def get_average_mwu(_itemType):
results = {}
multi = os.path.join(
_basepath, u'New_Multiple%s_%s.txt' %
('_noTest' if _withoutTest is True else '', _itemType))
titles, dataM = self.load_results_items(multi, ['str'] * 3 +
['float'] * 6)
# MWUresults = {}
# single = os.path.join(_basepath, u'New_Single_%s.txt' % _itemType)
# titles, dataS = self.load_results_items(single, ['str'] * 3 + ['float'] * 6)
for group in dataM:
if group not in results: results[group] = {}
#if group not in MWUresults: MWUresults[group] = {}
for project in dataM[group]:
CNTs = dict(zip(titles, CNTdata[group][project]))
results[group][project] = self.get_technique_averages(
dataM[group][project], CNTs)
#MWUresults[group][project] = self.MWUtest(dataS[group][project], dataM[group][project], CNTs, CNTs)
return results #, MWUresults
techinques, CNTdata = self.load_results(
os.path.join(_basepath, u'BugCNT.txt'), ['str'] * 2 + ['int'] * 6)
APresults = get_average_mwu('AP')
TPresults = get_average_mwu('TP')
print(u'')
print(u'\t' + u'\t\t'.join(self.techniques))
print(
u'Subject\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR\tMAP\tMRR'
)
S = Subjects()
for group in S.groups:
for project in S.projects[group]:
text = u'%s' % project
APmax = self.get_max_technique(APresults[group][project],
self.techniques)
TPmax = self.get_max_technique(TPresults[group][project],
self.techniques)
for tech in self.techniques:
text += self.make_max_text(APresults[group][project][tech],
APmax == tech, u'blue!25')
text += self.make_max_text(TPresults[group][project][tech],
TPmax == tech, u'green!25')
text += u' \\\\'
print(text)
# make average information
avgAPs = dict(zip(self.techniques, [0] * len(self.techniques)))
avgTPs = dict(zip(self.techniques, [0] * len(self.techniques)))
for tech in self.techniques:
count = 0
for group in S.groups:
count += len(S.projects[group])
avgAPs[tech] += sum(APresults[group][project][tech]
for project in S.projects[group])
avgTPs[tech] += sum(TPresults[group][project][tech]
for project in S.projects[group])
avgAPs[tech] /= count
avgTPs[tech] /= count
text = u'Average'
APmax = self.get_max_technique(avgAPs, self.techniques)
TPmax = self.get_max_technique(avgTPs, self.techniques)
for tech in self.techniques:
text += self.make_max_text(avgAPs[tech], APmax == tech, u'blue!25')
text += self.make_max_text(avgAPs[tech], TPmax == tech,
u'green!25')
text += u' \\\\'
print(u'\\hline')
print(text)
# print(u'\\hline')
# text = u'Average'
# for tech in self.techniques:
# avgAP = avgTP = 0.0
# count = 0
# for group in S.groups:
# count += len(S.projects[group])
# avgAP += sum(APresults[group][project][tech] for project in S.projects[group])
# avgTP += sum(TPresults[group][project][tech] for project in S.projects[group])
# text += u' & {:.4f} & {:.4f}'.format(avgAP / count, avgTP/count)
# text += u" \\\\"
# print(text)
pass
class DupMergeRepositoryMaker:
'''
'''
__name__ = u'DupMergeRepositoryMaker'
def __init__(self):
pass
def make(self, _group, _project):
'''
make repository files
:param _group:
:param _project:
:return:
'''
self.S = Subjects()
print(u'working with %s / %s' % (_group, _project))
filename = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository.xml')
bugs = self.load_bugs(filename)
dups = self.S.duplicates[_project]
merges = self.merge(_group, _project, bugs, dups)
print(u'created %d merged reports.' % len(merges))
versionItems = self.getItemsByVersion(merges,
self.S.versions[_project].keys())
print(u'created %d version repositories from merged reports.' %
len(versionItems))
print(u'storing....', end=u'')
repositoryPath = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository_merge')
if os.path.exists(repositoryPath) is True:
shutil.rmtree(repositoryPath)
if os.path.exists(repositoryPath) is False:
os.makedirs(repositoryPath)
self.outputXML(
_project, merges,
os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository_merge.xml'))
for ver in versionItems.keys():
filename = os.path.join(
repositoryPath,
VersionUtil.get_versionName(ver, _project) + u'.xml')
self.outputXML(_project, versionItems[ver], filename)
print(u'Done')
answers = self.make_answers(_project, merges, versionItems)
self.save_answers({_project: answers},
os.path.join(self.S.getPath_base(_group, _project),
u'answers_merge.txt'))
pass
def make_answers(self, _project, _items, _versionItems):
answers = {'all': {}}
for item in _items:
answers['all'][int(item['id'])] = len(item['fixedFiles'])
for version in _versionItems:
versionName = VersionUtil.get_versionName(version, _project)
answers[versionName] = {}
for item in _versionItems[version]:
answers[versionName][int(item['id'])] = len(item['fixedFiles'])
return answers
def save_answers(self, _answers, _filename):
from utils.PrettyStringBuilder import PrettyStringBuilder
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext(_answers)
f = open(_filename, 'w')
f.write(text)
f.close()
pass
def merge(self, _group, _project, _bugs, _dups):
merges = []
bugID = 0
for src, dup in _dups:
if src not in _bugs or dup not in _bugs:
print('passed %s\t%s\t%d\t%d' % (_group, _project, src, dup))
continue
bugID += 1
bug = {}
bug['id'] = str(bugID)
bug['master'] = _bugs[src]['id']
bug['duplicate'] = _bugs[dup]['id']
bug['summary'] = _bugs[src]['summary'] + u' ' + _bugs[dup][
'summary']
bug['description'] = _bugs[src]['description'] + u' ' + _bugs[dup][
'description']
verSrc = _bugs[src]['version']
verDup = _bugs[dup]['version']
bug['version'] = verSrc if VersionUtil.cmpVersion(
verSrc, verDup) > 0 else verDup
verSrc = _bugs[src]['fixedVersion']
verDup = _bugs[dup]['fixedVersion']
bug['fixedVersion'] = verSrc if VersionUtil.cmpVersion(
verSrc, verDup) > 0 else verDup
bug['resolution'] = _bugs[src]['resolution']
bug['opendate'] = min(_bugs[src]['opendate'],
_bugs[dup]['opendate'])
bug['fixdate'] = max(_bugs[src]['fixdate'], _bugs[dup]['fixdate'])
bug['type'] = _bugs[src]['type']
bug['fixedFiles'] = _bugs[src]['fixedFiles']
bug['links'] = _bugs[src]['links'] + _bugs[dup]['links']
merges.append(bug)
return merges
def load_bugs(self, _filename):
bugs = {}
text = open(_filename, 'r').read()
doc = BeautifulSoup(text, 'html.parser')
bugTags = doc.select('bugrepository > bug')
for tag in bugTags:
bug = {}
info = tag.select('buginformation')[0]
bug['id'] = tag.attrs['id']
bug['summary'] = info.select('summary')[0].get_text()
bug['description'] = info.select('description')[0].get_text()
bug['type'] = info.select('type')[0].get_text()
bug['version'] = info.select('version')[0].get_text()
bug['fixedVersion'] = info.select('fixedversion')[0].get_text()
bug['resolution'] = tag.attrs['resolution']
bug['opendate'] = datetime.strptime(tag.attrs['opendate'],
'%Y-%m-%d %H:%M:%S')
bug['fixdate'] = datetime.strptime(tag.attrs['fixdate'],
'%Y-%m-%d %H:%M:%S')
bug['summary'] = cgi.escape(
re.sub(r'[^\x00-\x80]+', '', bug['summary'])
) #re.sub(r'[^\w\s&\^\|/()\[\]\{\}<>+\-=*/`~!@#$%^,.:;\\\'"?]', '', bug['summary']))
bug['description'] = BeautifulSoup(bug['description'],
"html.parser").get_text()
bug['description'] = cgi.escape(
re.sub(r'[^\x00-\x80]+', '', bug['description']))
bug['description'] = cgi.escape(
re.sub(chr(27), '', bug['description']))
bug['fixedFiles'] = []
fixedFiles = tag.select('fixedfiles > file')
for file in fixedFiles:
bug['fixedFiles'].append({
'type': file.attrs['type'],
'name': file.get_text()
})
bug['links'] = []
links = tag.select('links > link')
for link in links:
bug['links'].append({
'type': link.attrs['type'],
'description': link.attrs['description'],
'id': link.get_text()
})
bugs[int(bug['id'])] = bug
return bugs
def getItemsByVersion(self, _items, _versions):
_versions.sort(cmp=VersionUtil.cmpVersion)
version_bugs = dict((ver, list()) for ver in _versions)
size = len(_versions)
for idx in range(0, size):
version = _versions[idx]
nextVersion = _versions[
idx +
1] if idx != size - 1 else u'10000.0' # assign big. version number
for bugitem in _items:
if VersionUtil.cmpVersion(version,
bugitem['version']) > 0 and idx != 0:
continue
if VersionUtil.cmpVersion(bugitem['version'],
nextVersion) >= 0:
continue
version_bugs[version].append(bugitem)
vKeys = version_bugs.keys()
for version in vKeys:
if len(version_bugs[version]) != 0: continue
del version_bugs[version]
return version_bugs
#######################################################################
# Convert XML
#######################################################################
def convertText(self, _bug):
'''
Convert bug object to XML
:param _bug:
:return:
'''
format = u'\t<bug id="%s" master="%s" duplicate="%s" opendate="%s" fixdate="%s" resolution="%s">\n'
format += u'\t\t<buginformation>\n'
format += u'\t\t\t<summary>%s</summary>\n'
format += u'\t\t\t<description>%s</description>\n'
format += u'\t\t\t<version>%s</version>\n'
format += u'\t\t\t<fixedVersion>%s</fixedVersion>\n'
format += u'\t\t\t<type>%s</type>\n'
format += u'\t\t</buginformation>\n'
format += u'\t\t<fixedFiles>\n%s\n\t\t</fixedFiles>\n'
format += u'%s' #this section for links
format += u'\t</bug>\n'
fixedfiles = u'\n'.join(u'\t\t\t<file type="' + f['type'] + u'">' +
f['name'] + u'</file>'
for f in _bug['fixedFiles'])
links = u'\n'.join(u'\t\t\t<link type="' + link['type'] +
u'" description="' + link['description'] + u'">' +
link['id'] + u'</link>' for link in _bug['links'])
if links != u'':
links = u'\t\t<links>\n%s\n\t\t</links>\n' % links
text = format % (_bug['id'], _bug['master'], _bug['duplicate'],
_bug['opendate'].strftime(u'%Y-%m-%d %H:%M:%S'),
_bug['fixdate'].strftime(u'%Y-%m-%d %H:%M:%S'),
_bug['resolution'], _bug['summary'],
_bug['description'], _bug['version'],
_bug['fixedVersion'], _bug['type'], fixedfiles, links)
return text
def outputXML(self, _project, _items, _targetPath):
#write XML File
output = codecs.open(_targetPath, 'w', 'utf-8')
output.write(
u'<?xml version = "1.0" encoding = "UTF-8" ?>\n<bugrepository name="%s">\n'
% _project)
for item in _items:
output.write(self.convertText(item))
output.write(u'</bugrepository>')
output.flush()
output.close()
pass
def work():
obj = DupMergeRepositoryMaker()
S = Subjects()
for group in S.groups:
for project in S.projects[group]:
obj.make(group, project)
for tech in techniques: for version in _versions: vname = VersionUtil.get_versionName(version, _project) src_path = os.path.join(_src, u'%s_%s_%s' % (tech, _project, vname), u'recommended') if os.path.exists(src_path) is False: print(u'%s is not exists!!'%src_path) continue target_path = os.path.join(_dest, tech, _group, _project, u'%s_%s' % (tech, vname)) shutil.copytree(src_path, target_path) print(u'%s is Done!!' % src_path) pass ############################################################################################################### ############################################################################################################### ############################################################################################################### if __name__ == "__main__": import os WorkType = u'Full' S = Subjects() for group in S.groups: # ['Commons', 'JBoss', 'Wildfly', 'Spring'] for project in S.projects[group]: obj = CopyResults() obj.run(group, project, S.versions[project], S.getPath_result_folder(WorkType, group, project), os.path.join(S.root_result, 'organize_full')) pass
def __init__(self):
self.S = Subjects()
pass