def getItemsByVersion(self, _items, _versions):
_versions.sort(cmp=VersionUtil.cmpVersion)
version_bugs = dict((ver, list()) for ver in _versions)
size = len(_versions)
for idx in range(0, size):
version = _versions[idx]
nextVersion = _versions[
idx +
1] if idx != size - 1 else u'10000.0' # assign big. version number
for bugitem in _items:
if VersionUtil.cmpVersion(version,
bugitem['version']) > 0 and idx != 0:
continue
if VersionUtil.cmpVersion(bugitem['version'],
nextVersion) >= 0:
continue
version_bugs[version].append(bugitem)
vKeys = version_bugs.keys()
for version in vKeys:
if len(version_bugs[version]) != 0: continue
del version_bugs[version]
return version_bugs
def runOLD(self, _type, _sGroup=None, _sProject=None, _sProgram=None):
self.TYPE = _type
nameTag = self.TYPE + (u'_%s' % _sGroup if _sGroup is not None else u'') + (u'_%s' % _sProject if _sProject is not None else u'')
self.log = open(os.path.join(self.ProgramPATH, 'logs', self.logFileName%nameTag), 'w')
self.S = Previous()
for program in self.ProgramNames:
if _sProgram is not None and program != _sProgram: continue
for group in self.S.groups:
if _sGroup is not None and group != _sGroup: continue
for project in self.S.projects[group]:
if _sProject is not None and project != _sProject: continue
maxVersion = self.S.get_max_versions(program, project)
versionName = u'%s_%s' % (project, VersionUtil.get_versionName(maxVersion))
alpha = 0.2 if not (program == 'BugLocator' and project == 'AspectJ') else 0.3
params = self.get_paramsOLD(program, group, project, alpha, versionName)
#print(u'java %s -jar %s%s.jar ' % (self.JavaOptions, self.ProgramPATH, program) + self.createArguments(params))
self.executeJava(program, params, _project=project, _vname =versionName)
t = datetime.now()
timestr = t.strftime(u'Done:%Y/%m/%d %H:%M')
print(u'\n\n[%s]' % timestr)
self.log.write('\n\n[%s]' % timestr)
pass
def bug_counting(self, _group, _project):
statistics = {}
repo = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository.xml')
statistics['all'] = self.getBugs(repo)
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository', u'%s.xml' % vname)
result = self.getBugs(repo)
if result is None: continue
statistics[vname] = result
# Check missed items
idset = set(statistics['all'])
for key in statistics:
if key == 'all': continue
idset -= set(statistics[key])
statistics['miss'] = list(idset)
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(
os.path.join(self.S.getPath_base(_group, _project), u'bugs.txt'),
'w')
f.write(text)
f.close()
pass
def source_counting(self, _group, _project):
statistics = {}
progress = Progress('source counting', 2, 10, True)
progress.set_upperbound(len(self.S.versions[_project].keys()))
progress.start()
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_source(_group, _project,
vname), )
result = self.getCodeCount(repo)
if result is None: continue
statistics[vname] = result
progress.check()
progress.done()
maxValue = 0
for vname in statistics:
if maxValue < statistics[vname]:
maxValue = statistics[vname]
statistics['max'] = maxValue
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(
os.path.join(self.S.getPath_base(_group, _project),
u'sources.txt'), 'w')
f.write(text)
f.close()
def make_minimumVersion(self, _bugs):
for bug in _bugs:
min_version = u'10000.0' # assign big version
for version in bug['version'].split(u', '):
if VersionUtil.cmpVersion(version, min_version) < 0:
min_version = version
bug['version'] = min_version
pass
def run(self, _type, _sGroup=None, _sProject=None, _sProgram=None, _sVersion=None, _isUnion=False, _isDist=False, _useMerge=False):
self.TYPE = _type
nameTag = self.TYPE + (u'_%s'%_sGroup if _sGroup is not None else u'') + (u'_%s'%_sProject if _sProject is not None else u'') + (u'_%s'%_sVersion if _sVersion is not None else u'') + (u'_%s'%_sProgram if _sProgram is not None else u'')
self.log = open(os.path.join(self.ProgramPATH, 'logs', self.logFileName%(nameTag)), 'w')
# select target subjects or select all.
for program in (self.ProgramNames if _sProgram is None else [_sProgram]):
for group in (self.S.groups if _sGroup is None else [_sGroup]):
for project in (self.S.projects[group] if _sProject is None else [_sProject]):
#working selected subject and program.
versions = self.S.bugs[project].keys()
if _isDist is True:
codeVersion = VersionUtil.get_latest_version(self.S.bugs[project].keys())
for verName in (versions if _sVersion is None else [_sVersion]):
if verName == 'all': continue
params = self.get_paramsDist(program, group, project, 0.2, verName, codeVersion)
self.executeJava(program+u'_dist', params, _project=project, _vname =verName)
elif _isUnion is True:
# if the self.S.version[project] uses, the error occurs because there are versions with no bug report
verName = VersionUtil.get_latest_version(self.S.bugs[project].keys())
params = self.get_params(program, group, project, 0.2, verName, _isUnion)
self.executeJava(program, params, _project=project, _vname =verName)
else:
# In the version is not single case,
for verName in (versions if _sVersion is None else [_sVersion]):
if verName == 'all': continue
if _useMerge is True:
if verName not in self.S.answers_merge[project]: continue
# if the self.S.version[project] uses, the error occurs because there are versions with no bug report
outputFile = os.path.join(self.OutputPATH, _type, group, project, u'%s_%s_%s_output.txt'%(program, project, verName))
# if os.path.exists(outputFile) is True:
# print(u'Already exists :: %s '% outputFile)
# continue
params = self.get_params(program, group, project, 0.2, verName, _isUnion, _useMerge)
self.executeJava(program, params, _project=project, _vname =verName)
# for version
# for program
# for project
#for group
t = datetime.now()
timestr = t.strftime(u'Done:%Y/%m/%d %H:%M')
print(u'\n\n[%s]' % timestr)
self.log.write('\n\n[%s]' % timestr)
def inflate(self, _versions):
if _versions is None:
return None
self.projectPath = self.clone(True)
print(u'Git Repo: %s' % self.projectPath)
time.sleep(2)
#check output path
if os.path.exists(self.sourcesPath) is False:
os.makedirs(self.sourcesPath)
#get tags
# tags = self.get_tags()
# if tags is None: return False
#print(self.projectName + u':: the number of tags are ' + str(len(tags)))
size = len(_versions)
count = 0
for version, tag in _versions.items():
vname = VersionUtil.get_versionName(version, self.projectName)
count += 1
print(u'%s(%d/%d) :: [%s]' %
(self.projectName, count, size, vname),
end=u'')
dest = os.path.join(self.sourcesPath, vname)
if os.path.exists(dest) is True:
print(u' already exists!')
continue
tag = tag.strip()
if tag == u'':
print(u'invalidate tag name: "%s"' % tag)
continue
print(u' checkout %s... ' % tag, end=u'')
if self.checkout(tag) is False:
print(u'Failed')
continue
time.sleep(2)
#copy
dest = os.path.join(self.sourcesPath, vname)
print(u' copy...', end=u'')
if self.makecopy(dest) is False:
print(u'Failed!')
continue
print(u'Done')
time.sleep(2)
print(u'All checkout works done!!')
pass
def make_answers(self, _project, _items, _versionItems):
answers = {'all': {}}
for item in _items:
answers['all'][int(item['id'])] = len(item['fixedFiles'])
for version in _versionItems:
versionName = VersionUtil.get_versionName(version, _project)
answers[versionName] = {}
for item in _versionItems[version]:
answers[versionName][int(item['id'])] = len(item['fixedFiles'])
return answers
def merge(self, _group, _project, _bugs, _dups):
merges = []
bugID = 0
for src, dup in _dups:
if src not in _bugs or dup not in _bugs:
print('passed %s\t%s\t%d\t%d' % (_group, _project, src, dup))
continue
bugID += 1
bug = {}
bug['id'] = str(bugID)
bug['master'] = _bugs[src]['id']
bug['duplicate'] = _bugs[dup]['id']
bug['summary'] = _bugs[src]['summary'] + u' ' + _bugs[dup][
'summary']
bug['description'] = _bugs[src]['description'] + u' ' + _bugs[dup][
'description']
verSrc = _bugs[src]['version']
verDup = _bugs[dup]['version']
bug['version'] = verSrc if VersionUtil.cmpVersion(
verSrc, verDup) > 0 else verDup
verSrc = _bugs[src]['fixedVersion']
verDup = _bugs[dup]['fixedVersion']
bug['fixedVersion'] = verSrc if VersionUtil.cmpVersion(
verSrc, verDup) > 0 else verDup
bug['resolution'] = _bugs[src]['resolution']
bug['opendate'] = min(_bugs[src]['opendate'],
_bugs[dup]['opendate'])
bug['fixdate'] = max(_bugs[src]['fixdate'], _bugs[dup]['fixdate'])
bug['type'] = _bugs[src]['type']
bug['fixedFiles'] = _bugs[src]['fixedFiles']
bug['links'] = _bugs[src]['links'] + _bugs[dup]['links']
merges.append(bug)
return merges
def make(self, _group, _project):
'''
make repository files
:param _group:
:param _project:
:return:
'''
self.S = Subjects()
print(u'working with %s / %s' % (_group, _project))
filename = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository.xml')
bugs = self.load_bugs(filename)
dups = self.S.duplicates[_project]
merges = self.merge(_group, _project, bugs, dups)
print(u'created %d merged reports.' % len(merges))
versionItems = self.getItemsByVersion(merges,
self.S.versions[_project].keys())
print(u'created %d version repositories from merged reports.' %
len(versionItems))
print(u'storing....', end=u'')
repositoryPath = os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository_merge')
if os.path.exists(repositoryPath) is True:
shutil.rmtree(repositoryPath)
if os.path.exists(repositoryPath) is False:
os.makedirs(repositoryPath)
self.outputXML(
_project, merges,
os.path.join(self.S.getPath_bugrepo(_group, _project),
u'repository_merge.xml'))
for ver in versionItems.keys():
filename = os.path.join(
repositoryPath,
VersionUtil.get_versionName(ver, _project) + u'.xml')
self.outputXML(_project, versionItems[ver], filename)
print(u'Done')
answers = self.make_answers(_project, merges, versionItems)
self.save_answers({_project: answers},
os.path.join(self.S.getPath_base(_group, _project),
u'answers_merge.txt'))
pass
def answers_counting(self, _group, _project):
statistics = {}
repo = os.path.join(self.S.getPath_bugrepo(_group, _project), u'repository.xml')
statistics['all'] = self.getAnswers(repo)
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_bugrepo(_group, _project), u'repository', u'%s.xml'%vname)
result = self.getAnswers(repo)
if result is None: continue
statistics[vname] = result
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(os.path.join(self.S.getPath_base(_group, _project), u'answers.txt'), 'w')
f.write(text)
f.close()
def get_gitversion(self, _id):
'''
get bug version information from git repository
:param _id:
:return:
'''
if _id not in self.gitlogs: return u''
min_version = u''
commits = self.gitlogs[_id]
for commit in commits:
if commit['hash'] not in self.gitversions: continue
version = self.gitversions[commit['hash']]
if version is None: continue
if min_version == u'': min_version = version
if VersionUtil.cmpVersion(version, min_version) < 0:
min_version = version
return min_version
def run(self, _group, _project, _versions, _src, _dest): ''' create result file ''' techniques = ['BLIA']#'BugLocator', "BRTracer"]#, 'BLUiR', 'BLIA','AmaLgam', 'Locus'] for tech in techniques: for version in _versions: vname = VersionUtil.get_versionName(version, _project) src_path = os.path.join(_src, u'%s_%s_%s' % (tech, _project, vname), u'recommended') if os.path.exists(src_path) is False: print(u'%s is not exists!!'%src_path) continue target_path = os.path.join(_dest, tech, _group, _project, u'%s_%s' % (tech, vname)) shutil.copytree(src_path, target_path) print(u'%s is Done!!' % src_path) pass
def get_countings(self, _group, _project, _tech):
from Counting import Counting
counter = Counting()
if _tech == u'BLIA' and _project in [u'AspectJ', u'SWT', u'ZXing']:
filename = u'BLIA_repository.xml'
elif _tech == u'Locus' and _project == u'AspectJ':
filename = u'Locus_repository.xml'
else:
filename = u'repository.xml'
repoPath = os.path.join(self.S.getPath_bugrepo(_group, _project),
filename)
answers = counter.getAnswers(repoPath)
bugs = counter.getBugs(repoPath)
# make the count of source code
cache = os.path.join(self.S.getPath_base(_group, _project),
u'sources.txt')
if os.path.exists(cache) is False:
print(u'making sourcecode counts...', end=u'')
counts = {_project: {}}
baseCodePath = self.S.getPath_source(_group, _project)
for dir in os.listdir(baseCodePath):
counts[_project][dir] = counter.getCodeCount(
os.path.join(baseCodePath, dir))
from utils.PrettyStringBuilder import PrettyStringBuilder
builder = PrettyStringBuilder(_indent_depth=2)
text = builder.get_dicttext(counts)
f = open(cache, 'w')
f.write(text)
f.close()
print(u'Done.')
#read the count of source code
f = open(cache, 'r')
text = f.read()
f.close()
data = eval(text)
vname = VersionUtil.get_versionName(
self.S.get_max_versions(_tech, _project), _project)
sources = data[_project][vname]
return sources, bugs, answers
def complement_reports(self, _src, _dest, _both):
'''
complement information from the duplicate bug report
:param _bugitems:
:param _dupgroups:
:param _gitversions:
:return:
'''
# sync fixedfile
if _both is False:
if len(_src['fixedFiles']) == 0:
_src['fixedFiles'] = _dest['fixedFiles']
_src['commits'] = _dest['commits']
else:
_dest['fixedFiles'] = _src['fixedFiles']
_dest['commits'] = _src['commits']
# sync version
if _dest['version'] != u'' and _src['version'] == u'':
_src['version'] = _dest['version']
elif _src['version'] != u'' and _dest['version'] == u'':
_dest['version'] = _src['version']
elif _src['version'] == u'' and _dest['version'] == u'':
#if both report has no version, get version information from git repository
v1 = self.get_gitversion(_src['id'])
v2 = self.get_gitversion(_dest['id'])
if v1 != u'' and v2 != u'':
if v1 == u'': _src['version'] = v2
_src['version'] = v1 if VersionUtil.cmpVersion(v1,
v2) < 0 else v2
_dest['version'] = _src['version']
# sync fixdate
if _dest['fixdate'] != u'' and _src['fixdate'] == u'':
_src['fixdate'] = _dest['fixdate']
if _src['fixdate'] != u'' and _dest['fixdate'] == u'':
_src['fixdate'] = _src['fixdate']
pass
def getItemsOnlyVersion(self, _items):
newItems = []
for item in _items:
if VersionUtil.hasVersion(item['version']) is False: continue
newItems.append(item)
return newItems