def startMongoD():
global mongod
if not isOpen('127.0.0.1', 27017):
print 'Startind MongoDB daemon ...'
with cd(S.DATA_DIR):
mongod = subprocess.Popen(
['mongod', '--dbpath',
os.path.expanduser(S.DATA_DIR)])
def analyze_bugzilla(statList, bugzillaData, cfg):
print("Analyze bugzilla\n", end="", flush=True)
for key, row in bugzillaData['bugs'].items():
rowId = row['id']
#Ignore META bugs and deletionrequest bugs.
if not row['summary'].lower().startswith('[meta]') and row['component'].lower() != 'deletionrequest':
rowStatus = row['status']
rowResolution = row['resolution']
if rowStatus == 'VERIFIED' or rowStatus == 'RESOLVED':
rowStatus += "_" + rowResolution
rowKeywords = row['keywords']
rowCreator = row['creator_detail']['real_name']
if not rowCreator:
rowCreator = row['creator_detail']['email'].split('@')[0]
if rowStatus == "NEEDINFO" and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['needInfoPingPeriod']:
statList['needInfoPing'][rowId] = rowCreator
if common.isClosed(row['status']) and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['backportRequestPeriod']:
for i in row['whiteboard'].split(' '):
if 'backport' in i.lower():
statList['backportRequest']['remove'][rowId] = i
if common.isOpen(row['status']) and 'regression' in rowKeywords and \
'bibisectRequest' not in rowKeywords and 'bibisected' not in rowKeywords and \
'bisected' not in rowKeywords and 'preBibisect' not in rowKeywords and \
'bibisectNotNeeded' not in rowKeywords and 'notBibisectable' not in rowKeywords:
if row['severity'] is not 'enhancement':
if row['op_sys'] in ["All", "Windows (All)", "Linux (All)", "macOS (All)"]:
statList['tagRegression']['add'][rowId] = 'bibisectRequest'
else:
statList['tagRegression']['add'][rowId] = 'notBibisectable'
else:
statList['tagRegression']['remove'][rowId] = 'regression'
comments = row['comments'][1:]
bSameAuthor = True
for idx, comment in enumerate(comments):
#Check for duplicated comments
if idx > 0 and comment['text'] == comments[idx-1]['text']:
statList['tags']['addObsolete'].add(comment["id"])
if rowStatus != 'NEEDINFO' and \
"obsolete" not in [x.lower() for x in comment["tags"]] and \
('MassPing-UntouchedBug' in comment["text"] or \
comment["text"].startswith("A polite ping, still working on this bug") or \
'[Automated Action]' in comment["text"] or \
'MassPing-NeedInfo' in comment["text"]):
statList['tags']['addObsolete'].add(comment["id"])
if bSameAuthor and comment['creator'] != row['creator']:
bSameAuthor = False
if bSameAuthor and rowStatus == 'UNCONFIRMED':
if row['component'] == 'UI' and row['severity'] == 'enhancement' and 'needsUXEval' not in rowKeywords:
statList['needsUXEval']['add'][rowId] = 'needsUXEval'
elif needsCommentTag not in row['whiteboard'] and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['needsCommentPeriod']:
statList['needsComment']['add'][rowId] = needsCommentTag
elif not bSameAuthor and needsCommentTag in row['whiteboard']:
statList['needsComment']['remove'][rowId] = needsCommentTag
if len(comments) > 0:
if rowStatus == 'NEEDINFO' and \
comments[-1]['creator'] == row['creator']:
statList['needInfoToUnconfirmed'][rowId] = rowCreator
if 'MassPing-NeedInfo-Ping' in comments[-1]["text"]:
if rowStatus != 'NEEDINFO':
if "obsolete" not in [x.lower() for x in comments[-1]["tags"]]:
statList['tags']['addObsolete'].remove(comments[-1]["id"])
else:
statList['tags']['removeObsolete'].add(comments[-1]["id"])
else:
if datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['needInfoFollowUpPingPeriod']:
statList['needInfoFollowUpPing'][rowId] = rowCreator
elif 'MassPing-NeedInfo' in comments[-1]["text"] or \
comments[-1]["text"].startswith("A polite ping, still working on this bug"):
if rowStatus != 'NEEDINFO':
if "obsolete" not in [x.lower() for x in comments[-1]["tags"]]:
statList['tags']['addObsolete'].remove(comments[-1]["id"])
else:
statList['tags']['removeObsolete'].add(comments[-1]["id"])
else:
if 'MassPing-UntouchedBug' in comments[-1]["text"]:
if rowStatus != 'NEEDINFO':
if "obsolete" not in [x.lower() for x in comments[-1]["tags"]]:
statList['tags']['addObsolete'].remove(comments[-1]["id"])
else:
statList['tags']['removeObsolete'].add(comments[-1]["id"])
if datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['untouchedPeriod'] and \
rowStatus == 'NEW' and 'needsUXEval' not in rowKeywords and 'easyHack' not in rowKeywords and \
row['component'] != 'Documentation' and (row['product'] == 'LibreOffice' or \
row['product'] == 'Impress Remote') and row['severity'] != 'enhancement':
statList['untouched'][rowId] = rowCreator
if 'MassPing-UntouchedBug' in comments[-1]["text"]:
statList['tags']['addObsolete'].add(comments[-1]["id"])
if comments[-1]["id"] in statList['tags']['removeObsolete']:
statList['tags']['removeObsolete'].remove(comments[-1]["id"])
def analyze_bugzilla_wiki_stats(statList, bugzillaData, cfg):
print("Analyzing bugzilla\n", end="", flush=True)
statNewDate = statList['stat']['newest']
statOldDate = statList['stat']['oldest']
for key, row in bugzillaData['bugs'].items():
rowId = row['id']
#Ignore META bugs and deletionrequest bugs.
if not row['summary'].lower().startswith('[meta]') and row['component'].lower() != 'deletionrequest':
creationDate = datetime.datetime.strptime(row['creation_time'], "%Y-%m-%dT%H:%M:%SZ")
if creationDate < statOldDate:
statOldDate = creationDate
if creationDate > statNewDate:
statNewDate = creationDate
rowStatus = row['status']
rowResolution = row['resolution']
if rowStatus == 'VERIFIED' or rowStatus == 'RESOLVED':
rowStatus += "_" + rowResolution
rowKeywords = row['keywords']
creatorMail = row['creator']
common.util_check_bugzilla_mail(
statList, creatorMail, row['creator_detail']['real_name'], creationDate, rowId)
whiteboard_list = row['whiteboard'].split(' ')
bugTargets = []
for whiteboard in whiteboard_list:
if whiteboard.startswith("target:"):
bugVersion = whiteboard.split(':')[1][:5]
if bugVersion in targets_list:
bugTargets.append(bugVersion)
statList['targets'][bugVersion]['count'] += 1
for period in periods_list:
if creationDate >= cfg[period]:
statList['period'][period]['count'] += 1
util_increase_user_actions(statList, key, creatorMail, bugTargets, 'created', creationDate)
if common.isOpen(rowStatus) and len(row['cc']) >= 10:
typeName = 'bugs'
if row['severity'] == "enhancement":
typeName = 'enhancements'
elif 'regression' in rowKeywords:
typeName = 'regressions'
statList['CC'][typeName][rowId] = util_create_bug(
row['summary'], row['component'], row['op_sys'], row['version'], rowKeywords, creationDate, len(row['cc']))
rowDupeOf = common.util_check_duplicated(bugzillaData, rowId)
if rowDupeOf and str(rowDupeOf) in bugzillaData['bugs'] and \
common.isOpen(bugzillaData['bugs'][str(rowDupeOf)]['status']):
if rowDupeOf not in statList['dupesBugs']:
statList['dupesBugs'][rowDupeOf] = []
statList['dupesBugs'][rowDupeOf].append(rowId)
typeName = 'bugs'
if bugzillaData['bugs'][str(rowDupeOf)]['severity'] == "enhancement":
typeName = 'enhancements'
elif 'regression' in bugzillaData['bugs'][str(rowDupeOf)]['keywords']:
typeName = 'regressions'
statList['duplicates'][typeName][rowDupeOf] = util_create_bug(
bugzillaData['bugs'][str(rowDupeOf)]['summary'],
bugzillaData['bugs'][str(rowDupeOf)]['component'],
bugzillaData['bugs'][str(rowDupeOf)]['op_sys'],
bugzillaData['bugs'][str(rowDupeOf)]['version'],
bugzillaData['bugs'][str(rowDupeOf)]['keywords'],
datetime.datetime.strptime(
bugzillaData['bugs'][str(rowDupeOf)]['creation_time'], "%Y-%m-%dT%H:%M:%SZ"), 1)
for action in row['history']:
actionMail = action['who']
actionDate = datetime.datetime.strptime(action['when'], "%Y-%m-%dT%H:%M:%SZ")
# Use this variable in case the status is set before the resolution
newStatus = None
for change in action['changes']:
if change['field_name'] == 'blocks':
if change['added']:
for metabug in change['added'].split(', '):
continue
#TODO
#util_increase_user_actions(statList, key, actionMail, bugTargets, 'metabug_added', actionDate)
if change['removed']:
for metabug in change['removed'].split(', '):
continue
#TODO
#util_increase_user_actions(statList, key, actionMail, bugTargets, 'metabug_added', actionDate)
if change['field_name'] == 'status':
addedStatus = change['added']
removedStatus = change['removed']
if addedStatus == 'RESOLVED' or addedStatus == 'VERIFIED':
if(rowResolution):
addedStatus = addedStatus + "_" + rowResolution
util_increase_user_actions(statList, key, actionMail, bugTargets, 'status_changed', actionDate)
else:
newStatus = addedStatus
else:
util_increase_user_actions(statList, key, actionMail, bugTargets, 'status_changed', actionDate)
elif change['field_name'] == 'resolution':
if newStatus:
addedStatus = newStatus + "_" + change['added']
util_increase_user_actions(statList, key, actionMail, bugTargets, 'status_changed', actionDate)
newStatus = None
elif change['field_name'] == 'priority':
util_increase_user_actions(statList, key, actionMail, bugTargets, 'priority_changed', actionDate)
elif change['field_name'] == 'severity':
util_increase_user_actions(statList, key, actionMail, bugTargets, 'severity_changed', actionDate)
elif change['field_name'] == 'keywords':
keywordsAdded = change['added'].split(", ")
for keyword in keywordsAdded:
if keyword in common.keywords_list:
util_increase_user_actions(statList, key, actionMail, bugTargets, 'keyword_added', actionDate)
keywordsRemoved = change['removed'].split(", ")
for keyword in keywordsRemoved:
if keyword in common.keywords_list:
util_increase_user_actions(statList, key, actionMail, bugTargets, 'keyword_removed', actionDate)
elif change['field_name'] == 'op_sys':
newSystem = change['added']
util_increase_user_actions(statList, rowId, actionMail, bugTargets, 'system_changed', actionDate)
comments = row['comments'][1:]
for idx, comment in enumerate(comments):
commentMail = comment['creator']
commentDate = datetime.datetime.strptime(comment['time'], "%Y-%m-%dT%H:%M:%SZ")
util_increase_user_actions(statList, rowId, commentMail, bugTargets, 'comments', commentDate)
#this way we can get the users' name
for person in row['cc_detail']:
common.util_check_bugzilla_mail(statList, person['email'], person['real_name'])
statList['stat']['newest'] = statNewDate.strftime("%Y-%m-%d")
statList['stat']['oldest'] = statOldDate.strftime("%Y-%m-%d")
print(" from " + statList['stat']['oldest'] + " to " + statList['stat']['newest'])
def analyze_bugzilla_checkers(statList, bugzillaData, cfg):
print("Analyzing bugzilla chekers\n", end="", flush=True)
lResults = {}
dupesBugs = {}
for key, row in bugzillaData['bugs'].items():
rowId = row['id']
#Ignore META bugs and deletionrequest bugs.
if not row['summary'].lower().startswith(
'[meta]') and row['component'].lower() != 'deletionrequest':
creationDate = datetime.datetime.strptime(row['creation_time'],
"%Y-%m-%dT%H:%M:%SZ")
rowStatus = row['status']
rowResolution = row['resolution']
if rowStatus == 'VERIFIED' or rowStatus == 'RESOLVED':
rowStatus += "_" + rowResolution
rowKeywords = row['keywords']
creatorMail = row['creator']
rowVersion = row['version']
common.util_check_bugzilla_mail(statList, creatorMail,
row['creator_detail']['real_name'],
creationDate, rowId)
if common.isOpen(rowStatus) and rowId not in dupesBugs:
dupesBugs[rowId] = {
'totalDupes': 0,
'totalCC': len(row['cc']),
'priority': row['priority'],
'severity': row['severity'],
'isRegression': 'regression' in rowKeywords
}
rowDupeOf = common.util_check_duplicated(bugzillaData, rowId)
if rowDupeOf and common.isOpen(
bugzillaData['bugs'][str(rowDupeOf)]['status']):
if rowDupeOf not in dupesBugs:
dupesBugs[rowDupeOf] = {
'totalDupes':
0,
'totalCC':
len(bugzillaData['bugs'][str(rowDupeOf)]['cc']),
'priority':
bugzillaData['bugs'][str(rowDupeOf)]['priority'],
'severity':
bugzillaData['bugs'][str(rowDupeOf)]['severity'],
'isRegression':
'regression'
in bugzillaData['bugs'][str(rowDupeOf)]['keywords']
}
dupesBugs[rowDupeOf]['totalDupes'] += 1
crashSignature = row['cf_crashreport']
if crashSignature:
if not crashSignature.startswith('["'):
util_add_to_result(lResults, 'incorrect_crash_signature',
rowId)
else:
if common.isOpen(rowStatus):
lcrashSignature = ast.literal_eval(crashSignature)
for i in lcrashSignature:
crashReportUrl = crashReportDomain + str(
i).replace(' ', '%20').replace('`', '%60')
crashReportUrl = crashReportUrl.replace(
'<', '%3C').replace('>', '%3E')
#Link should be shorter than 255, otherwise Bugzilla returns an error
if crashReportUrl not in row['see_also'] and len(
crashReportUrl) < 255:
util_add_to_result(
lResults, 'add_crashReport_to_seeAlso',
rowId)
everConfirmed = False
autoConfirmed = False
versionChanged = False
oldestVersion = 999999
newerVersion = False
autoFixed = False
closeDate = None
movedToFixed = False
movedToNeedInfo = False
isReopened = False
addAssigned = False
movedToNew = False
addAssigned = False
actionMail = None
for action in row['history']:
actionMail = action['who']
actionDate = datetime.datetime.strptime(
action['when'], "%Y-%m-%dT%H:%M:%SZ")
common.util_check_bugzilla_mail(statList, actionMail, '',
actionDate, rowId)
# Use this variable in case the status is set before the resolution
newStatus = None
for change in action['changes']:
if change['field_name'] == 'version':
versionChanged = True
if actionDate >= cfg['reportPeriod'] and (
common.isOpen(rowStatus)
or rowStatus == 'UNCONFIRMED'):
addedVersion = change['added']
removedVersion = change['removed']
if addedVersion == 'unspecified':
addedVersion = 999999
elif addedVersion == 'Inherited From OOo':
addedVersion = 0
else:
addedVersion = int(''.join([
s for s in re.split('\.|\s', addedVersion)
if s.isdigit()
]).ljust(3, '0')[:2])
if removedVersion == 'unspecified':
removedVersion = 999999
elif removedVersion == 'Inherited From OOo':
removedVersion = 0
else:
removedVersion = int(''.join([
s
for s in re.split('\.|\s', removedVersion)
if s.isdigit()
]).ljust(3, '0')[:2])
if removedVersion < oldestVersion:
oldestVersion = removedVersion
if addedVersion <= oldestVersion:
oldestVersion = addedVersion
newerVersion = False
else:
newerVersion = True
elif change['field_name'] == 'status':
addedStatus = change['added']
removedStatus = change['removed']
if addedStatus == 'REOPENED' and rowStatus == 'REOPENED' and not movedToFixed:
isReopened = True
if actionDate >= cfg['reportPeriod'] and addedStatus == 'NEEDINFO' and \
rowStatus == 'NEEDINFO' and common.isOpen(removedStatus):
movedToNeedInfo = True
if movedToNeedInfo and removedStatus == 'NEEDINFO':
movedToNeedInfo = False
if addedStatus == 'RESOLVED' or addedStatus == 'VERIFIED':
if (rowResolution):
addedStatus = addedStatus + "_" + rowResolution
else:
newStatus = addedStatus
#if any other user moves it to open ( ASSIGNED, NEW or REOPENED ),
#the bug is no longer autoconfirmed
if not everConfirmed and common.isOpen(
addedStatus) and actionMail != creatorMail:
everConfirmed = True
autoConfirmed = False
#Check for autoconfirmed bugs:
#Bug's status is open ( ASSIGNED, NEW or REOPENED ), moved to open by the reporter
#from non-open status and never confirmed by someone else.
#Ignore bisected bugs or some trusted authors defined in configQA.json
if actionDate >= cfg['reportPeriod'] and not everConfirmed and actionMail == creatorMail and \
common.isOpen(rowStatus) and common.isOpen(addedStatus) and 'bisected' not in rowKeywords and \
creatorMail not in cfg['configQA']['ignore']['autoConfirmed']:
autoConfirmed = True
if autoFixed and removedStatus == 'RESOLVED':
autoFixed = False
if actionDate >= cfg['reportPeriod']:
if actionMail == creatorMail and addedStatus == 'RESOLVED_FIXED' and \
rowStatus == 'RESOLVED_FIXED' and 'target:' not in row['whiteboard']:
autoFixed = True
if removedStatus == "ASSIGNED" and addedStatus == "NEW" and \
rowStatus == "NEW" and row['assigned_to'] != '*****@*****.**':
util_add_to_result(lResults, 'remove_assignee',
rowId)
elif addedStatus == "ASSIGNED" and rowStatus == "ASSIGNED" and \
row['assigned_to'] == '*****@*****.**':
util_add_to_result(lResults, 'add_assignee',
rowId)
if addedStatus == 'NEW' and rowStatus == 'NEW' and row[
'product'] == 'LibreOffice':
movedToNew = True
if addedStatus == 'RESOLVED_FIXED' and rowStatus == 'RESOLVED_FIXED' and ('regression' in rowKeywords \
or 'crash' in row['summary'].lower() or 'perf' in rowKeywords):
util_add_to_result(lResults, 'verify_fix',
rowId)
elif change['field_name'] == 'resolution':
if newStatus:
addedStatus = newStatus + "_" + change['added']
newStatus = None
if change['added'] == 'FIXED':
movedToFixed = True
isReopened = False
if common.isOpen(rowStatus):
closeDate = actionDate
elif change['removed'] == 'FIXED' and actionDate >= cfg['reportPeriod'] and \
closeDate and (actionDate - closeDate).days > 30:
util_add_to_result(lResults,
'reopened_after_1_months',
rowId)
elif change['field_name'] == 'keywords':
if actionDate >= cfg['reportPeriod']:
keywordsAdded = change['added'].split(", ")
for keyword in keywordsAdded:
if keyword in common.keywords_list and keyword in rowKeywords:
if keyword == 'patch' and (
common.isOpen(rowStatus)
or rowStatus == 'UNCONFIRMED'):
util_add_to_result(
lResults, 'patch_added', rowId)
if row['status'] != 'RESOLVED':
if keyword == 'bibisectRequest' and 'regression' not in rowKeywords:
util_add_to_result(
lResults,
'bibisectRequest_added', rowId)
elif keyword == 'possibleRegression' and 'possibleRegression' in rowKeywords:
util_add_to_result(
lResults,
'possibleregression_added',
rowId)
elif keyword == 'needsUXEval' and '*****@*****.**' not in row[
'cc']:
util_add_to_result(
lResults,
'needsUXEval_missing_email',
rowId)
elif change['field_name'] == 'whiteboard':
if actionDate >= cfg['reportPeriod']:
for whiteboard in change['added'].split(' '):
if 'backportrequest' in whiteboard.lower() and \
whiteboard in row['whiteboard'] and common.isOpen(rowStatus):
util_add_to_result(lResults,
'backport_added', rowId)
elif change['field_name'] == 'assigned_to':
if actionDate >= cfg['reportPeriod']:
removedAssignee = change['removed']
addedAssignee = change['added']
if removedAssignee == "*****@*****.**" and \
row['assigned_to'] != '*****@*****.**' and \
( rowStatus == 'NEW' or rowStatus == 'UNCONFIRMED'):
addAssigned = True
util_add_to_result(lResults,
'change_status_assigned',
rowId)
if addedAssignee == "*****@*****.**" and \
row['assigned_to'] == '*****@*****.**' and rowStatus == 'ASSIGNED':
util_add_to_result(lResults,
'remove_assigned_status',
rowId)
commentMail = None
comments = row['comments'][1:]
for idx, comment in enumerate(comments):
commentMail = comment['creator']
commentDate = datetime.datetime.strptime(
comment['time'], "%Y-%m-%dT%H:%M:%SZ")
common.util_check_bugzilla_mail(statList, commentMail, '',
commentDate, rowId)
if len(comments) > 0:
if rowStatus == 'UNCONFIRMED' and 'needsDevAdvice' not in rowKeywords and row[
'severity'] != 'enhancement':
if comments[-1]['creator'] != creatorMail and '[Automated Action]' not in comments[-1]['text'] and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['retestUnconfirmedPeriod']:
util_add_to_result(
lResults,
'unconfirmed_last_comment_not_from_reporter',
rowId)
elif comments[-1]['creator'] == creatorMail and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['inactiveUnconfirmedPeriod']:
util_add_to_result(
lResults, 'unconfirmed_last_comment_from_reporter',
rowId)
if rowStatus == 'UNCONFIRMED' and row['severity'] == 'enhancement' and 'QA:needsComment' not in row['whiteboard'] and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['retestUnconfirmedPeriod']:
util_add_to_result(lResults,
'inactive_unconfirmed_enhacements', rowId)
if autoFixed:
util_add_to_result(lResults, 'auto_fixed', rowId)
if autoConfirmed:
util_add_to_result(lResults, 'auto_confirmed', rowId)
if newerVersion and rowVersion != 'unspecified':
util_add_to_result(lResults, 'newer_version', rowId)
if isReopened and not autoConfirmed:
util_add_to_result(lResults, 'is_reopened', rowId)
#In case the reporter assigned the bug to himself at creation time
if not addAssigned and creationDate >= cfg['reportPeriod'] and \
row['assigned_to'] != '*****@*****.**' and (rowStatus == 'NEW' or rowStatus == 'UNCONFIRMED'):
util_add_to_result(lResults, 'change_status_assigned', rowId)
if movedToNeedInfo and everConfirmed:
util_add_to_result(lResults, 'moved_to_needinfo', rowId)
if not versionChanged and movedToNew and not autoConfirmed and row['severity'] != 'enhancement' and \
'regression' not in rowKeywords and 'bisected' not in rowKeywords and \
'easyHack' not in rowKeywords and 'needsUXEval' not in rowKeywords and \
row['component'] != 'Documentation' and \
row['component'] != 'UI' and \
row['component'] != 'iOS' and row['component'] != 'Android Viewer' and \
actionMail not in cfg['configQA']['ignore']['confirmer'] and \
(rowVersion.startswith(versionsToCheck) or rowVersion == 'unspecified'):
util_add_to_result(lResults, 'version_not_changed', rowId)
if common.isOpen(rowStatus) and 'target:' in row[
'whiteboard'] and 'easyHack' not in row['keywords']:
it = 1
#Check the last 3 comments
totalComments = 3
while it <= totalComments:
negIt = it * -1
if len(comments) >= it:
commentMail = comments[negIt]['creator']
commentText = comments[negIt]['text']
commentDate = datetime.datetime.strptime(
comments[negIt]['time'], "%Y-%m-%dT%H:%M:%SZ")
if commentDate < cfg[
'PingFixedBugPeriod'] and commentDate >= cfg[
'pingFixedBugDiff']:
if it == 1 and 'Is this bug fixed?' in commentText and commentMail == '*****@*****.**':
util_add_to_result(lResults,
'take_action_fixed_bug',
rowId)
break
elif commentMail == '*****@*****.**':
# Check the commit hasn't been reverted
if 'evert' in commentText:
break
else:
util_add_to_result(lResults,
'ping_bug_fixed', rowId)
break
# Ignore duplicated comments
elif 'has been marked as a duplicate of this bug' in commentText:
totalComments += 1
it += 1
else:
break
else:
break
if rowStatus == 'ASSIGNED' and \
datetime.datetime.strptime(row['last_change_time'], "%Y-%m-%dT%H:%M:%SZ") < cfg['inactiveAssignedPeriod'] and \
rowId not in cfg['configQA']['ignore']['inactiveAssigned']:
util_add_to_result(lResults, 'inactive_assignee', rowId)
elif row['summary'].lower().startswith('[meta]'):
if not row['alias'] and common.isOpen(row['status']):
util_add_to_result(lResults, 'empty_alias', rowId)
for k, v in dupesBugs.items():
if v['severity'] == 'enhancement':
if v['totalDupes'] < minNumOfDupes and v[
'totalCC'] < minNumOfCC and (v['priority'] == 'high' or
v['priority'] == 'highest'):
util_add_to_result(lResults,
'change_enhancement_priority_to_medium', k)
elif (v['totalDupes'] >= minNumOfDupes
or v['totalCC'] >= minNumOfCC) and (
v['priority'] != 'high' and v['priority'] != 'highest'):
util_add_to_result(lResults,
'change_enhancement_priority_to_high', k)
else:
if v['totalDupes'] >= minNumOfDupes or v['totalCC'] >= minNumOfCC:
if v['isRegression'] and v['priority'] != 'highest':
util_add_to_result(lResults,
'change_bug_priority_to_highest', k)
elif (v['priority'] != 'high' and v['priority'] != 'highest'):
util_add_to_result(lResults, 'change_bug_priority_to_high',
k)
fp = open(bugzillaReportPath, 'w', encoding='utf-8')
print("Creating file " + bugzillaReportPath)
for dKey, dValue in sorted(lResults.items()):
if dValue:
print('\n=== ' + dKey.replace('_', ' ').upper() + ' ( ' +
str(len(dValue)) + ' ) ===',
file=fp)
sortedValues = sorted([str(item) for item in dValue])
if len(sortedValues) == 1:
print('\t1. ' + common.urlShowBug + sortedValues[0], file=fp)
else:
count = 1
for i in range(0, len(sortedValues), 20):
subList = sortedValues[i:i + 20]
url = "https://bugs.documentfoundation.org/buglist.cgi?bug_id=" + ','.join(
subList)
print('\t' + str(count) + '. ' + url, file=fp)
count += 1
fp.close()
fp = open(bugzillaUserReportPath, 'w', encoding='utf-8')
print("Creating file " + bugzillaUserReportPath)
for k, v in statList['people'].items():
if not statList['people'][k]['name']:
statList['people'][k]['name'] = statList['people'][k][
'email'].split('@')[0]
if statList['people'][k]['oldest'] >= cfg['newUserPeriod'] and len(statList['people'][k]['bugs']) >= newUserBugs and \
statList['people'][k]['email'] not in cfg['configQA']['ignore']['newContributors']:
print('\n=== New contributor: ' + statList['people'][k]['name'] +
" (" + statList['people'][k]['email'] + ") ===",
file=fp)
lBugs = list(statList['people'][k]['bugs'])
for idx in range(len(lBugs)):
isEasyHack = False
if 'easyHack' in bugzillaData['bugs'][str(
lBugs[idx])]['keywords']:
isEasyHack = True
print("{:<3} | {:<58} | {}".format(
str(idx + 1), common.urlShowBug + str(lBugs[idx]),
'easyHack: ' + str(isEasyHack)),
file=fp)
cfg['configQA']['ignore']['newContributors'].append(
statList['people'][k]['email'])
if statList['people'][k]['oldest'] >= cfg['memberPeriod'] and \
len(statList['people'][k]['bugs']) >= memberBugs and statList['people'][k]['email'] not in cfg['configQA']['ignore']['members']:
print('\n=== New MEMBER: ' + statList['people'][k]['name'] + " (" +
statList['people'][k]['email'] + ") ===",
file=fp)
print('\tOldest: ' +
statList['people'][k]['oldest'].strftime("%Y-%m-%d"),
file=fp)
print('\tNewest: ' +
statList['people'][k]['newest'].strftime("%Y-%m-%d"),
file=fp)
print('\tTotal: ' + str(len(statList['people'][k]['bugs'])),
file=fp)
cfg['configQA']['ignore']['members'].append(
statList['people'][k]['email'])
if statList['people'][k]['newest'] < cfg['oldUserPeriod'] and statList['people'][k]['newest'] >= cfg['oldUserPeriod2'] and \
len(statList['people'][k]['bugs']) >= oldUserBugs and statList['people'][k]['email'] not in cfg['configQA']['ignore']['oldContributors']:
print('\n=== Old Contributor: ' + statList['people'][k]['name'] +
" (" + statList['people'][k]['email'] + ") ===",
file=fp)
print('\tOldest: ' +
statList['people'][k]['oldest'].strftime("%Y-%m-%d"),
file=fp)
print('\tNewest: ' +
statList['people'][k]['newest'].strftime("%Y-%m-%d"),
file=fp)
print('\tTotal: ' + str(len(statList['people'][k]['bugs'])),
file=fp)
cfg['configQA']['ignore']['oldContributors'].append(
statList['people'][k]['email'])
statList['people'][k]['oldest'] = statList['people'][k][
'oldest'].strftime("%Y-%m-%d")
statList['people'][k]['newest'] = statList['people'][k][
'newest'].strftime("%Y-%m-%d")
fp.close()
dumpResult = {}
dumpResult['configQA'] = cfg['configQA']
common.util_dump_config(dumpResult)
def analyze_bugzilla_data(statList, bugzillaData, cfg):
print("Analyzing bugzilla\n", end="", flush=True)
unconfirmedCountPerDay = {}
regressionsCountPerDay = {}
bibisectRequestCountPerDay = {}
highestCountPerDay = {}
highCountPerDay = {}
fixedBugs = {}
for key, row in bugzillaData['bugs'].items():
rowId = row['id']
#Ignore META bugs and deletionrequest bugs.
if not row['summary'].lower().startswith(
'[meta]') and row['component'].lower() != 'deletionrequest':
creationDate = datetime.strptime(row['creation_time'],
"%Y-%m-%dT%H:%M:%SZ")
#Some old bugs were directly created as NEW, skipping the UNCONFIRMED status
#Use the oldest bug ID in the unconfirmed list
if rowId >= 89589:
actionDay = creationDate.strftime("%Y-%m-%d")
if actionDay not in unconfirmedCountPerDay:
unconfirmedCountPerDay[actionDay] = 0
unconfirmedCountPerDay[actionDay] += 1
rowStatus = row['status']
rowResolution = row['resolution']
rowKeywords = row['keywords']
creatorMail = row['creator']
#get information about created bugs in the period of time
if common.util_check_range_time(creationDate, cfg):
creationDay = str(creationDate.strftime("%Y-%m-%d"))
util_increase_action(statList['created'], rowId, creatorMail,
creationDay)
if row['severity'] == 'enhancement':
if 'enhancement' not in statList['created']:
statList['created']['enhancement'] = 0
statList['created']['enhancement'] += 1
common.util_check_bugzilla_mail(statList, creatorMail,
row['creator_detail']['real_name'],
creationDate, rowId)
isFixed = False
isWFM = False
isDuplicate = False
isResolved = False
isConfirmed = False
isVerified = False
dayConfirmed = None
dayVerified = None
dayWFM = None
dayDuplicate = None
authorConfirmed = None
authorVerified = None
authorWFM = None
authorDuplicate = None
isRegression = False
isRegressionClosed = False
isBibisectRequest = False
isBibisectRequestClosed = False
isHighest = False
isHighestClosed = False
isHigh = False
isHighClosed = False
isThisBugClosed = False
for action in row['history']:
actionMail = action['who']
actionDate = datetime.strptime(action['when'],
"%Y-%m-%dT%H:%M:%SZ")
common.util_check_bugzilla_mail(statList, actionMail, '',
actionDate, rowId)
actionDay = str(actionDate.strftime("%Y-%m-%d"))
diffTime = (actionDate - creationDate).days
for change in action['changes']:
if change['field_name'] == 'priority':
addedPriority = change['added']
removedPriority = change['removed']
# Sometimes the priority is increased to highest after the bug is fixed
# Ignore those cases
if not isThisBugClosed and not isHighestClosed:
if not isHighest and addedPriority == "highest":
if actionDay not in highestCountPerDay:
highestCountPerDay[actionDay] = 0
highestCountPerDay[actionDay] += 1
isHighest = True
if isHighest and removedPriority == "highest":
if actionDay not in highestCountPerDay:
highestCountPerDay[actionDay] = 0
highestCountPerDay[actionDay] -= 1
isHighest = False
# TODO: IsThisBugClosed should be check here, but the result is not accurate
if not isHighClosed:
if not isHigh and addedPriority == "high":
if actionDay not in highCountPerDay:
highCountPerDay[actionDay] = 0
highCountPerDay[actionDay] += 1
isHigh = True
if isHigh and removedPriority == "high":
if actionDay not in highCountPerDay:
highCountPerDay[actionDay] = 0
highCountPerDay[actionDay] -= 1
isHigh = False
if change['field_name'] == 'status':
addedStatus = change['added']
removedStatus = change['removed']
if common.isOpen(addedStatus):
isThisBugClosed = False
else:
isThisBugClosed = True
#See above
if rowId >= 89589:
if removedStatus == "UNCONFIRMED":
if actionDay not in unconfirmedCountPerDay:
unconfirmedCountPerDay[actionDay] = 0
unconfirmedCountPerDay[actionDay] -= 1
elif addedStatus == 'UNCONFIRMED':
if actionDay not in unconfirmedCountPerDay:
unconfirmedCountPerDay[actionDay] = 0
unconfirmedCountPerDay[actionDay] += 1
if isRegression:
# the regression is being reopened
if isRegressionClosed and not isThisBugClosed:
if actionDay not in regressionsCountPerDay:
regressionsCountPerDay[actionDay] = 0
regressionsCountPerDay[actionDay] += 1
isRegressionClosed = False
# the regression is being closed
if not isRegressionClosed and isThisBugClosed:
if actionDay not in regressionsCountPerDay:
regressionsCountPerDay[actionDay] = 0
regressionsCountPerDay[actionDay] -= 1
isRegressionClosed = True
if isBibisectRequest:
# the bibisectRequest is being reopened
if isBibisectRequestClosed and not isThisBugClosed:
if actionDay not in bibisectRequestCountPerDay:
bibisectRequestCountPerDay[actionDay] = 0
bibisectRequestCountPerDay[actionDay] += 1
isBibisectRequestClosed = False
# the bibisectRequest is being closed
if not isBibisectRequestClosed and isThisBugClosed:
if actionDay not in bibisectRequestCountPerDay:
bibisectRequestCountPerDay[actionDay] = 0
bibisectRequestCountPerDay[actionDay] -= 1
isBibisectRequestClosed = True
if isHighest:
# the Highest priority bug is being reopened
if isHighestClosed and not isThisBugClosed:
if actionDay not in highestCountPerDay:
highestCountPerDay[actionDay] = 0
highestCountPerDay[actionDay] += 1
isHighestClosed = False
# the Highest priority bug is being closed
if not isHighestClosed and isThisBugClosed:
if actionDay not in highestCountPerDay:
highestCountPerDay[actionDay] = 0
highestCountPerDay[actionDay] -= 1
isHighestClosed = True
if isHigh:
# the High priority bug is being reopened
if isHighClosed and not isThisBugClosed:
if actionDay not in highCountPerDay:
highCountPerDay[actionDay] = 0
highCountPerDay[actionDay] += 1
isHighClosed = False
# the High priority bug is being closed
if not isHighClosed and isThisBugClosed:
if actionDay not in highCountPerDay:
highCountPerDay[actionDay] = 0
highCountPerDay[actionDay] -= 1
isHighClosed = True
if common.util_check_range_time(actionDate, cfg):
if removedStatus == "UNCONFIRMED":
util_increase_action(statList['confirmed'],
rowId, actionMail,
actionDay, diffTime)
dayConfirmed = actionDay
authorConfirmed = actionMail
isConfirmed = True
elif addedStatus == 'UNCONFIRMED' and isConfirmed:
util_decrease_action(statList['confirmed'],
authorConfirmed,
dayConfirmed)
isConfirmed = False
if addedStatus == 'VERIFIED':
util_increase_action(statList['verified'],
rowId, actionMail,
actionDay, diffTime)
dayVerified = actionDay
authorVerified = actionMail
isVerified = True
elif removedStatus == 'VERIFIED' and isVerified and common.isOpen(
addedStatus):
util_decrease_action(statList['verified'],
authorVerified,
dayVerified)
isVerified = False
elif change['field_name'] == 'resolution':
if common.util_check_range_time(actionDate, cfg):
addedResolution = change['added']
removedResolution = change['removed']
if isResolved and removedResolution:
statList['resolvedStatuses'][
removedResolution] -= 1
isResolved = False
if addedResolution:
if addedResolution not in statList[
'resolvedStatuses']:
statList['resolvedStatuses'][
addedResolution] = 0
statList['resolvedStatuses'][
addedResolution] += 1
isResolved = True
if addedResolution == 'FIXED':
fixedBugs[rowId] = actionDate
isFixed = True
elif removedResolution == 'FIXED' and isFixed:
del fixedBugs[rowId]
isFixed = False
if addedResolution == 'WORKSFORME':
isWFM = True
dayWFM = actionDay
authorWFM = actionMail
util_increase_action(statList['wfm'], rowId,
actionMail, actionDay,
diffTime)
elif removedResolution == 'WORKSFORME' and isWFM:
util_decrease_action(statList['wfm'],
authorWFM, dayWFM)
isWFM = False
if addedResolution == 'DUPLICATE':
isDuplicate = True
dayDuplicate = actionDay
authorDuplicate = actionMail
util_increase_action(statList['duplicate'],
rowId, actionMail,
actionDay, diffTime)
elif removedResolution == 'DUPLICATE' and isDuplicate:
util_decrease_action(statList['duplicate'],
authorDuplicate,
dayDuplicate)
isDuplicate = False
elif change['field_name'] == 'keywords':
keywordsAdded = change['added'].lower().split(", ")
keywordsRemoved = change['removed'].lower().split(", ")
if common.util_check_range_time(actionDate, cfg):
for keyword in keywordsAdded:
if keyword in lKeywords:
util_increase_action(
statList['keywords'][keyword], rowId,
actionMail, actionDay, diffTime)
# TODO: IsThisBugClosed should be check here, but the result is not accurate
if not isRegressionClosed:
if not isRegression and 'regression' in keywordsAdded:
if actionDay not in regressionsCountPerDay:
regressionsCountPerDay[actionDay] = 0
regressionsCountPerDay[actionDay] += 1
isRegression = True
if isRegression and 'regression' in keywordsRemoved:
if actionDay not in regressionsCountPerDay:
regressionsCountPerDay[actionDay] = 0
regressionsCountPerDay[actionDay] -= 1
isRegression = False
# In the past, 'bibisectRequest' was added after the bug got fixed
# to find the commit fixing it. Ignore them
if not isThisBugClosed and not isBibisectRequestClosed:
if not isBibisectRequest and 'bibisectrequest' in keywordsAdded:
if actionDay not in bibisectRequestCountPerDay:
bibisectRequestCountPerDay[actionDay] = 0
bibisectRequestCountPerDay[actionDay] += 1
isBibisectRequest = True
if isBibisectRequest and 'bibisectrequest' in keywordsRemoved:
if actionDay not in bibisectRequestCountPerDay:
bibisectRequestCountPerDay[actionDay] = 0
bibisectRequestCountPerDay[actionDay] -= 1
isBibisectRequest = False
elif change['field_name'] == 'blocks':
if common.util_check_range_time(actionDate, cfg):
if change['added']:
for metabug in change['added'].split(', '):
if int(metabug) in row['blocks']:
util_increase_action(
statList['metabug'], rowId,
actionMail, actionDay, diffTime)
commentMail = None
comments = row['comments'][1:]
bugFixers = []
commitNoticiation = False
for idx, comment in enumerate(comments):
commentMail = comment['creator']
commentDate = datetime.strptime(comment['time'],
"%Y-%m-%dT%H:%M:%SZ")
common.util_check_bugzilla_mail(statList, commentMail, '',
commentDate, rowId)
if common.util_check_range_time(commentDate,
cfg) and rowId in fixedBugs:
if commentMail == "*****@*****.**":
commentText = comment['text']
author = commentText.split(
' committed a patch related')[0]
if author not in bugFixers and 'uitest' not in commentText.lower() and\
'unittest' not in commentText.lower():
bugFixers.append(author)
diffTime = (commentDate - creationDate).days
commentDay = commentDate.strftime("%Y-%m-%d")
util_increase_action(statList['fixed'], rowId,
author, commentDay, diffTime)
commitNoticiation = True
if row['priority'] == "highest":
statList['criticalFixed'][rowId] = {
'summary': row['summary'],
'author': author
}
if row['priority'] == "high":
statList['highFixed'][rowId] = {
'summary': row['summary'],
'author': author
}
if 'crash' in row['summary'].lower():
statList['crashFixed'][rowId] = {
'summary': row['summary'],
'author': author
}
if 'perf' in row['keywords']:
statList['perfFixed'][rowId] = {
'summary': row['summary'],
'author': author
}
if creationDate < common.util_convert_days_to_datetime(
oldBugsYears * 365):
statList['oldBugsFixed'][rowId] = {
'summary': row['summary'],
'author': author
}
if rowId in fixedBugs and not commitNoticiation:
actionDate = fixedBugs[rowId]
actionDay = actionDate.strftime("%Y-%m-%d")
diffTime = (actionDate - creationDate).days
util_increase_action(statList['fixed'], rowId, 'UNKNOWN',
actionDay, diffTime)
for person in row['cc_detail']:
email = person['email']
if commentMail == email or actionMail == email:
common.util_check_bugzilla_mail(statList, email,
person['real_name'])
for k, v in statList['people'].items():
if not statList['people'][k]['name']:
statList['people'][k]['name'] = statList['people'][k][
'email'].split('@')[0]
statList['people'][k]['oldest'] = statList['people'][k][
'oldest'].strftime("%Y-%m-%d")
statList['people'][k]['newest'] = statList['people'][k][
'newest'].strftime("%Y-%m-%d")
for single_date in daterange(cfg):
single_day = single_date.strftime("%Y-%m-%d")
#Fill empty days to be displayed on the charts
for k0, v0 in statList.items():
if k0 == 'keywords':
for k1, v1 in statList['keywords'].items():
if single_day not in statList['keywords'][k1]['day']:
statList['keywords'][k1]['day'][single_day] = 0
else:
if 'day' in statList[k0]:
if single_day not in statList[k0]['day']:
statList[k0]['day'][single_day] = 0
totalCount1 = 0
for k, v in unconfirmedCountPerDay.items():
xDay = datetime.strptime(k, "%Y-%m-%d")
if xDay < single_date:
totalCount1 += v
statList['unconfirmedCount'][single_day] = totalCount1
totalCount2 = 0
for k, v in regressionsCountPerDay.items():
xDay = datetime.strptime(k, "%Y-%m-%d")
if xDay < single_date:
totalCount2 += v
statList['regressionCount'][single_day] = totalCount2
totalCount3 = 0
for k, v in highestCountPerDay.items():
xDay = datetime.strptime(k, "%Y-%m-%d")
if xDay < single_date:
totalCount3 += v
statList['highestCount'][single_day] = totalCount3
totalCount4 = 0
for k, v in highCountPerDay.items():
xDay = datetime.strptime(k, "%Y-%m-%d")
if xDay < single_date:
totalCount4 += v
statList['highCount'][single_day] = totalCount4
totalCount5 = 0
for k, v in bibisectRequestCountPerDay.items():
xDay = datetime.strptime(k, "%Y-%m-%d")
if xDay < single_date:
totalCount5 += v
statList['bibisectRequestCount'][single_day] = totalCount5