def get_valid_labels(self):
# use the repo wrapper to enable caching+updating
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
return vlabels
def collect_issues(self):
'''Populate the local cache of issues'''
# this should do a few things:
# 1. collect all issues (open+closed) via a webcrawler
# 2. index the issues into a sqllite database so we can query on update times
# 3. set an abstracted object that takes in queries
logging.info('start collecting issues')
logging.debug('creating github connection object')
self.gh = self._connect()
logging.info('creating github connection wrapper')
self.ghw = GithubWrapper(self.gh)
for repo in REPOS:
if repo in self.skiprepo:
continue
#import epdb; epdb.st()
logging.info('getting repo obj for %s' % repo)
cachedir = os.path.join(self.cachedir, repo)
self.repos[repo] = {}
self.repos[repo]['repo'] = self.ghw.get_repo(repo, verbose=False)
self.repos[repo]['issues'] = {}
logging.info('getting issue objs for %s' % repo)
issues = self.repos[repo]['repo'].get_issues()
for issue in issues:
iw = IssueWrapper(
repo=self.repos[repo]['repo'],
issue=issue,
cachedir=cachedir
)
self.repos[repo]['issues'][iw.number] = iw
logging.info('getting issue objs for %s complete' % repo)
logging.info('finished collecting issues')
def get_valid_labels(self):
# use the repo wrapper to enable caching+updating
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
return vlabels
def get_valid_labels(self, repo=None):
# use the repo wrapper to enable caching+updating
if not self.ghw:
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
if not repo:
# OLD workflow
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
else:
# v3 workflow
rw = self.ghw.get_repo(repo)
vlabels = []
for vl in rw.get_labels():
vlabels.append(vl.name)
return vlabels
def run(self, useapiwrapper=True):
"""Starts a triage run"""
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
self.process()
else:
last_run = None
now = self.get_current_time()
last_run_file = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
last_run_file += '/ansible/ansible/'
else:
last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo
last_run_file += 'issues/last_run.pickle'
last_run_file = os.path.expanduser(last_run_file)
if os.path.isfile(last_run_file):
try:
with open(last_run_file, 'rb') as f:
last_run = pickle.load(f)
except Exception as e:
print(e)
if last_run and not self.no_since:
self.debug('Getting issues updated/created since %s' % last_run)
issues = self.repo.get_issues(since=last_run)
else:
self.debug('Getting ALL issues')
issues = self.repo.get_issues()
for issue in issues:
self.icount += 1
if self.start_at and issue.number > self.start_at:
continue
if self.is_pr(issue):
continue
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if action_res:
while action_res['REDO']:
issue = self.repo.get_issue(int(issue.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if not action_res:
action_res = {'REDO': False}
# save this run time
with open(last_run_file, 'wb') as f:
pickle.dump(now, f)
class TriageIssues(DefaultTriager):
VALID_COMMANDS = ['needs_info', '!needs_info', 'notabug',
'bot_broken', 'bot_skip',
'wontfix', 'bug_resolved', 'resolved_by_pr',
'needs_contributor', 'duplicate_of']
CLOSURE_COMMANDS = [
'notabug',
'wontfix',
'bug_resolved',
'resolved_by_pr',
'duplicate_of'
]
# re-notify interval for maintainers
RENOTIFY_INTERVAL = 14
# max limit for needs_info notifications
RENOTIFY_EXPIRE = 56
# re-notify interval by this number for features
FEATURE_RENOTIFY_INTERVAL = 60
# the max comments per week before ansibot becomes a "spammer"
MAX_BOT_COMMENTS_PER_WEEK = 5
def run(self, useapiwrapper=True):
"""Starts a triage run"""
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
self.process()
else:
last_run = None
now = self.get_current_time()
last_run_file = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
last_run_file += '/ansible/ansible/'
else:
last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo
last_run_file += 'issues/last_run.pickle'
last_run_file = os.path.expanduser(last_run_file)
if os.path.isfile(last_run_file):
try:
with open(last_run_file, 'rb') as f:
last_run = pickle.load(f)
except Exception as e:
print(e)
if last_run and not self.no_since:
self.debug('Getting issues updated/created since %s' % last_run)
issues = self.repo.get_issues(since=last_run)
else:
self.debug('Getting ALL issues')
issues = self.repo.get_issues()
for issue in issues:
self.icount += 1
if self.start_at and issue.number > self.start_at:
continue
if self.is_pr(issue):
continue
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if action_res:
while action_res['REDO']:
issue = self.repo.get_issue(int(issue.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if not action_res:
action_res = {'REDO': False}
# save this run time
with open(last_run_file, 'wb') as f:
pickle.dump(now, f)
@RateLimited
def process(self, usecache=True):
"""Processes the Issue"""
# basic processing
self._process()
# who maintains this?
maintainers = []
if self.meta.get('component_valid', False):
correct_repo = self.match.get('repository')
if correct_repo != self.github_repo:
self.meta['correct_repo'] = False
else:
self.meta['correct_repo'] = True
maintainers = self.get_module_maintainers()
if not maintainers:
#issue_module_no_maintainer
#import epdb; epdb.st()
pass
###########################################################
# Enumerate Actions
###########################################################
self.keep_current_main_labels()
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.process_history(usecache=usecache)
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.create_actions()
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
print("Module: %s" % self.module)
if self.match:
print("MModule: %s" % self.match['name'])
else:
print("MModule: %s" % self.match)
print("Maintainer(s): %s" \
% ', '.join(self.get_module_maintainers(expand=False)))
print("Submitter: %s" % self.issue.get_submitter())
print("Total Comments: %s" % len(self.issue.current_comments))
self.print_comment_list()
print("Current Labels: %s" % ', '.join(sorted(self.issue.current_labels)))
# invoke the wizard
import pprint; pprint.pprint(self.actions)
action_meta = self.apply_actions()
return action_meta
def create_actions(self):
"""Create actions from the desired label/unlabel/comment actions"""
# do nothing for bot_skip
if self.meta['bot_skip']:
return
if 'bot_broken' in self.issue.desired_labels:
# If the bot is broken, do nothing other than set the broken label
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
self.actions['close'] = False
if not 'bot_broken' in self.issue.current_labels:
self.actions['newlabel'] = ['bot_broken']
return
# add the version labels
self.create_label_version_actions()
if self.issue.desired_state != self.issue.instance.state:
if self.issue.desired_state == 'closed':
# close the issue ...
self.actions['close'] = True
# We only want up to 1 comment when an issue is closed
if 'issue_closure' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_closure']
elif 'issue_deprecated_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_deprecated_module']
else:
self.issue.desired_comments = []
if self.issue.desired_comments:
comment = self.render_comment(
boilerplate=self.issue.desired_comments[0]
)
self.actions['comments'].append(comment)
return
resolved_desired_labels = []
for desired_label in self.issue.desired_labels:
resolved_desired_label = self.issue.resolve_desired_labels(
desired_label
)
if desired_label != resolved_desired_label:
resolved_desired_labels.append(resolved_desired_label)
if (resolved_desired_label not in self.issue.get_current_labels()):
self.issue.add_desired_comment(desired_label)
self.actions['newlabel'].append(resolved_desired_label)
else:
resolved_desired_labels.append(desired_label)
if desired_label not in self.issue.get_current_labels():
self.actions['newlabel'].append(desired_label)
'''
if os.path.exists("templates/" + 'issue_' + desired_label + ".j2"):
self.issue.add_desired_comment('issue_' + desired_label)
'''
for current_label in self.issue.get_current_labels():
if current_label in self.IGNORE_LABELS:
continue
if current_label not in resolved_desired_labels:
self.actions['unlabel'].append(current_label)
# should only make one comment at a time
if len(self.issue.desired_comments) > 1:
if 'issue_wrong_repo' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_wrong_repo']
elif 'issue_invalid_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif 'issue_module_no_maintainer' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_module_no_maintainer']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments = ['issue_needs_info']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' not in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
else:
import epdb; epdb.st()
# Do not comment needs_info if it's not a label
if 'issue_needs_info' in self.issue.desired_comments \
and not 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
# render the comments
for boilerplate in self.issue.desired_comments:
comment = self.render_comment(boilerplate=boilerplate)
self.debug(msg=boilerplate)
self.actions['comments'].append(comment)
# do not re-comment
for idx,comment in enumerate(self.actions['comments']):
if self.issue.current_comments:
if self.issue.current_comments[-1].body == comment:
self.debug(msg="Removing repeat comment from actions")
self.actions['comments'].remove(comment)
#import epdb; epdb.st()
def create_label_version_actions(self):
if not self.ansible_label_version:
return
expected = 'affects_%s' % self.ansible_label_version
if expected not in self.valid_labels:
print("NEED NEW LABEL: %s" % expected)
import epdb; epdb.st()
candidates = [x for x in self.issue.current_labels]
candidates = [x for x in candidates if x.startswith('affects_')]
candidates = [x for x in candidates if x != expected]
if len(candidates) > 0:
#for cand in candidates:
# self.issue.pop_desired_label(name=cand)
#return
pass
else:
if expected not in self.issue.current_labels \
or expected not in self.issue.desired_labels:
self.issue.add_desired_label(name=expected)
#import epdb; epdb.st()
def create_commment_actions(self):
'''Render desired comment templates'''
# should only make one comment at a time
if len(self.issue.desired_comments) > 1:
if 'issue_wrong_repo' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_wrong_repo']
elif 'issue_invalid_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif 'issue_module_no_maintainer' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_module_no_maintainer']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments = ['issue_needs_info']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' not in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
else:
import epdb; epdb.st()
# Do not comment needs_info if it's not a label
if 'issue_needs_info' in self.issue.desired_comments \
and not 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
# render the comments
for boilerplate in self.issue.desired_comments:
comment = self.render_comment(boilerplate=boilerplate)
self.debug(msg=boilerplate)
self.actions['comments'].append(comment)
# do not re-comment
for idx,comment in enumerate(self.actions['comments']):
if self.issue.current_comments:
if self.issue.current_comments[-1].body == comment:
self.debug(msg="Removing repeat comment from actions")
self.actions['comments'].remove(comment)
def create_label_actions(self):
"""Create actions from the desired label/unlabel/comment actions"""
if 'bot_broken' in self.issue.desired_labels:
# If the bot is broken, do nothing other than set the broken label
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
self.actions['close'] = False
if not 'bot_broken' in self.issue.current_labels:
self.actions['newlabel'] = ['bot_broken']
return
if self.meta['bot_skip']:
return
resolved_desired_labels = []
for desired_label in self.issue.desired_labels:
resolved_desired_label = self.issue.resolve_desired_labels(
desired_label
)
if desired_label != resolved_desired_label:
resolved_desired_labels.append(resolved_desired_label)
if (resolved_desired_label not in self.issue.get_current_labels()):
self.issue.add_desired_comment(desired_label)
self.actions['newlabel'].append(resolved_desired_label)
else:
resolved_desired_labels.append(desired_label)
if desired_label not in self.issue.get_current_labels():
self.actions['newlabel'].append(desired_label)
for current_label in self.issue.get_current_labels():
if current_label in self.IGNORE_LABELS:
continue
if current_label not in resolved_desired_labels:
self.actions['unlabel'].append(current_label)
def process_history(self, usecache=True):
'''Steps through all known meta about the issue and decides what to do'''
self.meta.update(self.get_facts())
self.add_desired_labels_by_issue_type()
self.add_desired_labels_by_ansible_version()
self.add_desired_labels_by_namespace()
#################################################
# FINAL LOGIC LOOP
#################################################
if self.meta['bot_broken']:
self.debug(msg='broken bot stanza')
self.issue.add_desired_label('bot_broken')
elif self.meta['bot_skip']:
self.debug(msg='bot skip stanza')
# clear out all actions and do nothing
for k,v in self.actions.iteritems():
if type(v) == list:
self.actions[k] = []
self.actions['close'] = False
elif self.meta['bot_spam']:
self.debug(msg='bot spam stanza')
# clear out all actions and do nothing
for k,v in self.actions.iteritems():
if type(v) == list:
self.actions[k] = []
self.actions['close'] = False
# do we mark this somehow?
self.issue.add_desired_label('bot_broken')
elif self.match and self.match.get('deprecated', False) \
and 'feature_idea' in self.issue.desired_labels:
self.debug(msg='deprecated module stanza')
# Make the deprecated comment
self.issue.desired_comments = ['issue_deprecated_module']
# Close the issue ...
self.issue.set_desired_state('closed')
elif not self.meta['maintainers_known'] and self.meta['valid_module']:
self.debug(msg='unknown maintainer stanza')
self.issue.desired_comments = ['issue_module_no_maintainer']
elif self.meta['maintainer_closure']:
self.debug(msg='maintainer closure stanza')
# Need to close the issue ...
self.issue.set_desired_state('closed')
elif self.meta['new_module_request']:
self.debug(msg='new module request stanza')
self.issue.desired_comments = []
for label in self.issue.current_labels:
if not label in self.issue.desired_labels:
self.issue.desired_labels.append(label)
elif not self.meta['correct_repo']:
self.debug(msg='wrong repo stanza')
self.issue.desired_comments = ['issue_wrong_repo']
self.actions['close'] = True
elif not self.meta['valid_module'] and \
not self.meta['maintainer_command_needsinfo']:
self.debug(msg='invalid module stanza')
self.issue.add_desired_label('needs_info')
if 'issue_invalid_module' not in self.issue.current_bot_comments \
and not 'issue_needs_info' in self.issue.current_bot_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif not self.meta['notification_maintainers'] and \
not self.meta['maintainer_command_needsinfo']:
self.debug(msg='no maintainer stanza')
self.issue.add_desired_label('waiting_on_maintainer')
self.issue.add_desired_comment("issue_module_no_maintainer")
elif self.meta['maintainer_command'] == 'needs_contributor':
# maintainer can't or won't fix this, but would like someone else to
self.debug(msg='maintainer needs contributor stanza')
self.issue.add_desired_label('waiting_on_contributor')
elif self.meta['maintainer_waiting_on']:
self.debug(msg='maintainer wait stanza')
self.issue.add_desired_label('waiting_on_maintainer')
if len(self.issue.current_comments) == 0:
# new issue
if self.meta['issue_type']:
if self.meta['submitter'] not in self.meta['notification_maintainers']:
# ping the maintainer
self.issue.add_desired_comment('issue_new')
else:
# do not send intial ping to maintainer if also submitter
if 'issue_new' in self.issue.desired_comments:
self.issue.desired_comments.remove('issue_new')
else:
# old issue -- renotify
if not self.match['deprecated'] and self.meta['notification_maintainers']:
if self.meta['maintainer_to_ping']:
self.issue.add_desired_comment("issue_notify_maintainer")
elif self.meta['maintainer_to_reping']:
self.issue.add_desired_comment("issue_renotify_maintainer")
elif self.meta['submitter_waiting_on']:
self.debug(msg='submitter wait stanza')
if 'waiting_on_maintainer' in self.issue.desired_labels:
self.issue.desired_labels.remove('waiting_on_maintainer')
if (self.meta['needsinfo_add'] or self.meta['missing_sections']) \
or (not self.meta['needsinfo_remove'] and self.meta['missing_sections']) \
or (self.meta['needsinfo_add'] and not self.meta['missing_sections']):
#import epdb; epdb.st()
self.issue.add_desired_label('needs_info')
if len(self.issue.current_comments) == 0 or \
not self.meta['maintainer_commented']:
if self.issue.current_bot_comments:
if 'issue_needs_info' not in self.issue.current_bot_comments:
self.issue.add_desired_comment("issue_needs_info")
else:
self.issue.add_desired_comment("issue_needs_info")
# needs_info: warn if stale, close if expired
elif self.meta['needsinfo_expired']:
self.issue.add_desired_comment("issue_closure")
self.issue.set_desired_state('closed')
elif self.meta['needsinfo_stale'] \
and (self.meta['submitter_to_ping'] or self.meta['submitter_to_reping']):
self.issue.add_desired_comment("issue_pending_closure")
def get_history_facts(self, usecache=True):
return self.get_facts(usecache=usecache)
def get_facts(self, usecache=True):
'''Only used by the ansible/ansible triager at the moment'''
hfacts = {}
today = self.get_current_time()
self.history = HistoryWrapper(
self.issue,
usecache=usecache,
cachedir=self.cachedir
)
# what was the last commment?
bot_broken = False
if self.issue.current_comments:
for comment in self.issue.current_comments:
if 'bot_broken' in comment.body:
bot_broken = True
# did someone from ansible want this issue skipped?
bot_skip = False
if self.issue.current_comments:
for comment in self.issue.current_comments:
if 'bot_skip' in comment.body and comment.user.login in self.ansible_members:
bot_skip = True
#########################################################
# Facts for new "triage" workflow
#########################################################
# these logins don't count for the next set of facts
exclude_logins = ['ansibot', 'gregdek', self.issue.instance.user.login]
# Has a human (not ansibot + not submitter) ever changed labels?
lchanges = [x for x in self.history.history if x['event'] in ['labeled', 'unlabeled'] and x['actor'] not in exclude_logins]
if lchanges:
hfacts['human_labeled'] = True
else:
hfacts['human_labeled'] = False
# Has a human (not ansibot + not submitter) ever changed the title?
tchanges = [x for x in self.history.history if x['event'] in ['renamed'] and x['actor'] not in exclude_logins]
if tchanges:
hfacts['human_retitled'] = True
else:
hfacts['human_retitled'] = False
# Has a human (not ansibot + not submitter) ever changed the description?
# ... github doesn't record an event for this. Lame.
# Has a human (not ansibot + not submitter) ever set a milestone?
mchanges = [x for x in self.history.history if x['event'] in ['milestoned', 'demilestoned'] and x['actor'] not in exclude_logins]
if mchanges:
hfacts['human_milestoned'] = True
else:
hfacts['human_milestoned'] = False
# Has a human (not ansibot + not submitter) ever set/unset assignees?
achanges = [x for x in self.history.history if x['event'] in ['assigned', 'unassigned'] and x['actor'] not in exclude_logins]
if achanges:
hfacts['human_assigned'] = True
else:
hfacts['human_assigned'] = False
#########################################################
# Has the bot been overzealous with comments?
hfacts['bot_spam'] = False
bcg = self.history.get_user_comments_groupby('ansibot', groupby='w')
for k,v in bcg.iteritems():
if v >= self.MAX_BOT_COMMENTS_PER_WEEK:
hfacts['bot_spam'] = True
# Is this a new module?
hfacts['new_module_request'] = False
if 'feature_idea' in self.issue.desired_labels:
if self.template_data['component name'] == 'new':
hfacts['new_module_request'] = True
# who made this and when did they last comment?
submitter = self.issue.get_submitter()
submitter_last_commented = self.history.last_commented_at(submitter)
if not submitter_last_commented:
submitter_last_commented = self.issue.instance.created_at
#import epdb; epdb.st()
submitter_last_comment = self.history.last_comment(submitter)
submitter_last_notified = self.history.last_notified(submitter)
# what did they not provide?
missing_sections = self.issue.get_missing_sections()
# Is this a valid module?
if self.match:
self.meta['valid_module'] = True
else:
self.meta['valid_module'] = False
# Filed in the right place?
if self.meta['valid_module']:
if self.match['repository'] != self.github_repo:
hfacts['correct_repo'] = False
else:
hfacts['correct_repo'] = True
else:
hfacts['correct_repo'] = True
# DEBUG + FIXME - speeds up bulk triage
if 'component name' in missing_sections \
and (self.match or self.github_repo == 'ansible'):
missing_sections.remove('component name')
#import epdb; epdb.st()
# Who are the maintainers?
maintainers = [x for x in self.get_module_maintainers()]
#hfacts['maintainers'] = maintainers
#import epdb; epdb.st()
# Set a fact to indicate that we know the maintainer
self.meta['maintainers_known'] = False
if maintainers:
self.meta['maintainers_known'] = True
if 'ansible' in maintainers:
maintainers.remove('ansible')
maintainers.extend(self.ansible_members)
if 'ansibot' in maintainers:
maintainers.remove('ansibot')
if submitter in maintainers:
maintainers.remove(submitter)
maintainers = sorted(set(maintainers))
# Has maintainer been notified? When?
notification_maintainers = [x for x in self.get_module_maintainers()]
if 'ansible' in notification_maintainers:
notification_maintainers.extend(self.ansible_members)
if 'ansibot' in notification_maintainers:
notification_maintainers.remove('ansibot')
hfacts['notification_maintainers'] = notification_maintainers
maintainer_last_notified = self.history.\
last_notified(notification_maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
maintainer_last_viewed = self.history.last_viewed_at(maintainers)
#import epdb; epdb.st()
# Has maintainer been mentioned?
maintainer_mentioned = self.history.is_mentioned(maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
# Has the maintainer ever responded?
maintainer_commented = self.history.has_commented(maintainers)
maintainer_last_commented = self.history.last_commented_at(maintainers)
maintainer_last_comment = self.history.last_comment(maintainers)
maintainer_comments = self.history.get_user_comments(maintainers)
#import epdb; epdb.st()
# Was the maintainer the last commentor?
last_commentor_ismaintainer = False
last_commentor_issubmitter = False
last_commentor = self.history.last_commentor()
if last_commentor in maintainers and last_commentor != self.github_user:
last_commentor_ismaintainer = True
elif last_commentor == submitter:
last_commentor_issubmitter = True
# Did the maintainer issue a command?
maintainer_commands = self.history.get_commands(maintainers,
self.VALID_COMMANDS)
# Keep all commands
hfacts['maintainer_commands'] = maintainer_commands
# Set a bit for the last command given
if hfacts['maintainer_commands']:
hfacts['maintainer_command'] = hfacts['maintainer_commands'][-1]
else:
hfacts['maintainer_command'] = None
# Is the last command a closure command?
if hfacts['maintainer_command'] in self.CLOSURE_COMMANDS:
hfacts['maintainer_closure'] = True
else:
hfacts['maintainer_closure'] = False
# handle resolved_by_pr ...
if 'resolved_by_pr' in maintainer_commands:
maintainer_comments = self.history.get_user_comments(maintainers)
maintainer_comments = [x for x in reversed(maintainer_comments) \
if 'resolved_by_pr' in x]
for comment in maintainer_comments:
pr_number = extract_pr_number_from_comment(comment)
hfacts['resolved_by_pr'] = {
'number': pr_number,
'merged': self.is_pr_merged(pr_number),
}
if not hfacts['resolved_by_pr']['merged']:
hfacts['maintainer_closure'] = False
break
# needs_info toggles
ni_commands = [x for x in maintainer_commands if 'needs_info' in x]
# Has the maintainer ever subscribed?
maintainer_subscribed = self.history.has_subscribed(maintainers)
# Was it ever needs_info?
was_needs_info = self.history.was_labeled(label='needs_info')
needsinfo_last_applied = self.history.label_last_applied('needs_info')
needsinfo_last_removed = self.history.label_last_removed('needs_info')
# Still needs_info?
needsinfo_add = False
needsinfo_remove = False
if 'needs_info' in self.issue.current_labels:
if not needsinfo_last_applied or not submitter_last_commented:
import epdb; epdb.st()
if submitter_last_commented > needsinfo_last_applied:
needsinfo_add = False
needsinfo_remove = True
#if 'needs_info' in maintainer_commands and maintainer_last_commented:
if ni_commands and maintainer_last_commented:
if ni_commands[-1] == 'needs_info':
#import epdb; epdb.st()
if submitter_last_commented and maintainer_last_commented:
if submitter_last_commented > maintainer_last_commented:
needsinfo_add = False
needsinfo_remove = True
else:
needsinfo_add = True
needsinfo_remove = False
else:
needsinfo_add = False
needsinfo_remove = True
# Save existing needs_info if not time to remove ...
if 'needs_info' in self.issue.current_labels \
and not needsinfo_add \
and not needsinfo_remove:
needsinfo_add = True
if ni_commands and maintainer_last_commented:
if maintainer_last_commented > submitter_last_commented:
if ni_commands[-1] == 'needs_info':
needsinfo_add = True
needsinfo_remove = False
else:
needsinfo_add = False
needsinfo_remove = True
#import epdb; epdb.st()
# Is needs_info stale or expired?
needsinfo_age = None
needsinfo_stale = False
needsinfo_expired = False
if 'needs_info' in self.issue.current_labels:
time_delta = today - needsinfo_last_applied
needsinfo_age = time_delta.days
if needsinfo_age > self.RENOTIFY_INTERVAL:
needsinfo_stale = True
if needsinfo_age > self.RENOTIFY_EXPIRE:
needsinfo_expired = True
# Should we be in waiting_on_maintainer mode?
maintainer_waiting_on = False
if (needsinfo_remove or not needsinfo_add) \
or not was_needs_info \
and not missing_sections:
maintainer_waiting_on = True
# Should we [re]notify the submitter?
submitter_waiting_on = False
submitter_to_ping = False
submitter_to_reping = False
if not maintainer_waiting_on:
submitter_waiting_on = True
if missing_sections:
submitter_waiting_on = True
maintainer_waiting_on = False
# use [!]needs_info to set final state
if ni_commands:
if ni_commands[-1] == '!needs_info':
submitter_waiting_on = False
maintainer_waiting_on = True
elif ni_commands[-1] == 'needs_info':
submitter_waiting_on = True
maintainer_waiting_on = False
# Time to [re]ping maintainer?
maintainer_to_ping = False
maintainer_to_reping = False
if maintainer_waiting_on:
# if feature idea, extend the notification interval
interval = self.RENOTIFY_INTERVAL
if self.meta.get('issue_type', None) == 'feature idea' \
or 'feature_idea' in self.issue.current_labels:
interval = self.FEATURE_RENOTIFY_INTERVAL
if maintainer_viewed and not maintainer_last_notified:
time_delta = today - maintainer_last_viewed
view_age = time_delta.days
if view_age > interval:
maintainer_to_reping = True
elif maintainer_last_notified:
time_delta = today - maintainer_last_notified
ping_age = time_delta.days
if ping_age > interval:
maintainer_to_reping = True
else:
maintainer_to_ping = True
# Time to [re]ping the submitter?
if submitter_waiting_on:
if submitter_last_notified:
time_delta = today - submitter_last_notified
notification_age = time_delta.days
if notification_age > self.RENOTIFY_INTERVAL:
submitter_to_reping = True
else:
submitter_to_reping = False
submitter_to_ping = False
else:
submitter_to_ping = True
submitter_to_reping = False
# needs_contributor ...
hfacts['needs_contributor'] = False
for command in maintainer_commands:
if command == 'needs_contributor':
hfacts['needs_contributor'] = True
elif command == '!needs_contributor':
hfacts['needs_contributor'] = False
hfacts['bot_broken'] = bot_broken
hfacts['bot_skip'] = bot_skip
hfacts['missing_sections'] = missing_sections
hfacts['was_needsinfo'] = was_needs_info
hfacts['needsinfo_age'] = needsinfo_age
hfacts['needsinfo_stale'] = needsinfo_stale
hfacts['needsinfo_expired'] = needsinfo_expired
hfacts['needsinfo_add'] = needsinfo_add
hfacts['needsinfo_remove'] = needsinfo_remove
hfacts['notification_maintainers'] = self.get_module_maintainers() or 'ansible'
hfacts['maintainer_last_notified'] = maintainer_last_notified
hfacts['maintainer_commented'] = maintainer_commented
hfacts['maintainer_viewed'] = maintainer_viewed
hfacts['maintainer_subscribed'] = maintainer_subscribed
hfacts['maintainer_command_needsinfo'] = 'needs_info' in maintainer_commands
hfacts['maintainer_command_not_needsinfo'] = '!needs_info' in maintainer_commands
hfacts['maintainer_waiting_on'] = maintainer_waiting_on
hfacts['maintainer_to_ping'] = maintainer_to_ping
hfacts['maintainer_to_reping'] = maintainer_to_reping
hfacts['submitter'] = submitter
hfacts['submitter_waiting_on'] = submitter_waiting_on
hfacts['submitter_to_ping'] = submitter_to_ping
hfacts['submitter_to_reping'] = submitter_to_reping
hfacts['last_commentor_ismaintainer'] = last_commentor_ismaintainer
hfacts['last_commentor_issubmitter'] = last_commentor_issubmitter
hfacts['last_commentor'] = last_commentor
return hfacts
class DefaultTriager(object):
BOTLIST = ['gregdek', 'robynbergeron', 'ansibot']
VALID_ISSUE_TYPES = ['bug report', 'feature idea', 'documentation report']
IGNORE_LABELS = [
"aws","azure","cloud",
"feature_pull_request",
"feature_idea",
"bugfix_pull_request",
"bug_report",
"docs_pull_request",
"docs_report",
"in progress",
"docs_pull_request",
"easyfix",
"pending_action",
"gce",
"python3",
"P1","P2","P3","P4",
]
def __init__(self, verbose=None, github_user=None, github_pass=None,
github_token=None, github_repo=None, number=None,
start_at=None, always_pause=False, force=False, safe_force=False,
dry_run=False, no_since=False):
self.verbose = verbose
self.github_user = github_user
self.github_pass = github_pass
self.github_token = github_token
self.github_repo = github_repo
self.number = number
self.start_at = start_at
self.always_pause = always_pause
self.force = force
self.safe_force = safe_force
self.dry_run = dry_run
self.no_since = no_since
self.issue = None
self.maintainers = {}
self.module_maintainers = []
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# set the cache dir
self.cachedir = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
self.cachedir += '/ansible/ansible/'
else:
self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo
self.cachedir += 'issues'
self.cachedir = os.path.expanduser(self.cachedir)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
print("Initializing AnsibleVersionIndexer")
self.version_indexer = AnsibleVersionIndexer()
#import epdb; epdb.st()
print("Initializing ModuleIndexer")
self.module_indexer = ModuleIndexer()
self.module_indexer.get_ansible_modules()
print("Initializing FileIndexer")
self.file_indexer = FileIndexer()
self.file_indexer.get_files()
print("Getting ansible members")
self.ansible_members = self.get_ansible_members()
print("Getting valid labels")
self.valid_labels = self.get_valid_labels()
# processed metadata
self.meta = {}
def _process(self, usecache=True):
'''Do some initial processing of the issue'''
# clear all actions
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# clear module maintainers
self.module_maintainers = []
# print some general info about the Issue to be processed
print("\n")
print("Issue #%s [%s]: %s" % (self.issue.number, self.icount,
(self.issue.instance.title).encode('ascii','ignore')))
print("%s" % self.issue.instance.html_url)
print("Created at %s" % self.issue.instance.created_at)
print("Updated at %s" % self.issue.instance.updated_at)
# get the template data
self.template_data = self.issue.get_template_data()
# was the issue type defined correctly?
issue_type_defined = False
issue_type_valid = False
issue_type = False
if 'issue type' in self.template_data:
issue_type_defined = True
issue_type = self.template_data['issue type']
if issue_type.lower() in self.VALID_ISSUE_TYPES:
issue_type_valid = True
self.meta['issue_type_defined'] = issue_type_defined
self.meta['issue_type_valid'] = issue_type_valid
self.meta['issue_type'] = issue_type
if self.meta['issue_type_valid']:
self.meta['issue_type_label'] = self.issue_type_to_label(issue_type)
else:
self.meta['issue_type_label'] = None
# What is the ansible version?
self.ansible_version = self.get_ansible_version()
if not isinstance(self.debug, bool):
self.debug('version: %s' % self.ansible_version)
self.ansible_label_version = self.get_ansible_version_major_minor()
if not isinstance(self.debug, bool):
self.debug('lversion: %s' % self.ansible_label_version)
# was component specified?
component_defined = 'component name' in self.template_data
self.meta['component_defined'] = component_defined
# extract the component
component = self.template_data.get('component name', None)
# save the real name
if self.github_repo != 'ansible':
self.match = self.module_indexer.find_match(component) or {}
else:
self.match = self.module_indexer.find_match(component, exact=True) or {}
self.module = self.match.get('name', None)
# check if component is a known module
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# smart match modules (only on module repos)
if not component_isvalid and self.github_repo != 'ansible' and not self.match:
#smatch = self.smart_match_module()
if hasattr(self, 'meta'):
self.meta['fuzzy_match_called'] = True
kwargs = dict(
repo=self.github_repo,
title=self.issue.instance.title,
component=self.template_data.get('component name')
)
smatch = self.module_indexer.fuzzy_match(**kwargs)
if self.module_indexer.is_valid(smatch):
self.module = smatch
component = smatch
self.match = self.module_indexer.find_match(smatch)
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# Allow globs for module groups
# https://github.com/ansible/ansible-modules-core/issues/3831
craw = self.template_data.get('component_raw')
if self.module_indexer.is_multi(craw):
self.meta['multiple_components'] = True
# get all of the matches
self.matches = self.module_indexer.multi_match(craw)
if self.matches:
# get maintainers for all of the matches
mmap = {}
for match in self.matches:
key = match['filename']
mmap[key] = self.get_maintainers_by_match(match)
# is there a match that represents all included maintainers?
mtuples = [x[1] for x in mmap.items()]
umtuples = [list(x) for x in set(tuple(x) for x in mtuples)]
all_maintainers = []
for mtup in umtuples:
for x in mtup:
if x not in all_maintainers:
all_maintainers.append(x)
best_match = None
for k,v in mmap.iteritems():
if sorted(set(v)) == sorted(set(all_maintainers)):
best_match = k
break
if best_match:
self.match = self.module_indexer.find_match(best_match)
else:
# there's no good match that would include all maintainers
# just skip multi-module processing for now since the rest
# of the code doesn't know what to do with it.
if not isinstance(self.debug, bool):
self.debug('multi-match maintainers: %s' % umtuples)
#print(craw)
#import epdb; epdb.st()
pass
else:
self.meta['multiple_components'] = False
# set the maintainer(s)
self.module_maintainer = [x for x in self.get_module_maintainers()]
self.meta['module_maintainers'] = self.module_maintainer
# fixme: too many places where the module is set
if self.match:
self.module = self.match['name']
# Helper to fix issue descriptions ...
DF = DescriptionFixer(self.issue, self.module_indexer, self.match)
self.issue.new_description = DF.new_description
def _connect(self):
"""Connects to GitHub's API"""
return Github(login_or_token=self.github_token or self.github_user,
password=self.github_pass)
#return GithubWrapper(
# login_or_token=self.github_token or self.github_user,
# password=self.github_pass
# )
def _get_repo_path(self):
if self.github_repo in ['core', 'extras']:
return "ansible/ansible-modules-%s" % self.github_repo
else:
return "ansible/%s" % self.github_repo
def is_pr(self, issue):
if '/pull/' in issue.html_url:
return True
else:
return False
def is_issue(self, issue):
return not self.is_pr(issue)
@ratecheck()
def get_ansible_members(self):
ansible_members = []
update = False
write_cache = False
now = self.get_current_time()
org = self._connect().get_organization("ansible")
cachedir = self.cachedir
if cachedir.endswith('/issues'):
cachedir = os.path.dirname(cachedir)
cachefile = os.path.join(cachedir, 'members.pickle')
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
if os.path.isfile(cachefile):
with open(cachefile, 'rb') as f:
mdata = pickle.load(f)
ansible_members = mdata[1]
if mdata[0] < org.updated_at:
update = True
else:
update = True
write_cache = True
if update:
members = org.get_members()
ansible_members = [x.login for x in members]
# save the data
if write_cache:
mdata = [now, ansible_members]
with open(cachefile, 'wb') as f:
pickle.dump(mdata, f)
#import epdb; epdb.st()
return ansible_members
@ratecheck()
def get_valid_labels(self):
# use the repo wrapper to enable caching+updating
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
return vlabels
def _get_maintainers(self, usecache=True):
"""Reads all known maintainers from files and their owner namespace"""
if not self.maintainers or not usecache:
for repo in ['core', 'extras']:
f = open(MAINTAINERS_FILES[repo])
for line in f:
owner_space = (line.split(': ')[0]).strip()
maintainers_string = (line.split(': ')[-1]).strip()
self.maintainers[owner_space] = maintainers_string.split(' ')
f.close()
# meta is special
self.maintainers['meta'] = ['ansible']
return self.maintainers
def debug(self, msg=""):
"""Prints debug message if verbosity is given"""
if self.verbose:
print("Debug: " + msg)
def get_ansible_version(self):
aversion = None
rawdata = self.template_data.get('ansible version', '')
if rawdata:
aversion = self.version_indexer.strip_ansible_version(rawdata)
if not aversion or aversion == 'devel':
aversion = self.version_indexer.ansible_version_by_date(self.issue.instance.created_at)
if aversion:
if aversion.endswith('.'):
aversion += '0'
# re-run for versions ending with .x
if aversion:
if aversion.endswith('.x'):
aversion = self.version_indexer.strip_ansible_version(aversion)
#import epdb; epdb.st()
if self.version_indexer.is_valid_version(aversion) and aversion != None:
return aversion
else:
# try to go through the submitter's comments and look for the
# first one that specifies a valid version
cversion = None
for comment in self.issue.current_comments:
if comment.user.login != self.issue.instance.user.login:
continue
xver = self.version_indexer.strip_ansible_version(comment.body)
if self.version_indexer.is_valid_version(xver):
cversion = xver
break
# use the comment version
aversion = cversion
return aversion
def get_ansible_version_major_minor(self):
return self.version_indexer.get_major_minor(self.ansible_version)
def get_maintainers_by_match(self, match):
module_maintainers = []
maintainers = self._get_maintainers()
if match['name'] in maintainers:
module_maintainers = maintainers[match['name']]
elif match['repo_filename'] in maintainers:
module_maintainers = maintainers[match['repo_filename']]
elif (match['deprecated_filename']) in maintainers:
module_maintainers = maintainers[match['deprecated_filename']]
elif match['namespaced_module'] in maintainers:
module_maintainers = maintainers[match['namespaced_module']]
elif match['fulltopic'] in maintainers:
module_maintainers = maintainers[match['fulltopic']]
elif (match['topic'] + '/') in maintainers:
module_maintainers = maintainers[match['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
#import epdb; epdb.st()
return module_maintainers
def get_module_maintainers(self, expand=True, usecache=True):
"""Returns the list of maintainers for the current module"""
# expand=False means don't use cache and don't expand the 'ansible' group
if self.module_maintainers and usecache:
return self.module_maintainers
module_maintainers = []
module = self.module
if not module:
return module_maintainers
if not self.module_indexer.is_valid(module):
return module_maintainers
if self.match:
mdata = self.match
else:
mdata = self.module_indexer.find_match(module)
if mdata['repository'] != self.github_repo:
# this was detected and handled in the process loop
pass
# get cached or non-cached maintainers list
if not expand:
maintainers = self._get_maintainers(usecache=False)
else:
maintainers = self._get_maintainers()
if mdata['name'] in maintainers:
module_maintainers = maintainers[mdata['name']]
elif mdata['repo_filename'] in maintainers:
module_maintainers = maintainers[mdata['repo_filename']]
elif (mdata['deprecated_filename']) in maintainers:
module_maintainers = maintainers[mdata['deprecated_filename']]
elif mdata['namespaced_module'] in maintainers:
module_maintainers = maintainers[mdata['namespaced_module']]
elif mdata['fulltopic'] in maintainers:
module_maintainers = maintainers[mdata['fulltopic']]
elif (mdata['topic'] + '/') in maintainers:
module_maintainers = maintainers[mdata['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
# need to set the no maintainer template or assume ansible?
if not module_maintainers and self.module and self.match:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
return module_maintainers
def get_current_labels(self):
"""Pull the list of labels on this Issue"""
if not self.current_labels:
labels = self.issue.instance.labels
for label in labels:
self.current_labels.append(label.name)
return self.current_labels
def run(self):
pass
def create_actions(self):
pass
def component_from_comments(self):
"""Extracts a component name from special comments"""
# https://github.com/ansible/ansible-modules/core/issues/2618
# comments like: [module: packaging/os/zypper.py] ... ?
component = None
for idx, x in enumerate(self.issue.current_comments):
if '[' in x.body and ']' in x.body and ('module' in x.body or 'component' in x.body or 'plugin' in x.body):
if x.user.login in BOTLIST:
component = x.body.split()[-1]
component = component.replace('[', '')
return component
def has_maintainer_commented(self):
"""Has the maintainer -ever- commented on the issue?"""
commented = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# ignore comments from submitter
if comment.user.login == self.issue.get_submitter():
continue
# "ansible" is special ...
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
commented = True
elif comment.user.login in self.module_maintainers:
commented = True
return commented
def is_maintainer_mentioned(self):
mentioned = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# "ansible" is special ...
if 'ansible' in self.module_maintainers:
for x in self.ansible_members:
if ('@%s' % x) in comment.body:
mentioned = True
break
else:
for x in self.module_maintainers:
if ('@%s' % x) in comment.body:
mentioned = True
break
return mentioned
def get_current_time(self):
#now = datetime.now()
now = datetime.utcnow()
#now = datetime.now(pytz.timezone('US/Pacific'))
#import epdb; epdb.st()
return now
def age_of_last_maintainer_comment(self):
"""How long ago did the maintainer comment?"""
last_comment = None
if self.module_maintainers:
for idx,comment in enumerate(self.issue.current_comments):
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer:
last_comment = comment
break
if not last_comment:
return -1
else:
now = self.get_current_time()
diff = now - last_comment.created_at
age = diff.days
return age
def is_waiting_on_maintainer(self):
"""Is the issue waiting on the maintainer to comment?"""
waiting = False
if self.module_maintainers:
if not self.issue.current_comments:
return True
creator_last_index = -1
maintainer_last_index = -1
for idx,comment in enumerate(self.issue.current_comments):
if comment.user.login == self.issue.get_submitter():
if creator_last_index == -1 or idx < creator_last_index:
creator_last_index = idx
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer and \
(maintainer_last_index == -1 or idx < maintainer_last_index):
maintainer_last_index = idx
if creator_last_index == -1 and maintainer_last_index == -1:
waiting = True
elif creator_last_index == -1 and maintainer_last_index > -1:
waiting = False
elif creator_last_index < maintainer_last_index:
waiting = True
return waiting
def keep_current_main_labels(self):
current_labels = self.issue.get_current_labels()
for current_label in current_labels:
if current_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
self.issue.add_desired_label(name=current_label)
def add_desired_labels_by_issue_type(self, comments=True):
"""Adds labels by defined issue type"""
issue_type = self.template_data.get('issue type', False)
if issue_type is False:
self.issue.add_desired_label('needs_info')
return
if not issue_type.lower() in self.VALID_ISSUE_TYPES:
# special handling for PRs
if self.issue.instance.pull_request:
mel = [x for x in self.issue.current_labels if x in self.MUTUALLY_EXCLUSIVE_LABELS]
if not mel:
# if only adding new files, assume it is a feature
if self.patch_contains_only_new_files():
issue_type = 'feature pull request'
else:
if not isinstance(self.debug, bool):
self.debug('"%s" was not a valid issue type, adding "needs_info"' % issue_type)
self.issue.add_desired_label('needs_info')
return
else:
if not isinstance(self.debug, bool):
self.debug('"%s" was not a valid issue type, adding "needs_info"' % issue_type)
self.issue.add_desired_label('needs_info')
return
desired_label = issue_type.replace(' ', '_')
desired_label = desired_label.lower()
desired_label = desired_label.replace('documentation', 'docs')
# FIXME - shouldn't have to do this
if desired_label == 'test_pull_request':
desired_label = 'test_pull_requests'
#import epdb; epdb.st()
# is there a mutually exclusive label already?
if desired_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
mel = [x for x in self.issue.MUTUALLY_EXCLUSIVE_LABELS \
if x in self.issue.current_labels]
if len(mel) > 0:
return
if desired_label not in self.issue.get_current_labels():
self.issue.add_desired_label(name=desired_label)
if len(self.issue.current_comments) == 0 and comments:
# only set this if no other comments
self.issue.add_desired_comment(boilerplate='issue_new')
def patch_contains_only_new_files(self):
'''Does the PR edit any existing files?'''
oldfiles = False
for x in self.issue.files:
if x.filename.encode('ascii', 'ignore') in self.file_indexer.files:
if not isinstance(self.debug, bool):
self.debug('old file match on %s' % x.filename.encode('ascii', 'ignore'))
oldfiles = True
break
return not oldfiles
def add_desired_labels_by_ansible_version(self):
if not 'ansible version' in self.template_data:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version section")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
if not self.template_data['ansible version']:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version defined")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
def add_desired_labels_by_namespace(self):
"""Adds labels regarding module namespaces"""
SKIPTOPICS = ['network/basics/']
if not self.match:
return False
'''
if 'component name' in self.template_data and self.match:
if self.match['repository'] != self.github_repo:
self.issue.add_desired_comment(boilerplate='issue_wrong_repo')
'''
for key in ['topic', 'subtopic']:
# ignore networking/basics
if self.match[key] and not self.match['fulltopic'] in SKIPTOPICS:
thislabel = self.issue.TOPIC_MAP.\
get(self.match[key], self.match[key])
if thislabel in self.valid_labels:
self.issue.add_desired_label(thislabel)
def render_comment(self, boilerplate=None):
"""Renders templates into comments using the boilerplate as filename"""
maintainers = self.get_module_maintainers(expand=False)
if not maintainers:
maintainers = ['NO_MAINTAINER_FOUND'] #FIXME - why?
submitter = self.issue.get_submitter()
missing_sections = [x for x in self.issue.REQUIRED_SECTIONS \
if not x in self.template_data \
or not self.template_data.get(x)]
if not self.match and missing_sections:
# be lenient on component name for ansible/ansible
if self.github_repo == 'ansible' and 'component name' in missing_sections:
missing_sections.remove('component name')
#if missing_sections:
# import epdb; epdb.st()
issue_type = self.template_data.get('issue type', None)
if issue_type:
issue_type = issue_type.lower()
correct_repo = self.match.get('repository', None)
template = environment.get_template('%s.j2' % boilerplate)
comment = template.render(maintainers=maintainers,
submitter=submitter,
issue_type=issue_type,
correct_repo=correct_repo,
component_name=self.template_data.get('component name', 'NULL'),
missing_sections=missing_sections)
#import epdb; epdb.st()
return comment
def process_comments(self):
""" Processes ISSUE comments for matching criteria to add labels"""
if not self.github_user in self.BOTLIST:
self.BOTLIST.append(self.github_user)
module_maintainers = self.get_module_maintainers()
comments = self.issue.get_comments()
today = datetime.today()
if not isinstance(self.debug, bool):
self.debug(msg="--- START Processing Comments:")
for idc,comment in enumerate(comments):
if comment.user.login in self.BOTLIST:
if not isinstance(self.debug, bool):
self.debug(msg="%s is in botlist: " % comment.user.login)
time_delta = today - comment.created_at
comment_days_old = time_delta.days
if not isinstance(self.debug, bool):
self.debug(msg="Days since last bot comment: %s" % comment_days_old)
if comment_days_old > 14:
labels = self.issue.desired_labels
if 'pending' not in comment.body:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_first_warning")
self.issue.add_desired_comment(
boilerplate="submitter_first_warning"
)
break
if "maintainer_review" not in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_first_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_first_warning"
)
break
# pending in comment.body
else:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_second_warning")
self.issue.add_desired_comment(
boilerplate="submitter_second_warning"
)
break
if "maintainer_review" in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_second_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_second_warning"
)
break
if not isinstance(self.debug, bool):
self.debug(msg="STATUS: no useful state change since last pass"
"( %s )" % comment.user.login)
break
if comment.user.login in module_maintainers \
or comment.user.login.lower() in module_maintainers\
or ('ansible' in module_maintainers and comment.user.login in self.ansible_members):
if not isinstance(self.debug, bool):
self.debug(msg="%s is module maintainer commented on %s." % (comment.user.login, comment.created_at))
if 'needs_info' in comment.body:
if not isinstance(self.debug, bool):
self.debug(msg="...said needs_info!")
self.issue.add_desired_label(name="needs_info")
elif "close_me" in comment.body:
if not isinstance(self.debug, bool): self.debug(msg="...said close_me!")
self.issue.add_desired_label(name="pending_action_close_me")
break
if comment.user.login == self.issue.get_submitter():
if not isinstance(self.debug, bool): self.debug(msg="submitter %s, commented on %s." % (comment.user.login, comment.created_at))
if comment.user.login not in self.BOTLIST and comment.user.login in self.ansible_members:
if not isinstance(self.debug, bool): self.debug(msg="%s is a ansible member" % comment.user.login)
if not isinstance(self.debug, bool): self.debug(msg="--- END Processing Comments")
def issue_type_to_label(self, issue_type):
if issue_type:
issue_type = issue_type.lower()
issue_type = issue_type.replace(' ', '_')
issue_type = issue_type.replace('documentation', 'docs')
return issue_type
def check_safe_match(self):
""" Turn force on or off depending on match characteristics """
safe_match = False
if self.action_count() == 0:
safe_match = True
elif not self.actions['close'] and not self.actions['unlabel']:
if len(self.actions['newlabel']) == 1:
if self.actions['newlabel'][0].startswith('affects_'):
safe_match = True
else:
safe_match = False
if self.module:
if self.module in self.issue.instance.title.lower():
safe_match = True
# be more lenient on re-notifications
if not safe_match:
if not self.actions['close'] and \
not self.actions['unlabel'] and \
not self.actions['newlabel']:
if len(self.actions['comments']) == 1:
if 'still waiting' in self.actions['comments'][0]:
safe_match = True
#import epdb; epdb.st()
if safe_match:
self.force = True
else:
self.force = False
def action_count(self):
""" Return the number of actions that are to be performed """
count = 0
for k,v in self.actions.iteritems():
if k == 'close' and v:
count += 1
elif k != 'close':
count += len(v)
return count
def apply_actions(self):
action_meta = {'REDO': False}
if self.safe_force:
self.check_safe_match()
if self.action_count() > 0:
if self.dry_run:
print("Dry-run specified, skipping execution of actions")
else:
if self.force:
print("Running actions non-interactive as you forced.")
self.execute_actions()
return
cont = raw_input("Take recommended actions (y/N/a/R/T)? ")
if cont in ('a', 'A'):
sys.exit(0)
if cont in ('Y', 'y'):
self.execute_actions()
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
if cont == 'r' or cont == 'R':
action_meta['REDO'] = True
if cont == 'DEBUG':
import epdb; epdb.st()
elif self.always_pause:
print("Skipping, but pause.")
cont = raw_input("Continue (Y/n/a/R/T)? ")
if cont in ('a', 'A', 'n', 'N'):
sys.exit(0)
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
elif cont == 'REDO':
action_meta['REDO'] = True
elif cont == 'DEBUG':
import epdb; epdb.st()
action_meta['REDO'] = True
else:
print("Skipping.")
# let the upper level code redo this issue
return action_meta
def template_wizard(self):
print('################################################')
print(self.issue.new_description)
print('################################################')
cont = raw_input("Apply this new description? (Y/N)")
if cont == 'Y':
self.issue.set_description(self.issue.new_description)
#import epdb; epdb.st()
def execute_actions(self):
"""Turns the actions into API calls"""
#time.sleep(1)
for comment in self.actions['comments']:
#import epdb; epdb.st()
if not isinstance(self.debug, bool):
self.debug(msg="API Call comment: " + comment)
self.issue.add_comment(comment=comment)
if self.actions['close']:
# https://github.com/PyGithub/PyGithub/blob/master/github/Issue.py#L263
self.issue.instance.edit(state='closed')
return
for unlabel in self.actions['unlabel']:
if not isinstance(self.debug, bool):
self.debug(msg="API Call unlabel: " + unlabel)
self.issue.remove_label(label=unlabel)
for newlabel in self.actions['newlabel']:
if not isinstance(self.debug, bool):
self.debug(msg="API Call newlabel: " + newlabel)
self.issue.add_label(label=newlabel)
if 'assign' in self.actions:
for user in self.actions['assign']:
self.issue.assign_user(user)
if 'unassign' in self.actions:
for user in self.actions['unassign']:
self.issue.unassign_user(user)
def smart_match_module(self):
'''Fuzzy matching for modules'''
if hasattr(self, 'meta'):
self.meta['smart_match_module_called'] = True
match = None
known_modules = []
for k,v in self.module_indexer.modules.iteritems():
known_modules.append(v['name'])
title = self.issue.instance.title.lower()
title = title.replace(':', '')
title_matches = [x for x in known_modules if x + ' module' in title]
if not title_matches:
title_matches = [x for x in known_modules if title.startswith(x + ' ')]
if not title_matches:
title_matches = [x for x in known_modules if ' ' + x + ' ' in title]
cmatches = None
if self.template_data.get('component name'):
component = self.template_data.get('component name')
cmatches = [x for x in known_modules if x in component]
cmatches = [x for x in cmatches if not '_' + x in component]
import epdb; epdb.st()
# use title ... ?
if title_matches:
cmatches = [x for x in cmatches if x in title_matches]
if cmatches:
if len(cmatches) >= 1:
match = cmatches[0]
if not match:
if 'docs.ansible.com' in component:
#import epdb; epdb.st()
pass
else:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
if not match:
if len(title_matches) == 1:
match = title_matches[0]
else:
print("module - title matches: %s" % title_matches)
print("module - component matches: %s" % cmatches)
return match
def cache_issue(self, issue):
iid = issue.instance.number
fpath = os.path.join(self.cachedir, str(iid))
if not os.path.isdir(fpath):
os.makedirs(fpath)
fpath = os.path.join(fpath, 'iwrapper.pickle')
with open(fpath, 'wb') as f:
pickle.dump(issue, f)
#import epdb; epdb.st()
def load_cached_issues(self, state='open'):
issues = []
idirs = glob.glob('%s/*' % self.cachedir)
idirs = [x for x in idirs if not x.endswith('.pickle')]
for idir in idirs:
wfile = os.path.join(idir, 'iwrapper.pickle')
if os.path.isfile(wfile):
with open(wfile, 'rb') as f:
wrapper = pickle.load(f)
issues.append(wrapper.instance)
return issues
def wait_for_rate_limit(self):
gh = self._connect()
GithubWrapper.wait_for_rate_limit(githubobj=gh)
@ratecheck()
def is_pr_merged(self, number):
'''Check if a PR# has been merged or not'''
merged = False
pr = None
try:
pr = self.repo.get_pullrequest(number)
except Exception as e:
print(e)
if pr:
merged = pr.merged
return merged
def print_comment_list(self):
"""Print comment creators and the commands they used"""
for x in self.issue.current_comments:
command = None
if x.user.login != 'ansibot':
command = [y for y in self.VALID_COMMANDS if y in x.body \
and not '!' + y in x.body]
command = ', '.join(command)
else:
# What template did ansibot use?
try:
command = x.body.split('\n')[-1].split()[-2]
except:
pass
if command:
print("\t%s %s (%s)" % (x.created_at.isoformat(),
x.user.login, command))
else:
print("\t%s %s" % (x.created_at.isoformat(), x.user.login))
def wait_for_rate_limit(self):
gh = self._connect()
GithubWrapper.wait_for_rate_limit(githubobj=gh)
class DefaultTriager(object):
BOTLIST = ['gregdek', 'robynbergeron', 'ansibot']
VALID_ISSUE_TYPES = ['bug report', 'feature idea', 'documentation report']
IGNORE_LABELS = [
"aws",
"azure",
"cloud",
"feature_pull_request",
"feature_idea",
"bugfix_pull_request",
"bug_report",
"docs_pull_request",
"docs_report",
"in progress",
"docs_pull_request",
"easyfix",
"pending_action",
"gce",
"python3",
"P1",
"P2",
"P3",
"P4",
]
def __init__(self,
verbose=None,
github_user=None,
github_pass=None,
github_token=None,
github_repo=None,
number=None,
start_at=None,
always_pause=False,
force=False,
safe_force=False,
dry_run=False,
no_since=False):
self.verbose = verbose
self.github_user = github_user
self.github_pass = github_pass
self.github_token = github_token
self.github_repo = github_repo
self.number = number
self.start_at = start_at
self.always_pause = always_pause
self.force = force
self.safe_force = safe_force
self.dry_run = dry_run
self.no_since = no_since
self.issue = None
self.maintainers = {}
self.module_maintainers = []
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# set the cache dir
self.cachedir = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
self.cachedir += '/ansible/ansible/'
else:
self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo
self.cachedir += 'issues'
self.cachedir = os.path.expanduser(self.cachedir)
if not os.path.isdir(self.cachedir):
os.makedirs(self.cachedir)
print("Initializing AnsibleVersionIndexer")
self.version_indexer = AnsibleVersionIndexer()
#import epdb; epdb.st()
print("Initializing ModuleIndexer")
self.module_indexer = ModuleIndexer()
self.module_indexer.get_ansible_modules()
print("Initializing FileIndexer")
self.file_indexer = FileIndexer()
self.file_indexer.get_files()
print("Getting ansible members")
self.ansible_members = self.get_ansible_members()
print("Getting valid labels")
self.valid_labels = self.get_valid_labels()
# processed metadata
self.meta = {}
def _process(self, usecache=True):
'''Do some initial processing of the issue'''
# clear all actions
self.actions = {
'newlabel': [],
'unlabel': [],
'comments': [],
'close': False,
}
# clear module maintainers
self.module_maintainers = []
# print some general info about the Issue to be processed
print("\n")
print("Issue #%s [%s]: %s" %
(self.issue.number, self.icount,
self.issue.instance.title.encode('ascii', 'ignore')))
print("%s" % self.issue.instance.html_url)
print("Created at %s" % self.issue.instance.created_at)
print("Updated at %s" % self.issue.instance.updated_at)
# get the template data
self.template_data = self.issue.get_template_data()
# was the issue type defined correctly?
issue_type_defined = False
issue_type_valid = False
issue_type = False
if 'issue type' in self.template_data:
issue_type_defined = True
issue_type = self.template_data['issue type']
if issue_type.lower() in self.VALID_ISSUE_TYPES:
issue_type_valid = True
self.meta['issue_type_defined'] = issue_type_defined
self.meta['issue_type_valid'] = issue_type_valid
self.meta['issue_type'] = issue_type
if self.meta['issue_type_valid']:
self.meta['issue_type_label'] = self.issue_type_to_label(
issue_type)
else:
self.meta['issue_type_label'] = None
# What is the ansible version?
self.ansible_version = self.get_ansible_version()
if not isinstance(self.debug, bool):
self.debug('version: %s' % self.ansible_version)
self.ansible_label_version = self.get_ansible_version_major_minor()
if not isinstance(self.debug, bool):
self.debug('lversion: %s' % self.ansible_label_version)
# was component specified?
component_defined = 'component name' in self.template_data
self.meta['component_defined'] = component_defined
# extract the component
component = self.template_data.get('component name', None)
# save the real name
if self.github_repo != 'ansible':
self.match = self.module_indexer.find_match(component) or {}
else:
self.match = \
self.module_indexer.find_match(component, exact=True) or {}
self.module = self.match.get('name', None)
# check if component is a known module
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# smart match modules (only on module repos)
if not component_isvalid and \
self.github_repo != 'ansible' and \
not self.match:
if hasattr(self, 'meta'):
self.meta['fuzzy_match_called'] = True
kwargs = dict(repo=self.github_repo,
title=self.issue.instance.title,
component=self.template_data.get('component name'))
smatch = self.module_indexer.fuzzy_match(**kwargs)
if self.module_indexer.is_valid(smatch):
self.module = smatch
component = smatch
self.match = self.module_indexer.find_match(smatch)
component_isvalid = self.module_indexer.is_valid(component)
self.meta['component_valid'] = component_isvalid
# Allow globs for module groups
# https://github.com/ansible/ansible-modules-core/issues/3831
craw = self.template_data.get('component_raw')
if self.module_indexer.is_multi(craw):
self.meta['multiple_components'] = True
# get all of the matches
self.matches = self.module_indexer.multi_match(craw)
if self.matches:
# get maintainers for all of the matches
mmap = {}
for match in self.matches:
key = match['filename']
mmap[key] = self.get_maintainers_by_match(match)
# is there a match that represents all included maintainers?
mtuples = [x[1] for x in mmap.items()]
umtuples = [list(x) for x in set(tuple(x) for x in mtuples)]
all_maintainers = []
for mtup in umtuples:
for x in mtup:
if x not in all_maintainers:
all_maintainers.append(x)
best_match = None
for k, v in mmap.iteritems():
if sorted(set(v)) == sorted(set(all_maintainers)):
best_match = k
break
if best_match:
self.match = self.module_indexer.find_match(best_match)
else:
# there's no good match that would include all maintainers
# just skip multi-module processing for now since the rest
# of the code doesn't know what to do with it.
if not isinstance(self.debug, bool):
self.debug('multi-match maintainers: %s' % umtuples)
#print(craw)
#import epdb; epdb.st()
pass
else:
self.meta['multiple_components'] = False
# set the maintainer(s)
self.module_maintainer = [x for x in self.get_module_maintainers()]
self.meta['module_maintainers'] = self.module_maintainer
# fixme: too many places where the module is set
if self.match:
self.module = self.match['name']
# Helper to fix issue descriptions ...
DF = DescriptionFixer(self.issue, self.module_indexer, self.match)
self.issue.new_description = DF.new_description
@RateLimited
def _connect(self):
"""Connects to GitHub's API"""
if self.github_token != 'False':
return Github(login_or_token=self.github_token)
else:
return Github(login_or_token=self.github_user,
password=self.github_pass)
def _get_repo_path(self):
if self.github_repo in ['core', 'extras']:
return "ansible/ansible-modules-%s" % self.github_repo
else:
return "ansible/%s" % self.github_repo
def is_pr(self, issue):
if '/pull/' in issue.html_url:
return True
else:
return False
def is_issue(self, issue):
return not self.is_pr(issue)
@RateLimited
def get_ansible_members(self):
ansible_members = []
update = False
write_cache = False
now = self.get_current_time()
org = self._connect().get_organization("ansible")
cachedir = self.cachedir
if cachedir.endswith('/issues'):
cachedir = os.path.dirname(cachedir)
cachefile = os.path.join(cachedir, 'members.pickle')
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
if os.path.isfile(cachefile):
with open(cachefile, 'rb') as f:
mdata = pickle.load(f)
ansible_members = mdata[1]
if mdata[0] < org.updated_at:
update = True
else:
update = True
write_cache = True
if update:
members = org.get_members()
ansible_members = [x.login for x in members]
# save the data
if write_cache:
mdata = [now, ansible_members]
with open(cachefile, 'wb') as f:
pickle.dump(mdata, f)
#import epdb; epdb.st()
return ansible_members
@RateLimited
def get_ansible_core_team(self):
teamlist = [
'ansible-commit', 'ansible-community', 'ansible-commit-external'
]
teams = []
ansible_members = []
conn = self._connect()
org = conn.get_organization('ansible')
for x in org.get_teams():
if x.name in teamlist:
teams.append(x)
for x in teams:
for y in x.get_members():
ansible_members.append(y.login)
ansible_members = sorted(set(ansible_members))
return ansible_members
#@RateLimited
def get_valid_labels(self, repo=None):
# use the repo wrapper to enable caching+updating
if not self.ghw:
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
if not repo:
# OLD workflow
self.repo = self.ghw.get_repo(self._get_repo_path())
vlabels = []
for vl in self.repo.get_labels():
vlabels.append(vl.name)
else:
# v3 workflow
rw = self.ghw.get_repo(repo)
vlabels = []
for vl in rw.get_labels():
vlabels.append(vl.name)
return vlabels
def _get_maintainers(self, usecache=True):
"""Reads all known maintainers from files and their owner namespace"""
if not self.maintainers or not usecache:
for repo in ['core', 'extras']:
f = open(MAINTAINERS_FILES[repo])
for line in f:
owner_space = (line.split(': ')[0]).strip()
maintainers_string = (line.split(': ')[-1]).strip()
self.maintainers[owner_space] = \
maintainers_string.split(' ')
f.close()
# meta is special
self.maintainers['meta'] = ['ansible']
return self.maintainers
def debug(self, msg=""):
"""Prints debug message if verbosity is given"""
if self.verbose:
print("Debug: " + msg)
def get_ansible_version(self):
aversion = None
rawdata = self.template_data.get('ansible version', '')
if rawdata:
aversion = self.version_indexer.strip_ansible_version(rawdata)
if not aversion or aversion == 'devel':
aversion = \
self.version_indexer.ansible_version_by_date(
self.issue.instance.created_at
)
if aversion:
if aversion.endswith('.'):
aversion += '0'
# re-run for versions ending with .x
if aversion:
if aversion.endswith('.x'):
aversion = self.version_indexer.strip_ansible_version(aversion)
#import epdb; epdb.st()
if self.version_indexer.is_valid_version(aversion) and \
aversion is not None:
return aversion
else:
# try to go through the submitter's comments and look for the
# first one that specifies a valid version
cversion = None
for comment in self.issue.current_comments:
if comment.user.login != self.issue.instance.user.login:
continue
xver = self.version_indexer.strip_ansible_version(comment.body)
if self.version_indexer.is_valid_version(xver):
cversion = xver
break
# use the comment version
aversion = cversion
return aversion
def get_ansible_version_by_issue(self, issuewrapper):
iw = issuewrapper
aversion = None
rawdata = iw.get_template_data().get('ansible version', '')
if rawdata:
aversion = self.version_indexer.strip_ansible_version(rawdata)
if not aversion or aversion == 'devel':
aversion = self.version_indexer.ansible_version_by_date(
self.issue.instance.created_at)
if aversion:
if aversion.endswith('.'):
aversion += '0'
# re-run for versions ending with .x
if aversion:
if aversion.endswith('.x'):
aversion = self.version_indexer.strip_ansible_version(aversion)
#import epdb; epdb.st()
if self.version_indexer.is_valid_version(aversion) and \
aversion is not None:
return aversion
else:
# try to go through the submitter's comments and look for the
# first one that specifies a valid version
cversion = None
for comment in self.issue.current_comments:
if comment.user.login != self.issue.instance.user.login:
continue
xver = self.version_indexer.strip_ansible_version(comment.body)
if self.version_indexer.is_valid_version(xver):
cversion = xver
break
# use the comment version
aversion = cversion
return aversion
def get_ansible_version_major_minor(self, version=None):
if not version:
# old workflow
if not hasattr(self, 'ansible_version'):
logging.debug('breakpoint!')
import epdb
epdb.st()
return self.version_indexer.get_major_minor(self.ansible_version)
else:
# v3 workflow
return self.version_indexer.get_major_minor(version)
def get_maintainers_by_match(self, match):
module_maintainers = []
maintainers = self._get_maintainers()
if match['name'] in maintainers:
module_maintainers = maintainers[match['name']]
elif match['repo_filename'] in maintainers:
module_maintainers = maintainers[match['repo_filename']]
elif (match['deprecated_filename']) in maintainers:
module_maintainers = maintainers[match['deprecated_filename']]
elif match['namespaced_module'] in maintainers:
module_maintainers = maintainers[match['namespaced_module']]
elif match['fulltopic'] in maintainers:
module_maintainers = maintainers[match['fulltopic']]
elif (match['topic'] + '/') in maintainers:
module_maintainers = maintainers[match['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
#import epdb; epdb.st()
return module_maintainers
def get_module_maintainers(self, expand=True, usecache=True):
"""Returns the list of maintainers for the current module"""
# expand=False ... ?
if self.module_maintainers and usecache:
return self.module_maintainers
module_maintainers = []
module = self.module
if not module:
return module_maintainers
if not self.module_indexer.is_valid(module):
return module_maintainers
if self.match:
mdata = self.match
else:
mdata = self.module_indexer.find_match(module)
if mdata['repository'] != self.github_repo:
# this was detected and handled in the process loop
pass
# get cached or non-cached maintainers list
if not expand:
maintainers = self._get_maintainers(usecache=False)
else:
maintainers = self._get_maintainers()
if mdata['name'] in maintainers:
module_maintainers = maintainers[mdata['name']]
elif mdata['repo_filename'] in maintainers:
module_maintainers = maintainers[mdata['repo_filename']]
elif (mdata['deprecated_filename']) in maintainers:
module_maintainers = maintainers[mdata['deprecated_filename']]
elif mdata['namespaced_module'] in maintainers:
module_maintainers = maintainers[mdata['namespaced_module']]
elif mdata['fulltopic'] in maintainers:
module_maintainers = maintainers[mdata['fulltopic']]
elif (mdata['topic'] + '/') in maintainers:
module_maintainers = maintainers[mdata['topic'] + '/']
else:
pass
# Fallback to using the module author(s)
if not module_maintainers and self.match:
if self.match['authors']:
module_maintainers = [x for x in self.match['authors']]
# need to set the no maintainer template or assume ansible?
if not module_maintainers and self.module and self.match:
#import epdb; epdb.st()
pass
#import epdb; epdb.st()
return module_maintainers
def get_current_labels(self):
"""Pull the list of labels on this Issue"""
if not self.current_labels:
labels = self.issue.instance.labels
for label in labels:
self.current_labels.append(label.name)
return self.current_labels
def run(self):
pass
def create_actions(self):
pass
def component_from_comments(self):
"""Extracts a component name from special comments"""
# https://github.com/ansible/ansible-modules/core/issues/2618
# comments like: [module: packaging/os/zypper.py] ... ?
component = None
for idx, x in enumerate(self.issue.current_comments):
if '[' in x.body and \
']' in x.body and \
('module' in x.body or
'component' in x.body or
'plugin' in x.body):
if x.user.login in BOTLIST:
component = x.body.split()[-1]
component = component.replace('[', '')
return component
def has_maintainer_commented(self):
"""Has the maintainer -ever- commented on the issue?"""
commented = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# ignore comments from submitter
if comment.user.login == self.issue.get_submitter():
continue
# "ansible" is special ...
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
commented = True
elif comment.user.login in self.module_maintainers:
commented = True
return commented
def is_maintainer_mentioned(self):
mentioned = False
if self.module_maintainers:
for comment in self.issue.current_comments:
# "ansible" is special ...
if 'ansible' in self.module_maintainers:
for x in self.ansible_members:
if ('@%s' % x) in comment.body:
mentioned = True
break
else:
for x in self.module_maintainers:
if ('@%s' % x) in comment.body:
mentioned = True
break
return mentioned
def get_current_time(self):
#now = datetime.now()
now = datetime.utcnow()
#now = datetime.now(pytz.timezone('US/Pacific'))
#import epdb; epdb.st()
return now
def age_of_last_maintainer_comment(self):
"""How long ago did the maintainer comment?"""
last_comment = None
if self.module_maintainers:
for idx, comment in enumerate(self.issue.current_comments):
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer:
last_comment = comment
break
if not last_comment:
return -1
else:
now = self.get_current_time()
diff = now - last_comment.created_at
age = diff.days
return age
def is_waiting_on_maintainer(self):
"""Is the issue waiting on the maintainer to comment?"""
waiting = False
if self.module_maintainers:
if not self.issue.current_comments:
return True
creator_last_index = -1
maintainer_last_index = -1
for idx, comment in enumerate(self.issue.current_comments):
if comment.user.login == self.issue.get_submitter():
if creator_last_index == -1 or idx < creator_last_index:
creator_last_index = idx
# "ansible" is special ...
is_maintainer = False
if 'ansible' in self.module_maintainers \
and comment.user.login in self.ansible_members:
is_maintainer = True
elif comment.user.login in self.module_maintainers:
is_maintainer = True
if is_maintainer and \
(maintainer_last_index == -1 or
idx < maintainer_last_index):
maintainer_last_index = idx
if creator_last_index == -1 and maintainer_last_index == -1:
waiting = True
elif creator_last_index == -1 and maintainer_last_index > -1:
waiting = False
elif creator_last_index < maintainer_last_index:
waiting = True
return waiting
def keep_current_main_labels(self):
current_labels = self.issue.get_current_labels()
for current_label in current_labels:
if current_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
self.issue.add_desired_label(name=current_label)
def add_desired_labels_by_issue_type(self, comments=True):
"""Adds labels by defined issue type"""
issue_type = self.template_data.get('issue type', False)
if issue_type is False:
self.issue.add_desired_label('needs_info')
return
if not issue_type.lower() in self.VALID_ISSUE_TYPES:
# special handling for PRs
if self.issue.instance.pull_request:
mel = [
x for x in self.issue.current_labels
if x in self.MUTUALLY_EXCLUSIVE_LABELS
]
if not mel:
# if only adding new files, assume it is a feature
if self.patch_contains_only_new_files():
issue_type = 'feature pull request'
else:
if not isinstance(self.debug, bool):
msg = '"%s"' % issue_type
msg += ' was not a valid issue type'
msg += ', adding "needs_info"'
self.debug(msg)
self.issue.add_desired_label('needs_info')
return
else:
if not isinstance(self.debug, bool):
msg = '"%s"' % issue_type
msg += ' was not a valid issue type'
msg += ', adding "needs_info"'
self.debug(msg)
self.issue.add_desired_label('needs_info')
return
desired_label = issue_type.replace(' ', '_')
desired_label = desired_label.lower()
desired_label = desired_label.replace('documentation', 'docs')
# FIXME - shouldn't have to do this
if desired_label == 'test_pull_request':
desired_label = 'test_pull_requests'
#import epdb; epdb.st()
# is there a mutually exclusive label already?
if desired_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS:
mel = [
x for x in self.issue.MUTUALLY_EXCLUSIVE_LABELS
if x in self.issue.current_labels
]
if len(mel) > 0:
return
if desired_label not in self.issue.get_current_labels():
self.issue.add_desired_label(name=desired_label)
if len(self.issue.current_comments) == 0 and comments:
# only set this if no other comments
self.issue.add_desired_comment(boilerplate='issue_new')
def patch_contains_only_new_files(self):
'''Does the PR edit any existing files?'''
oldfiles = False
for x in self.issue.files:
if x.filename.encode('ascii', 'ignore') in self.file_indexer.files:
if not isinstance(self.debug, bool):
msg = 'old file match on'
msg += ' %s' % x.filename.encode('ascii', 'ignore')
self.debug(msg)
oldfiles = True
break
return not oldfiles
def add_desired_labels_by_ansible_version(self):
if 'ansible version' not in self.template_data:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version section")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
if not self.template_data['ansible version']:
if not isinstance(self.debug, bool):
self.debug(msg="no ansible version defined")
self.issue.add_desired_label(name="needs_info")
#self.issue.add_desired_comment(
# boilerplate="issue_missing_data"
#)
return
def add_desired_labels_by_namespace(self):
"""Adds labels regarding module namespaces"""
SKIPTOPICS = ['network/basics/']
if not self.match:
return False
'''
if 'component name' in self.template_data and self.match:
if self.match['repository'] != self.github_repo:
self.issue.add_desired_comment(boilerplate='issue_wrong_repo')
'''
for key in ['topic', 'subtopic']:
# ignore networking/basics
if self.match[key] and not self.match['fulltopic'] in SKIPTOPICS:
thislabel = self.issue.TOPIC_MAP.\
get(self.match[key], self.match[key])
if thislabel in self.valid_labels:
self.issue.add_desired_label(thislabel)
def render_boilerplate(self, tvars, boilerplate=None):
template = environment.get_template('%s.j2' % boilerplate)
comment = template.render(**tvars)
return comment
def render_comment(self, boilerplate=None):
"""Renders templates into comments using the boilerplate as filename"""
maintainers = self.get_module_maintainers(expand=False)
if not maintainers:
# FIXME - why?
maintainers = ['NO_MAINTAINER_FOUND']
submitter = self.issue.get_submitter()
missing_sections = [
x for x in self.issue.REQUIRED_SECTIONS
if x not in self.template_data or not self.template_data.get(x)
]
if not self.match and missing_sections:
# be lenient on component name for ansible/ansible
if self.github_repo == 'ansible' and \
'component name' in missing_sections:
missing_sections.remove('component name')
#if missing_sections:
# import epdb; epdb.st()
issue_type = self.template_data.get('issue type', None)
if issue_type:
issue_type = issue_type.lower()
correct_repo = self.match.get('repository', None)
template = environment.get_template('%s.j2' % boilerplate)
component_name = self.template_data.get('component name', 'NULL'),
comment = template.render(maintainers=maintainers,
submitter=submitter,
issue_type=issue_type,
correct_repo=correct_repo,
component_name=component_name,
missing_sections=missing_sections)
return comment
def process_comments(self):
""" Processes ISSUE comments for matching criteria to add labels"""
if self.github_user not in self.BOTLIST:
self.BOTLIST.append(self.github_user)
module_maintainers = self.get_module_maintainers()
comments = self.issue.get_comments()
today = datetime.today()
if not isinstance(self.debug, bool):
self.debug(msg="--- START Processing Comments:")
for idc, comment in enumerate(comments):
if comment.user.login in self.BOTLIST:
if not isinstance(self.debug, bool):
self.debug(msg="%s is in botlist: " % comment.user.login)
time_delta = today - comment.created_at
comment_days_old = time_delta.days
if not isinstance(self.debug, bool):
msg = "Days since last bot comment: %s" % comment_days_old
self.debug(msg=msg)
if comment_days_old > 14:
labels = self.issue.desired_labels
if 'pending' not in comment.body:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_first_warning")
self.issue.add_desired_comment(
boilerplate="submitter_first_warning")
break
if "maintainer_review" not in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_first_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_first_warning")
break
# pending in comment.body
else:
if self.issue.is_labeled_for_interaction():
if not isinstance(self.debug, bool):
self.debug(msg="submitter_second_warning")
self.issue.add_desired_comment(
boilerplate="submitter_second_warning")
break
if "maintainer_review" in labels:
if not isinstance(self.debug, bool):
self.debug(msg="maintainer_second_warning")
self.issue.add_desired_comment(
boilerplate="maintainer_second_warning")
break
if not isinstance(self.debug, bool):
msg = "STATUS: no useful state change since last pass"
msg += "( %s )" % comment.user.login
self.debug(msg=msg)
break
if comment.user.login in module_maintainers \
or comment.user.login.lower() in module_maintainers\
or ('ansible' in module_maintainers and
comment.user.login in self.ansible_members):
if not isinstance(self.debug, bool):
msg = "%s" % comment.user.login
msg = " is module maintainer commented on"
msg += "%s." % comment.created_at
self.debug(msg=msg)
if 'needs_info' in comment.body:
if not isinstance(self.debug, bool):
self.debug(msg="...said needs_info!")
self.issue.add_desired_label(name="needs_info")
elif "close_me" in comment.body:
if not isinstance(self.debug, bool):
self.debug(msg="...said close_me!")
self.issue.add_desired_label(
name="pending_action_close_me")
break
if comment.user.login == self.issue.get_submitter():
if not isinstance(self.debug, bool):
msg = "submitter %s" % comment.user.login
msg += ", commented on %s." % comment.created_at
self.debug(msg=msg)
if comment.user.login not in self.BOTLIST and \
comment.user.login in self.ansible_members:
if not isinstance(self.debug, bool):
self.debug(msg="%s is a ansible member" %
comment.user.login)
if not isinstance(self.debug, bool):
self.debug(msg="--- END Processing Comments")
def issue_type_to_label(self, issue_type):
if issue_type:
issue_type = issue_type.lower()
issue_type = issue_type.replace(' ', '_')
issue_type = issue_type.replace('documentation', 'docs')
return issue_type
def check_safe_match(self):
""" Turn force on or off depending on match characteristics """
import epdb
epdb.st()
safe_match = False
if self.action_count() == 0:
safe_match = True
elif not self.actions['close'] and not self.actions['unlabel']:
if len(self.actions['newlabel']) == 1:
if self.actions['newlabel'][0].startswith('affects_'):
safe_match = True
else:
safe_match = False
if self.module:
if self.module in self.issue.instance.title.lower():
safe_match = True
# be more lenient on re-notifications
if not safe_match:
if not self.actions['close'] and \
not self.actions['unlabel'] and \
not self.actions['newlabel']:
if len(self.actions['comments']) == 1:
if 'still waiting' in self.actions['comments'][0]:
safe_match = True
#import epdb; epdb.st()
if safe_match:
self.force = True
else:
self.force = False
def action_count(self):
""" Return the number of actions that are to be performed """
count = 0
for k, v in self.actions.iteritems():
if k in ['close', 'open', 'merge', 'close_migrated'] and v:
count += 1
elif k != 'close' and k != 'open' and \
k != 'merge' and k != 'close_migrated':
count += len(v)
return count
def apply_actions(self):
action_meta = {'REDO': False}
if self.safe_force:
self.check_safe_match()
if self.action_count() > 0:
if self.dry_run:
print("Dry-run specified, skipping execution of actions")
else:
if self.force:
print("Running actions non-interactive as you forced.")
self.execute_actions()
return action_meta
cont = raw_input(
"Take recommended actions (y/N/a/R/T/DEBUG)? ")
if cont in ('a', 'A'):
sys.exit(0)
if cont in ('Y', 'y'):
self.execute_actions()
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
if cont == 'r' or cont == 'R':
action_meta['REDO'] = True
if cont == 'DEBUG':
# put the user into a breakpoint to do live debug
action_meta['REDO'] = True
import epdb
epdb.st()
elif self.always_pause:
print("Skipping, but pause.")
cont = raw_input("Continue (Y/n/a/R/T/DEBUG)? ")
if cont in ('a', 'A', 'n', 'N'):
sys.exit(0)
if cont == 'T':
self.template_wizard()
action_meta['REDO'] = True
elif cont == 'REDO':
action_meta['REDO'] = True
elif cont == 'DEBUG':
# put the user into a breakpoint to do live debug
import epdb
epdb.st()
action_meta['REDO'] = True
else:
print("Skipping.")
# let the upper level code redo this issue
return action_meta
def template_wizard(self):
DF = DescriptionFixer(self.issue, self.meta)
print('################################################')
print(DF.new_description)
print('################################################')
cont = raw_input("Apply this new description? (Y/N)")
if cont == 'Y':
self.issue.set_description(DF.new_description)
def execute_actions(self):
"""Turns the actions into API calls"""
#time.sleep(1)
for comment in self.actions['comments']:
logging.info("acton: comment - " + comment)
self.issue.add_comment(comment=comment)
if self.actions['close']:
# https://github.com/PyGithub/PyGithub/blob/master/github/Issue.py#L263
logging.info('action: close')
self.issue.instance.edit(state='closed')
return
if self.actions['close_migrated']:
mi = self.get_issue_by_repopath_and_number(
self.meta['migrated_issue_repo_path'],
self.meta['migrated_issue_number'])
logging.info('close migrated: %s' % mi.html_url)
mi.instance.edit(state='closed')
#import epdb; epdb.st()
for unlabel in self.actions['unlabel']:
logging.info('action: unlabel - ' + unlabel)
self.issue.remove_label(label=unlabel)
for newlabel in self.actions['newlabel']:
logging.info('action: label - ' + newlabel)
self.issue.add_label(label=newlabel)
if 'assign' in self.actions:
for user in self.actions['assign']:
logging.info('action: assign - ' + user)
self.issue.assign_user(user)
if 'unassign' in self.actions:
for user in self.actions['unassign']:
logging.info('action: unassign - ' + user)
self.issue.unassign_user(user)
if 'merge' in self.actions:
if self.actions['merge']:
self.issue.merge()
def smart_match_module(self):
'''Fuzzy matching for modules'''
if hasattr(self, 'meta'):
self.meta['smart_match_module_called'] = True
match = None
known_modules = []
for k, v in self.module_indexer.modules.iteritems():
known_modules.append(v['name'])
title = self.issue.instance.title.lower()
title = title.replace(':', '')
title_matches = [x for x in known_modules if x + ' module' in title]
if not title_matches:
title_matches = [
x for x in known_modules if title.startswith(x + ' ')
]
if not title_matches:
title_matches = [
x for x in known_modules if ' ' + x + ' ' in title
]
cmatches = None
if self.template_data.get('component name'):
component = self.template_data.get('component name')
cmatches = [x for x in known_modules if x in component]
cmatches = [x for x in cmatches if not '_' + x in component]
# use title ... ?
if title_matches:
cmatches = [x for x in cmatches if x in title_matches]
if cmatches:
if len(cmatches) >= 1:
match = cmatches[0]
if not match:
if 'docs.ansible.com' in component:
pass
else:
pass
if not match:
if len(title_matches) == 1:
match = title_matches[0]
else:
print("module - title matches: %s" % title_matches)
print("module - component matches: %s" % cmatches)
return match
def cache_issue(self, issue):
iid = issue.instance.number
fpath = os.path.join(self.cachedir, str(iid))
if not os.path.isdir(fpath):
os.makedirs(fpath)
fpath = os.path.join(fpath, 'iwrapper.pickle')
with open(fpath, 'wb') as f:
pickle.dump(issue, f)
#import epdb; epdb.st()
def load_cached_issues(self, state='open'):
issues = []
idirs = glob.glob('%s/*' % self.cachedir)
idirs = [x for x in idirs if not x.endswith('.pickle')]
for idir in idirs:
wfile = os.path.join(idir, 'iwrapper.pickle')
if os.path.isfile(wfile):
with open(wfile, 'rb') as f:
wrapper = pickle.load(f)
issues.append(wrapper.instance)
return issues
def wait_for_rate_limit(self):
gh = self._connect()
GithubWrapper.wait_for_rate_limit(githubobj=gh)
@RateLimited
def is_pr_merged(self, number, repo=None):
'''Check if a PR# has been merged or not'''
merged = False
pr = None
try:
if not repo:
pr = self.repo.get_pullrequest(number)
else:
pr = repo.get_pullrequest(number)
except Exception as e:
print(e)
if pr:
merged = pr.merged
return merged
def print_comment_list(self):
"""Print comment creators and the commands they used"""
for x in self.issue.current_comments:
command = None
if x.user.login != 'ansibot':
command = [
y for y in self.VALID_COMMANDS
if y in x.body and not '!' + y in x.body
]
command = ', '.join(command)
else:
# What template did ansibot use?
try:
command = x.body.split('\n')[-1].split()[-2]
except:
pass
if command:
print("\t%s %s (%s)" %
(x.created_at.isoformat(), x.user.login, command))
else:
print("\t%s %s" % (x.created_at.isoformat(), x.user.login))
def wait_for_rate_limit(self):
gh = self._connect()
GithubWrapper.wait_for_rate_limit(githubobj=gh)
def run(self, useapiwrapper=True):
"""Starts a triage run"""
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
self.process()
else:
last_run = None
now = self.get_current_time()
last_run_file = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
last_run_file += '/ansible/ansible/'
else:
last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo
last_run_file += 'issues/last_run.pickle'
last_run_file = os.path.expanduser(last_run_file)
if os.path.isfile(last_run_file):
try:
with open(last_run_file, 'rb') as f:
last_run = pickle.load(f)
except Exception as e:
print(e)
if last_run and not self.no_since:
self.debug('Getting issues updated/created since %s' % last_run)
issues = self.repo.get_issues(since=last_run)
else:
self.debug('Getting ALL issues')
issues = self.repo.get_issues()
for issue in issues:
self.icount += 1
if self.start_at and issue.number > self.start_at:
continue
if self.is_pr(issue):
continue
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if action_res:
while action_res['REDO']:
issue = self.repo.get_issue(int(issue.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if not action_res:
action_res = {'REDO': False}
# save this run time
with open(last_run_file, 'wb') as f:
pickle.dump(now, f)
class TriageIssues(DefaultTriager):
VALID_COMMANDS = ['needs_info', '!needs_info', 'notabug',
'bot_broken', 'bot_skip',
'wontfix', 'bug_resolved', 'resolved_by_pr',
'needs_contributor', 'duplicate_of']
CLOSURE_COMMANDS = [
'notabug',
'wontfix',
'bug_resolved',
'resolved_by_pr',
'duplicate_of'
]
# re-notify interval for maintainers
RENOTIFY_INTERVAL = 14
# max limit for needs_info notifications
RENOTIFY_EXPIRE = 56
# re-notify interval by this number for features
FEATURE_RENOTIFY_INTERVAL = 60
# the max comments per week before ansibot becomes a "spammer"
MAX_BOT_COMMENTS_PER_WEEK = 5
def run(self, useapiwrapper=True):
"""Starts a triage run"""
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
self.process()
else:
last_run = None
now = self.get_current_time()
last_run_file = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
last_run_file += '/ansible/ansible/'
else:
last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo
last_run_file += 'issues/last_run.pickle'
last_run_file = os.path.expanduser(last_run_file)
if os.path.isfile(last_run_file):
try:
with open(last_run_file, 'rb') as f:
last_run = pickle.load(f)
except Exception as e:
print(e)
if last_run and not self.no_since:
self.debug('Getting issues updated/created since %s' % last_run)
issues = self.repo.get_issues(since=last_run)
else:
self.debug('Getting ALL issues')
issues = self.repo.get_issues()
for issue in issues:
self.icount += 1
if self.start_at and issue.number > self.start_at:
continue
if self.is_pr(issue):
continue
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if action_res:
while action_res['REDO']:
issue = self.repo.get_issue(int(issue.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if not action_res:
action_res = {'REDO': False}
# save this run time
with open(last_run_file, 'wb') as f:
pickle.dump(now, f)
@RateLimited
def process(self, usecache=True):
"""Processes the Issue"""
# basic processing
self._process()
# who maintains this?
maintainers = []
if self.meta.get('component_valid', False):
correct_repo = self.match.get('repository')
if correct_repo != self.github_repo:
self.meta['correct_repo'] = False
else:
self.meta['correct_repo'] = True
maintainers = self.get_module_maintainers()
if not maintainers:
#issue_module_no_maintainer
#import epdb; epdb.st()
pass
###########################################################
# Enumerate Actions
###########################################################
self.keep_current_main_labels()
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.process_history(usecache=usecache)
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.create_actions()
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
print("Module: %s" % self.module)
if self.match:
print("MModule: %s" % self.match['name'])
else:
print("MModule: %s" % self.match)
print("Maintainer(s): %s" \
% ', '.join(self.get_module_maintainers(expand=False)))
print("Submitter: %s" % self.issue.get_submitter())
print("Total Comments: %s" % len(self.issue.current_comments))
self.print_comment_list()
print("Current Labels: %s" % ', '.join(sorted(self.issue.current_labels)))
# invoke the wizard
import pprint; pprint.pprint(self.actions)
action_meta = self.apply_actions()
return action_meta
def create_actions(self):
"""Create actions from the desired label/unlabel/comment actions"""
# do nothing for bot_skip
if self.meta['bot_skip']:
return
if 'bot_broken' in self.issue.desired_labels:
# If the bot is broken, do nothing other than set the broken label
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
self.actions['close'] = False
if not 'bot_broken' in self.issue.current_labels:
self.actions['newlabel'] = ['bot_broken']
return
# add the version labels
self.create_label_version_actions()
if self.issue.desired_state != self.issue.instance.state:
if self.issue.desired_state == 'closed':
# close the issue ...
self.actions['close'] = True
# We only want up to 1 comment when an issue is closed
if 'issue_closure' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_closure']
elif 'issue_deprecated_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_deprecated_module']
else:
self.issue.desired_comments = []
if self.issue.desired_comments:
comment = self.render_comment(
boilerplate=self.issue.desired_comments[0]
)
self.actions['comments'].append(comment)
return
resolved_desired_labels = []
for desired_label in self.issue.desired_labels:
resolved_desired_label = self.issue.resolve_desired_labels(
desired_label
)
if desired_label != resolved_desired_label:
resolved_desired_labels.append(resolved_desired_label)
if (resolved_desired_label not in self.issue.get_current_labels()):
self.issue.add_desired_comment(desired_label)
self.actions['newlabel'].append(resolved_desired_label)
else:
resolved_desired_labels.append(desired_label)
if desired_label not in self.issue.get_current_labels():
self.actions['newlabel'].append(desired_label)
'''
if os.path.exists("templates/" + 'issue_' + desired_label + ".j2"):
self.issue.add_desired_comment('issue_' + desired_label)
'''
for current_label in self.issue.get_current_labels():
if current_label in self.IGNORE_LABELS:
continue
if current_label not in resolved_desired_labels:
self.actions['unlabel'].append(current_label)
# should only make one comment at a time
if len(self.issue.desired_comments) > 1:
if 'issue_wrong_repo' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_wrong_repo']
elif 'issue_invalid_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif 'issue_module_no_maintainer' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_module_no_maintainer']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments = ['issue_needs_info']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' not in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
else:
import epdb; epdb.st()
# Do not comment needs_info if it's not a label
if 'issue_needs_info' in self.issue.desired_comments \
and not 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
# render the comments
for boilerplate in self.issue.desired_comments:
comment = self.render_comment(boilerplate=boilerplate)
self.debug(msg=boilerplate)
self.actions['comments'].append(comment)
# do not re-comment
for idx,comment in enumerate(self.actions['comments']):
if self.issue.current_comments:
if self.issue.current_comments[-1].body == comment:
self.debug(msg="Removing repeat comment from actions")
self.actions['comments'].remove(comment)
#import epdb; epdb.st()
def create_label_version_actions(self):
if not self.ansible_label_version:
return
expected = 'affects_%s' % self.ansible_label_version
if expected not in self.valid_labels:
print("NEED NEW LABEL: %s" % expected)
import epdb; epdb.st()
candidates = [x for x in self.issue.current_labels]
candidates = [x for x in candidates if x.startswith('affects_')]
candidates = [x for x in candidates if x != expected]
if len(candidates) > 0:
#for cand in candidates:
# self.issue.pop_desired_label(name=cand)
#return
pass
else:
if expected not in self.issue.current_labels \
or expected not in self.issue.desired_labels:
self.issue.add_desired_label(name=expected)
#import epdb; epdb.st()
def create_commment_actions(self):
'''Render desired comment templates'''
# should only make one comment at a time
if len(self.issue.desired_comments) > 1:
if 'issue_wrong_repo' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_wrong_repo']
elif 'issue_invalid_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif 'issue_module_no_maintainer' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_module_no_maintainer']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments = ['issue_needs_info']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' not in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
else:
import epdb; epdb.st()
# Do not comment needs_info if it's not a label
if 'issue_needs_info' in self.issue.desired_comments \
and not 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
# render the comments
for boilerplate in self.issue.desired_comments:
comment = self.render_comment(boilerplate=boilerplate)
self.debug(msg=boilerplate)
self.actions['comments'].append(comment)
# do not re-comment
for idx,comment in enumerate(self.actions['comments']):
if self.issue.current_comments:
if self.issue.current_comments[-1].body == comment:
self.debug(msg="Removing repeat comment from actions")
self.actions['comments'].remove(comment)
def create_label_actions(self):
"""Create actions from the desired label/unlabel/comment actions"""
if 'bot_broken' in self.issue.desired_labels:
# If the bot is broken, do nothing other than set the broken label
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
self.actions['close'] = False
if not 'bot_broken' in self.issue.current_labels:
self.actions['newlabel'] = ['bot_broken']
return
if self.meta['bot_skip']:
return
resolved_desired_labels = []
for desired_label in self.issue.desired_labels:
resolved_desired_label = self.issue.resolve_desired_labels(
desired_label
)
if desired_label != resolved_desired_label:
resolved_desired_labels.append(resolved_desired_label)
if (resolved_desired_label not in self.issue.get_current_labels()):
self.issue.add_desired_comment(desired_label)
self.actions['newlabel'].append(resolved_desired_label)
else:
resolved_desired_labels.append(desired_label)
if desired_label not in self.issue.get_current_labels():
self.actions['newlabel'].append(desired_label)
for current_label in self.issue.get_current_labels():
if current_label in self.IGNORE_LABELS:
continue
if current_label not in resolved_desired_labels:
self.actions['unlabel'].append(current_label)
def process_history(self, usecache=True):
'''Steps through all known meta about the issue and decides what to do'''
self.meta.update(self.get_facts())
self.add_desired_labels_by_issue_type()
self.add_desired_labels_by_ansible_version()
self.add_desired_labels_by_namespace()
#################################################
# FINAL LOGIC LOOP
#################################################
if self.meta['bot_broken']:
self.debug(msg='broken bot stanza')
self.issue.add_desired_label('bot_broken')
elif self.meta['bot_skip']:
self.debug(msg='bot skip stanza')
# clear out all actions and do nothing
for k,v in six.iteritems(self.actions):
if type(v) == list:
self.actions[k] = []
self.actions['close'] = False
elif self.meta['bot_spam']:
self.debug(msg='bot spam stanza')
# clear out all actions and do nothing
for k,v in six.iteritems(self.actions):
if type(v) == list:
self.actions[k] = []
self.actions['close'] = False
# do we mark this somehow?
self.issue.add_desired_label('bot_broken')
elif self.match and self.match.get('deprecated', False) \
and 'feature_idea' in self.issue.desired_labels:
self.debug(msg='deprecated module stanza')
# Make the deprecated comment
self.issue.desired_comments = ['issue_deprecated_module']
# Close the issue ...
self.issue.set_desired_state('closed')
elif not self.meta['maintainers_known'] and self.meta['valid_module']:
self.debug(msg='unknown maintainer stanza')
self.issue.desired_comments = ['issue_module_no_maintainer']
elif self.meta['maintainer_closure']:
self.debug(msg='maintainer closure stanza')
# Need to close the issue ...
self.issue.set_desired_state('closed')
elif self.meta['new_module_request']:
self.debug(msg='new module request stanza')
self.issue.desired_comments = []
for label in self.issue.current_labels:
if not label in self.issue.desired_labels:
self.issue.desired_labels.append(label)
elif not self.meta['correct_repo']:
self.debug(msg='wrong repo stanza')
self.issue.desired_comments = ['issue_wrong_repo']
self.actions['close'] = True
elif not self.meta['valid_module'] and \
not self.meta['maintainer_command_needsinfo']:
self.debug(msg='invalid module stanza')
self.issue.add_desired_label('needs_info')
if 'issue_invalid_module' not in self.issue.current_bot_comments \
and not 'issue_needs_info' in self.issue.current_bot_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif not self.meta['notification_maintainers'] and \
not self.meta['maintainer_command_needsinfo']:
self.debug(msg='no maintainer stanza')
self.issue.add_desired_label('waiting_on_maintainer')
self.issue.add_desired_comment("issue_module_no_maintainer")
elif self.meta['maintainer_command'] == 'needs_contributor':
# maintainer can't or won't fix this, but would like someone else to
self.debug(msg='maintainer needs contributor stanza')
self.issue.add_desired_label('waiting_on_contributor')
elif self.meta['maintainer_waiting_on']:
self.debug(msg='maintainer wait stanza')
self.issue.add_desired_label('waiting_on_maintainer')
if len(self.issue.current_comments) == 0:
# new issue
if self.meta['issue_type']:
if self.meta['submitter'] not in self.meta['notification_maintainers']:
# ping the maintainer
self.issue.add_desired_comment('issue_new')
else:
# do not send intial ping to maintainer if also submitter
if 'issue_new' in self.issue.desired_comments:
self.issue.desired_comments.remove('issue_new')
else:
# old issue -- renotify
if not self.match['deprecated'] and self.meta['notification_maintainers']:
if self.meta['maintainer_to_ping']:
self.issue.add_desired_comment("issue_notify_maintainer")
elif self.meta['maintainer_to_reping']:
self.issue.add_desired_comment("issue_renotify_maintainer")
elif self.meta['submitter_waiting_on']:
self.debug(msg='submitter wait stanza')
if 'waiting_on_maintainer' in self.issue.desired_labels:
self.issue.desired_labels.remove('waiting_on_maintainer')
if (self.meta['needsinfo_add'] or self.meta['missing_sections']) \
or (not self.meta['needsinfo_remove'] and self.meta['missing_sections']) \
or (self.meta['needsinfo_add'] and not self.meta['missing_sections']):
#import epdb; epdb.st()
self.issue.add_desired_label('needs_info')
if len(self.issue.current_comments) == 0 or \
not self.meta['maintainer_commented']:
if self.issue.current_bot_comments:
if 'issue_needs_info' not in self.issue.current_bot_comments:
self.issue.add_desired_comment("issue_needs_info")
else:
self.issue.add_desired_comment("issue_needs_info")
# needs_info: warn if stale, close if expired
elif self.meta['needsinfo_expired']:
self.issue.add_desired_comment("issue_closure")
self.issue.set_desired_state('closed')
elif self.meta['needsinfo_stale'] \
and (self.meta['submitter_to_ping'] or self.meta['submitter_to_reping']):
self.issue.add_desired_comment("issue_pending_closure")
def get_history_facts(self, usecache=True):
return self.get_facts(usecache=usecache)
def get_facts(self, usecache=True):
'''Only used by the ansible/ansible triager at the moment'''
hfacts = {}
today = self.get_current_time()
self.history = HistoryWrapper(
self.issue,
usecache=usecache,
cachedir=self.cachedir
)
# what was the last commment?
bot_broken = False
if self.issue.current_comments:
for comment in self.issue.current_comments:
if 'bot_broken' in comment.body:
bot_broken = True
# did someone from ansible want this issue skipped?
bot_skip = False
if self.issue.current_comments:
for comment in self.issue.current_comments:
if 'bot_skip' in comment.body and comment.user.login in self.ansible_members:
bot_skip = True
#########################################################
# Facts for new "triage" workflow
#########################################################
# these logins don't count for the next set of facts
exclude_logins = ['ansibot', 'gregdek', self.issue.instance.user.login]
# Has a human (not ansibot + not submitter) ever changed labels?
lchanges = [x for x in self.history.history if x['event'] in ['labeled', 'unlabeled'] and x['actor'] not in exclude_logins]
if lchanges:
hfacts['human_labeled'] = True
else:
hfacts['human_labeled'] = False
# Has a human (not ansibot + not submitter) ever changed the title?
tchanges = [x for x in self.history.history if x['event'] in ['renamed'] and x['actor'] not in exclude_logins]
if tchanges:
hfacts['human_retitled'] = True
else:
hfacts['human_retitled'] = False
# Has a human (not ansibot + not submitter) ever changed the description?
# ... github doesn't record an event for this. Lame.
# Has a human (not ansibot + not submitter) ever set a milestone?
mchanges = [x for x in self.history.history if x['event'] in ['milestoned', 'demilestoned'] and x['actor'] not in exclude_logins]
if mchanges:
hfacts['human_milestoned'] = True
else:
hfacts['human_milestoned'] = False
# Has a human (not ansibot + not submitter) ever set/unset assignees?
achanges = [x for x in self.history.history if x['event'] in ['assigned', 'unassigned'] and x['actor'] not in exclude_logins]
if achanges:
hfacts['human_assigned'] = True
else:
hfacts['human_assigned'] = False
#########################################################
# Has the bot been overzealous with comments?
hfacts['bot_spam'] = False
bcg = self.history.get_user_comments_groupby('ansibot', groupby='w')
for k,v in six.iteritems(bcg):
if v >= self.MAX_BOT_COMMENTS_PER_WEEK:
hfacts['bot_spam'] = True
# Is this a new module?
hfacts['new_module_request'] = False
if 'feature_idea' in self.issue.desired_labels:
if self.template_data['component name'] == 'new':
hfacts['new_module_request'] = True
# who made this and when did they last comment?
submitter = self.issue.get_submitter()
submitter_last_commented = self.history.last_commented_at(submitter)
if not submitter_last_commented:
submitter_last_commented = self.issue.instance.created_at
#import epdb; epdb.st()
submitter_last_comment = self.history.last_comment(submitter)
submitter_last_notified = self.history.last_notified(submitter)
# what did they not provide?
missing_sections = self.issue.get_missing_sections()
# Is this a valid module?
if self.match:
self.meta['valid_module'] = True
else:
self.meta['valid_module'] = False
# Filed in the right place?
if self.meta['valid_module']:
if self.match['repository'] != self.github_repo:
hfacts['correct_repo'] = False
else:
hfacts['correct_repo'] = True
else:
hfacts['correct_repo'] = True
# DEBUG + FIXME - speeds up bulk triage
if 'component name' in missing_sections \
and (self.match or self.github_repo == 'ansible'):
missing_sections.remove('component name')
#import epdb; epdb.st()
# Who are the maintainers?
maintainers = [x for x in self.get_module_maintainers()]
#hfacts['maintainers'] = maintainers
#import epdb; epdb.st()
# Set a fact to indicate that we know the maintainer
self.meta['maintainers_known'] = False
if maintainers:
self.meta['maintainers_known'] = True
if 'ansible' in maintainers:
maintainers.remove('ansible')
maintainers.extend(self.ansible_members)
if 'ansibot' in maintainers:
maintainers.remove('ansibot')
if submitter in maintainers:
maintainers.remove(submitter)
maintainers = sorted(set(maintainers))
# Has maintainer been notified? When?
notification_maintainers = [x for x in self.get_module_maintainers()]
if 'ansible' in notification_maintainers:
notification_maintainers.extend(self.ansible_members)
if 'ansibot' in notification_maintainers:
notification_maintainers.remove('ansibot')
hfacts['notification_maintainers'] = notification_maintainers
maintainer_last_notified = self.history.\
last_notified(notification_maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
maintainer_last_viewed = self.history.last_viewed_at(maintainers)
#import epdb; epdb.st()
# Has maintainer been mentioned?
maintainer_mentioned = self.history.is_mentioned(maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
# Has the maintainer ever responded?
maintainer_commented = self.history.has_commented(maintainers)
maintainer_last_commented = self.history.last_commented_at(maintainers)
maintainer_last_comment = self.history.last_comment(maintainers)
maintainer_comments = self.history.get_user_comments(maintainers)
#import epdb; epdb.st()
# Was the maintainer the last commentor?
last_commentor_ismaintainer = False
last_commentor_issubmitter = False
last_commentor = self.history.last_commentor()
if last_commentor in maintainers and last_commentor != self.github_user:
last_commentor_ismaintainer = True
elif last_commentor == submitter:
last_commentor_issubmitter = True
# Did the maintainer issue a command?
maintainer_commands = self.history.get_commands(maintainers,
self.VALID_COMMANDS)
# Keep all commands
hfacts['maintainer_commands'] = maintainer_commands
# Set a bit for the last command given
if hfacts['maintainer_commands']:
hfacts['maintainer_command'] = hfacts['maintainer_commands'][-1]
else:
hfacts['maintainer_command'] = None
# Is the last command a closure command?
if hfacts['maintainer_command'] in self.CLOSURE_COMMANDS:
hfacts['maintainer_closure'] = True
else:
hfacts['maintainer_closure'] = False
# handle resolved_by_pr ...
if 'resolved_by_pr' in maintainer_commands:
maintainer_comments = self.history.get_user_comments(maintainers)
maintainer_comments = [x for x in reversed(maintainer_comments) \
if 'resolved_by_pr' in x]
for comment in maintainer_comments:
pr_number = extract_pr_number_from_comment(comment)
hfacts['resolved_by_pr'] = {
'number': pr_number,
'merged': self.is_pr_merged(pr_number),
}
if not hfacts['resolved_by_pr']['merged']:
hfacts['maintainer_closure'] = False
break
# needs_info toggles
ni_commands = [x for x in maintainer_commands if 'needs_info' in x]
# Has the maintainer ever subscribed?
maintainer_subscribed = self.history.has_subscribed(maintainers)
# Was it ever needs_info?
was_needs_info = self.history.was_labeled(label='needs_info')
needsinfo_last_applied = self.history.label_last_applied('needs_info')
needsinfo_last_removed = self.history.label_last_removed('needs_info')
# Still needs_info?
needsinfo_add = False
needsinfo_remove = False
if 'needs_info' in self.issue.current_labels:
if not needsinfo_last_applied or not submitter_last_commented:
import epdb; epdb.st()
if submitter_last_commented > needsinfo_last_applied:
needsinfo_add = False
needsinfo_remove = True
#if 'needs_info' in maintainer_commands and maintainer_last_commented:
if ni_commands and maintainer_last_commented:
if ni_commands[-1] == 'needs_info':
#import epdb; epdb.st()
if submitter_last_commented and maintainer_last_commented:
if submitter_last_commented > maintainer_last_commented:
needsinfo_add = False
needsinfo_remove = True
else:
needsinfo_add = True
needsinfo_remove = False
else:
needsinfo_add = False
needsinfo_remove = True
# Save existing needs_info if not time to remove ...
if 'needs_info' in self.issue.current_labels \
and not needsinfo_add \
and not needsinfo_remove:
needsinfo_add = True
if ni_commands and maintainer_last_commented:
if maintainer_last_commented > submitter_last_commented:
if ni_commands[-1] == 'needs_info':
needsinfo_add = True
needsinfo_remove = False
else:
needsinfo_add = False
needsinfo_remove = True
#import epdb; epdb.st()
# Is needs_info stale or expired?
needsinfo_age = None
needsinfo_stale = False
needsinfo_expired = False
if 'needs_info' in self.issue.current_labels:
time_delta = today - needsinfo_last_applied
needsinfo_age = time_delta.days
if needsinfo_age > self.RENOTIFY_INTERVAL:
needsinfo_stale = True
if needsinfo_age > self.RENOTIFY_EXPIRE:
needsinfo_expired = True
# Should we be in waiting_on_maintainer mode?
maintainer_waiting_on = False
if (needsinfo_remove or not needsinfo_add) \
or not was_needs_info \
and not missing_sections:
maintainer_waiting_on = True
# Should we [re]notify the submitter?
submitter_waiting_on = False
submitter_to_ping = False
submitter_to_reping = False
if not maintainer_waiting_on:
submitter_waiting_on = True
if missing_sections:
submitter_waiting_on = True
maintainer_waiting_on = False
# use [!]needs_info to set final state
if ni_commands:
if ni_commands[-1] == '!needs_info':
submitter_waiting_on = False
maintainer_waiting_on = True
elif ni_commands[-1] == 'needs_info':
submitter_waiting_on = True
maintainer_waiting_on = False
# Time to [re]ping maintainer?
maintainer_to_ping = False
maintainer_to_reping = False
if maintainer_waiting_on:
# if feature idea, extend the notification interval
interval = self.RENOTIFY_INTERVAL
if self.meta.get('issue_type', None) == 'feature idea' \
or 'feature_idea' in self.issue.current_labels:
interval = self.FEATURE_RENOTIFY_INTERVAL
if maintainer_viewed and not maintainer_last_notified:
time_delta = today - maintainer_last_viewed
view_age = time_delta.days
if view_age > interval:
maintainer_to_reping = True
elif maintainer_last_notified:
time_delta = today - maintainer_last_notified
ping_age = time_delta.days
if ping_age > interval:
maintainer_to_reping = True
else:
maintainer_to_ping = True
# Time to [re]ping the submitter?
if submitter_waiting_on:
if submitter_last_notified:
time_delta = today - submitter_last_notified
notification_age = time_delta.days
if notification_age > self.RENOTIFY_INTERVAL:
submitter_to_reping = True
else:
submitter_to_reping = False
submitter_to_ping = False
else:
submitter_to_ping = True
submitter_to_reping = False
# needs_contributor ...
hfacts['needs_contributor'] = False
for command in maintainer_commands:
if command == 'needs_contributor':
hfacts['needs_contributor'] = True
elif command == '!needs_contributor':
hfacts['needs_contributor'] = False
hfacts['bot_broken'] = bot_broken
hfacts['bot_skip'] = bot_skip
hfacts['missing_sections'] = missing_sections
hfacts['was_needsinfo'] = was_needs_info
hfacts['needsinfo_age'] = needsinfo_age
hfacts['needsinfo_stale'] = needsinfo_stale
hfacts['needsinfo_expired'] = needsinfo_expired
hfacts['needsinfo_add'] = needsinfo_add
hfacts['needsinfo_remove'] = needsinfo_remove
hfacts['notification_maintainers'] = self.get_module_maintainers() or 'ansible'
hfacts['maintainer_last_notified'] = maintainer_last_notified
hfacts['maintainer_commented'] = maintainer_commented
hfacts['maintainer_viewed'] = maintainer_viewed
hfacts['maintainer_subscribed'] = maintainer_subscribed
hfacts['maintainer_command_needsinfo'] = 'needs_info' in maintainer_commands
hfacts['maintainer_command_not_needsinfo'] = '!needs_info' in maintainer_commands
hfacts['maintainer_waiting_on'] = maintainer_waiting_on
hfacts['maintainer_to_ping'] = maintainer_to_ping
hfacts['maintainer_to_reping'] = maintainer_to_reping
hfacts['submitter'] = submitter
hfacts['submitter_waiting_on'] = submitter_waiting_on
hfacts['submitter_to_ping'] = submitter_to_ping
hfacts['submitter_to_reping'] = submitter_to_reping
hfacts['last_commentor_ismaintainer'] = last_commentor_ismaintainer
hfacts['last_commentor_issubmitter'] = last_commentor_issubmitter
hfacts['last_commentor'] = last_commentor
return hfacts
class TriageIssues(DefaultTriager):
VALID_COMMANDS = [
'needs_info', '!needs_info', 'notabug', 'wontfix', 'bug_resolved',
'resolved_by_pr', 'needs_contributor', 'duplicate_of'
]
def run(self, useapiwrapper=True):
"""Starts a triage run"""
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo,
issue=issue,
cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
self.process()
else:
last_run = None
now = self.get_current_time()
last_run_file = '~/.ansibullbot/cache'
if self.github_repo == 'ansible':
last_run_file += '/ansible/ansible/'
else:
last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo
last_run_file += 'issues/last_run.pickle'
last_run_file = os.path.expanduser(last_run_file)
if os.path.isfile(last_run_file):
try:
with open(last_run_file, 'rb') as f:
last_run = pickle.load(f)
except Exception as e:
print(e)
if last_run and not self.no_since:
self.debug('Getting issues updated/created since %s' %
last_run)
issues = self.repo.get_issues(since=last_run)
else:
self.debug('Getting ALL issues')
issues = self.repo.get_issues()
for issue in issues:
self.icount += 1
if self.start_at and issue.number > self.start_at:
continue
if self.is_pr(issue):
continue
self.issue = IssueWrapper(repo=self.repo,
issue=issue,
cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if action_res:
while action_res['REDO']:
issue = self.repo.get_issue(int(issue.number))
self.issue = IssueWrapper(repo=self.repo,
issue=issue,
cachedir=self.cachedir)
self.issue.get_events()
self.issue.get_comments()
action_res = self.process()
if not action_res:
action_res = {'REDO': False}
# save this run time
with open(last_run_file, 'wb') as f:
pickle.dump(now, f)
#import epdb; epdb.st()
@ratecheck()
def process(self, usecache=True):
"""Processes the Issue"""
# basic processing
self._process()
# filed under the correct repository?
this_repo = False
correct_repo = None
# who maintains this?
maintainers = []
component_isvalid = self.meta.get('component_isvalid', False)
if not component_isvalid:
pass
else:
correct_repo = self.module_indexer.\
get_repository_for_module(component)
if correct_repo == self.github_repo:
this_repo = True
maintainers = self.get_module_maintainers()
# Has the maintainer -ever- commented?
maintainer_commented = False
if component_isvalid:
maintainer_commented = self.has_maintainer_commented()
waiting_on_maintainer = False
if component_isvalid:
waiting_on_maintainer = self.is_waiting_on_maintainer()
# How long ago did the maintainer last comment?
maintainer_last_comment_age = -1
if component_isvalid:
maintainer_last_comment_age = self.age_of_last_maintainer_comment()
###########################################################
# Enumerate Actions
###########################################################
self.keep_current_main_labels()
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.process_history(usecache=usecache)
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.create_actions()
self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
print("Module: %s" % self.module)
if self.match:
print("MModule: %s" % self.match['name'])
else:
print("MModule: %s" % self.match)
print("Maintainer(s): %s" \
% ', '.join(self.get_module_maintainers(expand=False)))
print("Submitter: %s" % self.issue.get_submitter())
print("Total Comments: %s" % len(self.issue.current_comments))
for x in self.issue.current_comments:
print("\t%s %s %s" % (x.created_at.isoformat(),
x.user.login,
[y for y in self.VALID_COMMANDS if y in x.body \
and not '!' + y in x.body] or ''))
print("Current Labels: %s" %
', '.join(sorted(self.issue.current_labels)))
import pprint
pprint.pprint(self.actions)
# invoke the wizard
action_meta = self.apply_actions()
return action_meta
def create_actions(self):
"""Create actions from the desired label/unlabel/comment actions"""
if 'bot_broken' in self.issue.desired_labels:
# If the bot is broken, do nothing other than set the broken label
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
self.actions['close'] = False
if not 'bot_broken' in self.issue.current_labels:
self.actions['newlabel'] = ['bot_broken']
return
# add the version labels
self.create_label_version_actions()
if self.issue.desired_state != self.issue.instance.state:
if self.issue.desired_state == 'closed':
# close the issue ...
self.actions['close'] = True
if 'issue_closure' in self.issue.desired_comments:
comment = self.render_comment(boilerplate='issue_closure')
self.actions['comments'].append(comment)
return
resolved_desired_labels = []
for desired_label in self.issue.desired_labels:
resolved_desired_label = self.issue.resolve_desired_labels(
desired_label)
if desired_label != resolved_desired_label:
resolved_desired_labels.append(resolved_desired_label)
if (resolved_desired_label
not in self.issue.get_current_labels()):
self.issue.add_desired_comment(desired_label)
self.actions['newlabel'].append(resolved_desired_label)
else:
resolved_desired_labels.append(desired_label)
if desired_label not in self.issue.get_current_labels():
self.actions['newlabel'].append(desired_label)
'''
if os.path.exists("templates/" + 'issue_' + desired_label + ".j2"):
self.issue.add_desired_comment('issue_' + desired_label)
'''
for current_label in self.issue.get_current_labels():
if current_label in self.IGNORE_LABELS:
continue
if current_label not in resolved_desired_labels:
self.actions['unlabel'].append(current_label)
# should only make one comment at a time
if len(self.issue.desired_comments) > 1:
if 'issue_wrong_repo' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_wrong_repo']
elif 'issue_invalid_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif 'issue_module_no_maintainer' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_module_no_maintainer']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments = ['issue_needs_info']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' not in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
else:
import epdb
epdb.st()
# Do not comment needs_info if it's not a label
if 'issue_needs_info' in self.issue.desired_comments \
and not 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
# render the comments
for boilerplate in self.issue.desired_comments:
comment = self.render_comment(boilerplate=boilerplate)
self.debug(msg=boilerplate)
self.actions['comments'].append(comment)
# do not re-comment
for idx, comment in enumerate(self.actions['comments']):
if self.issue.current_comments:
if self.issue.current_comments[-1].body == comment:
self.debug(msg="Removing repeat comment from actions")
self.actions['comments'].remove(comment)
#import epdb; epdb.st()
def create_label_version_actions(self):
if not self.ansible_label_version:
return
expected = 'affects_%s' % self.ansible_label_version
if expected not in self.valid_labels:
print("NEED NEW LABEL: %s" % expected)
import epdb
epdb.st()
candidates = [x for x in self.issue.current_labels]
candidates = [x for x in candidates if x.startswith('affects_')]
candidates = [x for x in candidates if x != expected]
if len(candidates) > 0:
#for cand in candidates:
# self.issue.pop_desired_label(name=cand)
#return
pass
else:
if expected not in self.issue.current_labels \
or expected not in self.issue.desired_labels:
self.issue.add_desired_label(name=expected)
#import epdb; epdb.st()
def create_commment_actions(self):
'''Render desired comment templates'''
# should only make one comment at a time
if len(self.issue.desired_comments) > 1:
if 'issue_wrong_repo' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_wrong_repo']
elif 'issue_invalid_module' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif 'issue_module_no_maintainer' in self.issue.desired_comments:
self.issue.desired_comments = ['issue_module_no_maintainer']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments = ['issue_needs_info']
elif 'issue_needs_info' in self.issue.desired_comments \
and 'needs_info' not in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
else:
import epdb
epdb.st()
# Do not comment needs_info if it's not a label
if 'issue_needs_info' in self.issue.desired_comments \
and not 'needs_info' in self.issue.desired_labels:
self.issue.desired_comments.remove('issue_needs_info')
# render the comments
for boilerplate in self.issue.desired_comments:
comment = self.render_comment(boilerplate=boilerplate)
self.debug(msg=boilerplate)
self.actions['comments'].append(comment)
# do not re-comment
for idx, comment in enumerate(self.actions['comments']):
if self.issue.current_comments:
if self.issue.current_comments[-1].body == comment:
self.debug(msg="Removing repeat comment from actions")
self.actions['comments'].remove(comment)
def create_label_actions(self):
"""Create actions from the desired label/unlabel/comment actions"""
if 'bot_broken' in self.issue.desired_labels:
# If the bot is broken, do nothing other than set the broken label
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
self.actions['close'] = False
if not 'bot_broken' in self.issue.current_labels:
self.actions['newlabel'] = ['bot_broken']
return
resolved_desired_labels = []
for desired_label in self.issue.desired_labels:
resolved_desired_label = self.issue.resolve_desired_labels(
desired_label)
if desired_label != resolved_desired_label:
resolved_desired_labels.append(resolved_desired_label)
if (resolved_desired_label
not in self.issue.get_current_labels()):
self.issue.add_desired_comment(desired_label)
self.actions['newlabel'].append(resolved_desired_label)
else:
resolved_desired_labels.append(desired_label)
if desired_label not in self.issue.get_current_labels():
self.actions['newlabel'].append(desired_label)
for current_label in self.issue.get_current_labels():
if current_label in self.IGNORE_LABELS:
continue
if current_label not in resolved_desired_labels:
self.actions['unlabel'].append(current_label)
def process_history(self, usecache=True):
self.meta = {}
today = self.get_current_time()
# Build the history
self.debug(msg="Building event history ...")
self.history = HistoryWrapper(self.issue,
usecache=usecache,
cachedir=self.cachedir)
# what was the last commment?
bot_broken = False
if self.issue.current_comments:
for comment in self.issue.current_comments:
if 'bot_broken' in comment.body:
bot_broken = True
# who made this and when did they last comment?
submitter = self.issue.get_submitter()
submitter_last_commented = self.history.last_commented_at(submitter)
submitter_last_comment = self.history.last_comment(submitter)
submitter_last_notified = self.history.last_notified(submitter)
# what did they not provide?
missing_sections = self.issue.get_missing_sections()
if 'ansible version' in missing_sections:
missing_sections.remove('ansible version')
# DEBUG + FIXME - speeds up bulk triage
if 'component name' in missing_sections and self.match:
missing_sections.remove('component name')
#import epdb; epdb.st()
# Is this a valid module?
valid_module = False
correct_repo = True
if self.match:
valid_module = True
if self.match['repository'] != self.github_repo:
correct_repo = False
#import epdb; epdb.st()
# Do these after evaluating the module
self.add_desired_labels_by_issue_type()
if 'ansible version' in missing_sections:
#self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.add_desired_labels_by_ansible_version()
#self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
self.add_desired_labels_by_namespace()
#self.debug(msg='desired_comments: %s' % self.issue.desired_comments)
# Who are the maintainers?
maintainers = [x for x in self.get_module_maintainers()]
#if 'ansible' in maintainers:
# maintainers.remove('ansible')
# maintainers += self.ansible_members
#print("MAINTAINERS: %s" % maintainers)
if 'ansible' in maintainers:
maintainers.remove('ansible')
maintainers.extend(self.ansible_members)
if 'ansibot' in maintainers:
maintainers.remove('ansibot')
if submitter in maintainers:
maintainers.remove(submitter)
maintainers = sorted(set(maintainers))
# Has maintainer been notified? When?
notification_maintainers = self.get_module_maintainers()
if 'ansible' in notification_maintainers:
notification_maintainers.extend(self.ansible_members)
if 'ansibot' in notification_maintainers:
notification_maintainers.remove('ansibot')
maintainer_last_notified = self.history.\
last_notified(notification_maintainers)
#import epdb; epdb.st()
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
maintainer_last_viewed = self.history.last_viewed_at(maintainers)
#import epdb; epdb.st()
# Has maintainer been mentioned?
maintainer_mentioned = self.history.is_mentioned(maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
# Has the maintainer ever responded?
maintainer_commented = self.history.has_commented(maintainers)
maintainer_last_commented = self.history.last_commented_at(maintainers)
maintainer_last_comment = self.history.last_comment(maintainers)
maintainer_comments = self.history.get_user_comments(maintainers)
#import epdb; epdb.st()
# Was the maintainer the last commentor?
last_commentor_ismaintainer = False
last_commentor_issubmitter = False
last_commentor = self.history.last_commentor()
if last_commentor in maintainers and last_commentor != self.github_user:
last_commentor_ismaintainer = True
elif last_commentor == submitter:
last_commentor_issubmitter = True
# Did the maintainer issue a command?
maintainer_commands = self.history.get_commands(
maintainers, self.VALID_COMMANDS)
#import epdb; epdb.st()
maintainer_command_close = False
maintainer_command_needsinfo = False
maintainer_command_not_needsinfo = False
maintainer_command_notabug = False
maintainer_command_wontfix = False
maintainer_command_resolved_bug = False
maintainer_command_resolved_pr = False
maintainer_command_needscontributor = False
maintainer_command_duplicateof = False
if maintainer_commented and not maintainer_last_comment:
print('ERROR: should have a comment from maintainer')
import epdb
epdb.st()
elif maintainer_last_comment and last_commentor_ismaintainer:
maintainer_last_comment = maintainer_last_comment.strip()
if 'needs_info' in maintainer_last_comment \
and not '!needs_info' in maintainer_last_comment:
maintainer_command_needsinfo = True
elif '!needs_info' in maintainer_last_comment:
maintainer_command_not_needsinfo = True
elif 'notabug' in maintainer_last_comment:
maintainer_command_notabug = True
maintainer_command_close = True
elif 'wontfix' in maintainer_last_comment:
maintainer_command_wontfix = True
maintainer_command_close = True
elif 'bug_resolved' in maintainer_last_comment:
maintainer_command_resolved_bug = True
maintainer_command_close = True
elif 'resolved_by_pr' in maintainer_last_comment:
maintainer_command_resolved_pr = True
maintainer_command_close = True
elif 'needs_contributor' in maintainer_last_comment:
maintainer_command_needscontributor = True
elif 'duplicate_of' in maintainer_last_comment:
maintainer_command_duplicateof = True
maintainer_command_close = True
elif maintainer_commands:
# are there any persistant commands?
if 'needs_contributor' in maintainer_commands:
maintainer_command_needscontributor = True
elif not missing_sections and not submitter_last_commented and maintainer_commands[
-1] == 'needs_info':
maintainer_command_needsinfo = True
elif not missing_sections and not submitter_last_commented and maintainer_commands[
-1] == '!needs_info':
maintainer_command_not_needsinfo = True
#import epdb; epdb.st()
#import epdb; epdb.st()
# Has the maintainer ever subscribed?
maintainer_subscribed = self.history.has_subscribed(maintainers)
# Was it ever needs_info?
was_needs_info = self.history.was_labeled(label='needs_info')
needsinfo_last_applied = self.history.label_last_applied('needs_info')
needsinfo_last_removed = self.history.label_last_removed('needs_info')
#import epdb; epdb.st()
# Still needs_info?
needsinfo_add = False
needsinfo_remove = False
if 'needs_info' in self.issue.current_labels:
if submitter_last_commented and needsinfo_last_applied:
if submitter_last_commented > needsinfo_last_applied \
and not missing_sections:
needsinfo_remove = True
if maintainer_command_needsinfo and maintainer_last_commented:
if submitter_last_commented and maintainer_last_commented:
if submitter_last_commented > maintainer_last_commented:
needsinfo_add = False
needsinfo_remove = True
else:
needsinfo_add = True
needsinfo_remove = False
# Is needs_info stale or expired?
needsinfo_age = None
needsinfo_stale = False
needsinfo_expired = False
if 'needs_info' in self.issue.current_labels:
time_delta = today - needsinfo_last_applied
needsinfo_age = time_delta.days
if needsinfo_age > 14:
needsinfo_stale = True
if needsinfo_age > 56:
needsinfo_expired = True
# Should we be in waiting_on_maintainer mode?
maintainer_waiting_on = False
if (needsinfo_remove or not needsinfo_add) \
or not was_needs_info \
and not missing_sections:
maintainer_waiting_on = True
# Should we [re]notify the submitter?
submitter_waiting_on = False
submitter_to_ping = False
submitter_to_reping = False
if not maintainer_waiting_on:
submitter_waiting_on = True
if missing_sections:
submitter_waiting_on = True
maintainer_waiting_on = False
else:
if 'needs_info' in self.issue.current_labels \
and not maintainer_command_not_needsinfo:
needsinfo_remove = True
submitter_waiting_on = False
maintainer_waiting_on = True
if maintainer_command_not_needsinfo:
submitter_waiting_on = False
maintainer_waiting_on = True
if maintainer_command_needsinfo:
submitter_waiting_on = True
maintainer_waiting_on = False
needsinfo_add = True
needsinfo_remove = False
# Time to [re]ping maintainer?
maintainer_to_ping = False
maintainer_to_reping = False
if maintainer_waiting_on:
#import epdb; epdb.st()
if maintainer_viewed and not maintainer_last_notified:
time_delta = today - maintainer_last_viewed
view_age = time_delta.days
if view_age > 14:
maintainer_to_reping = True
elif maintainer_last_notified:
time_delta = today - maintainer_last_notified
ping_age = time_delta.days
if ping_age > 14:
maintainer_to_reping = True
else:
maintainer_to_ping = True
# Time to [re]ping the submitter?
if submitter_waiting_on:
if submitter_last_notified:
time_delta = today - submitter_last_notified
notification_age = time_delta.days
if notification_age > 14:
submitter_to_reping = True
else:
submitter_to_reping = False
submitter_to_ping = False
else:
submitter_to_ping = True
submitter_to_reping = False
issue_type = self.template_data.get('issue type', None)
issue_type = self.issue_type_to_label(issue_type)
self.meta['issue_type'] = issue_type
# new module requests need to disable everything
# https://github.com/ansible/ansible-modules-core/issues/4112
# https://github.com/ansible/ansible-modules-core/issues/2267
# https://github.com/ansible/ansible-modules-core/issues/2626
# https://github.com/ansible/ansible-modules-core/issues/645
new_module_request = False
if 'feature_idea' in self.issue.desired_labels:
if self.template_data['component name'] == 'new':
new_module_request = True
# reset the maintainers
maintainers = self.get_module_maintainers()
#################################################
# FINAL LOGIC LOOP
#################################################
if bot_broken:
self.debug(msg='broken bot stanza')
self.issue.add_desired_label('bot_broken')
elif maintainer_command_close:
self.debug(msg='maintainer closure stanza')
# Need to close the issue ...
self.issue.set_desired_state('closed')
elif new_module_request:
self.debug(msg='new module request stanza')
self.issue.desired_comments = []
for label in self.issue.current_labels:
if not label in self.issue.desired_labels:
self.issue.desired_labels.append(label)
elif not correct_repo:
self.debug(msg='wrong repo stanza')
self.issue.desired_comments = ['issue_wrong_repo']
self.actions['close'] = True
#import epdb; epdb.st()
elif not valid_module and not maintainer_command_needsinfo:
self.debug(msg='invalid module stanza')
#import epdb; epdb.st()
self.issue.add_desired_label('needs_info')
if 'issue_invalid_module' not in self.issue.current_bot_comments \
and not 'issue_needs_info' in self.issue.current_bot_comments:
self.issue.desired_comments = ['issue_invalid_module']
elif not maintainers and not maintainer_command_needsinfo:
self.debug(msg='no maintainer stanza')
self.issue.add_desired_label('waiting_on_maintainer')
self.issue.add_desired_comment("issue_module_no_maintainer")
elif maintainer_command_needscontributor:
# maintainer can't or won't fix this, but would like someone else to
self.debug(msg='maintainer needs contributor stanza')
self.issue.add_desired_label('waiting_on_contributor')
#import epdb; epdb.st()
elif maintainer_waiting_on:
self.debug(msg='maintainer wait stanza')
self.issue.add_desired_label('waiting_on_maintainer')
if len(self.issue.current_comments) == 0:
if issue_type:
if submitter not in self.module_maintainers:
self.issue.add_desired_comment('issue_new')
else:
if 'issue_new' in self.issue.desired_comments:
self.issue.desired_comments.remove('issue_new')
#import epdb; epdb.st()
else:
if maintainers != ['DEPRECATED']:
if maintainer_to_ping and maintainers:
self.issue.add_desired_comment(
"issue_notify_maintainer")
elif maintainer_to_reping and maintainers:
self.issue.add_desired_comment(
"issue_renotify_maintainer")
#import epdb; epdb.st()
elif submitter_waiting_on:
self.debug(msg='submitter wait stanza')
#import epdb; epdb.st()
if 'waiting_on_maintainer' in self.issue.desired_labels:
self.issue.desired_labels.remove('waiting_on_maintainer')
if (needsinfo_add or missing_sections) \
or (not needsinfo_remove and missing_sections) \
or (needsinfo_add and not missing_sections):
self.issue.add_desired_label('needs_info')
#import epdb; epdb.st()
if len(self.issue.current_comments
) == 0 or not maintainer_commented:
self.issue.add_desired_comment("issue_needs_info")
# needs_info: warn if stale, close if expired
elif needsinfo_expired:
self.issue.add_desired_comment("issue_closure")
self.issue.set_desired_state('closed')
elif needsinfo_stale \
and (submitter_to_ping or submitter_to_reping):
self.issue.add_desired_comment("issue_pending_closure")
#import epdb; epdb.st()
#import epdb; epdb.st()
def get_history_facts(self, usecache=True):
hfacts = {}
today = self.get_current_time()
self.history = HistoryWrapper(self.issue,
usecache=usecache,
cachedir=self.cachedir)
# what was the last commment?
bot_broken = False
if self.issue.current_comments:
for comment in self.issue.current_comments:
if 'bot_broken' in comment.body:
bot_broken = True
# who made this and when did they last comment?
submitter = self.issue.get_submitter()
submitter_last_commented = self.history.last_commented_at(submitter)
if not submitter_last_commented:
submitter_last_commented = self.issue.instance.created_at
#import epdb; epdb.st()
submitter_last_comment = self.history.last_comment(submitter)
submitter_last_notified = self.history.last_notified(submitter)
# what did they not provide?
missing_sections = self.issue.get_missing_sections()
#if 'ansible version' in missing_sections:
# missing_sections.remove('ansible version')
# DEBUG + FIXME - speeds up bulk triage
if 'component name' in missing_sections \
and (self.match or self.github_repo == 'ansible'):
missing_sections.remove('component name')
#import epdb; epdb.st()
# Who are the maintainers?
maintainers = [x for x in self.get_module_maintainers()]
if 'ansible' in maintainers:
maintainers.remove('ansible')
maintainers.extend(self.ansible_members)
if 'ansibot' in maintainers:
maintainers.remove('ansibot')
if submitter in maintainers:
maintainers.remove(submitter)
maintainers = sorted(set(maintainers))
# Has maintainer been notified? When?
notification_maintainers = self.get_module_maintainers()
if 'ansible' in notification_maintainers:
notification_maintainers.extend(self.ansible_members)
if 'ansibot' in notification_maintainers:
notification_maintainers.remove('ansibot')
maintainer_last_notified = self.history.\
last_notified(notification_maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
maintainer_last_viewed = self.history.last_viewed_at(maintainers)
#import epdb; epdb.st()
# Has maintainer been mentioned?
maintainer_mentioned = self.history.is_mentioned(maintainers)
# Has maintainer viewed issue?
maintainer_viewed = self.history.has_viewed(maintainers)
# Has the maintainer ever responded?
maintainer_commented = self.history.has_commented(maintainers)
maintainer_last_commented = self.history.last_commented_at(maintainers)
maintainer_last_comment = self.history.last_comment(maintainers)
maintainer_comments = self.history.get_user_comments(maintainers)
#import epdb; epdb.st()
# Was the maintainer the last commentor?
last_commentor_ismaintainer = False
last_commentor_issubmitter = False
last_commentor = self.history.last_commentor()
if last_commentor in maintainers and last_commentor != self.github_user:
last_commentor_ismaintainer = True
elif last_commentor == submitter:
last_commentor_issubmitter = True
# Did the maintainer issue a command?
maintainer_commands = self.history.get_commands(
maintainers, self.VALID_COMMANDS)
# needs_info toggles
ni_commands = [x for x in maintainer_commands if 'needs_info' in x]
# Has the maintainer ever subscribed?
maintainer_subscribed = self.history.has_subscribed(maintainers)
# Was it ever needs_info?
was_needs_info = self.history.was_labeled(label='needs_info')
needsinfo_last_applied = self.history.label_last_applied('needs_info')
needsinfo_last_removed = self.history.label_last_removed('needs_info')
# Still needs_info?
needsinfo_add = False
needsinfo_remove = False
if 'needs_info' in self.issue.current_labels:
if not needsinfo_last_applied or not submitter_last_commented:
import epdb
epdb.st()
if submitter_last_commented > needsinfo_last_applied:
needsinfo_add = False
needsinfo_remove = True
#if 'needs_info' in maintainer_commands and maintainer_last_commented:
if ni_commands and maintainer_last_commented:
if ni_commands[-1] == 'needs_info':
#import epdb; epdb.st()
if submitter_last_commented and maintainer_last_commented:
if submitter_last_commented > maintainer_last_commented:
needsinfo_add = False
needsinfo_remove = True
else:
needsinfo_add = True
needsinfo_remove = False
else:
needsinfo_add = False
needsinfo_remove = True
# Save existing needs_info if not time to remove ...
if 'needs_info' in self.issue.current_labels \
and not needsinfo_add \
and not needsinfo_remove:
needsinfo_add = True
if ni_commands and maintainer_last_commented:
if maintainer_last_commented > submitter_last_commented:
if ni_commands[-1] == 'needs_info':
needsinfo_add = True
needsinfo_remove = False
else:
needsinfo_add = False
needsinfo_remove = True
#import epdb; epdb.st()
# Is needs_info stale or expired?
needsinfo_age = None
needsinfo_stale = False
needsinfo_expired = False
if 'needs_info' in self.issue.current_labels:
time_delta = today - needsinfo_last_applied
needsinfo_age = time_delta.days
if needsinfo_age > 14:
needsinfo_stale = True
if needsinfo_age > 56:
needsinfo_expired = True
# Should we be in waiting_on_maintainer mode?
maintainer_waiting_on = False
if (needsinfo_remove or not needsinfo_add) \
or not was_needs_info \
and not missing_sections:
maintainer_waiting_on = True
# Should we [re]notify the submitter?
submitter_waiting_on = False
submitter_to_ping = False
submitter_to_reping = False
if not maintainer_waiting_on:
submitter_waiting_on = True
if missing_sections:
submitter_waiting_on = True
maintainer_waiting_on = False
# use [!]needs_info to set final state
if ni_commands:
if ni_commands[-1] == '!needs_info':
submitter_waiting_on = False
maintainer_waiting_on = True
elif ni_commands[-1] == 'needs_info':
submitter_waiting_on = True
maintainer_waiting_on = False
# Time to [re]ping maintainer?
maintainer_to_ping = False
maintainer_to_reping = False
if maintainer_waiting_on:
#import epdb; epdb.st()
if maintainer_viewed and not maintainer_last_notified:
time_delta = today - maintainer_last_viewed
view_age = time_delta.days
if view_age > 14:
maintainer_to_reping = True
elif maintainer_last_notified:
time_delta = today - maintainer_last_notified
ping_age = time_delta.days
if ping_age > 14:
maintainer_to_reping = True
else:
maintainer_to_ping = True
# Time to [re]ping the submitter?
if submitter_waiting_on:
if submitter_last_notified:
time_delta = today - submitter_last_notified
notification_age = time_delta.days
if notification_age > 14:
submitter_to_reping = True
else:
submitter_to_reping = False
submitter_to_ping = False
else:
submitter_to_ping = True
submitter_to_reping = False
hfacts['bot_broken'] = bot_broken
hfacts['missing_sections'] = missing_sections
hfacts['was_needsinfo'] = was_needs_info
hfacts['needsinfo_age'] = needsinfo_age
hfacts['needsinfo_stale'] = needsinfo_stale
hfacts['needsinfo_expired'] = needsinfo_expired
hfacts['needsinfo_add'] = needsinfo_add
hfacts['needsinfo_remove'] = needsinfo_remove
hfacts['notification_maintainers'] = self.get_module_maintainers(
) or 'ansible'
hfacts['maintainer_last_notified'] = maintainer_last_notified
hfacts['maintainer_commented'] = maintainer_commented
hfacts['maintainer_viewed'] = maintainer_viewed
hfacts['maintainer_subscribed'] = maintainer_subscribed
hfacts[
'maintainer_command_needsinfo'] = 'needs_info' in maintainer_commands
hfacts[
'maintainer_command_not_needsinfo'] = '!needs_info' in maintainer_commands
hfacts['maintainer_waiting_on'] = maintainer_waiting_on
hfacts['maintainer_to_ping'] = maintainer_to_ping
hfacts['maintainer_to_reping'] = maintainer_to_reping
hfacts['submitter'] = submitter
hfacts['submitter_waiting_on'] = submitter_waiting_on
hfacts['submitter_to_ping'] = submitter_to_ping
hfacts['submitter_to_reping'] = submitter_to_reping
hfacts['last_commentor_ismaintainer'] = last_commentor_ismaintainer
hfacts['last_commentor_issubmitter'] = last_commentor_issubmitter
hfacts['last_commentor'] = last_commentor
#import epdb; epdb.st()
return hfacts
def run(self, useapiwrapper=True):
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# make a list of valid assignees
print('Getting valid assignees')
self.valid_assignees = [x.login for x in self.repo.get_assignees()]
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
# get the issue
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo,
issue=issue,
cachedir=self.cachedir)
self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS
self.issue.valid_assignees = self.valid_assignees
self.issue.get_events()
self.issue.get_comments()
# get the PR and it's properties
self.issue.pullrequest = self.repo.get_pullrequest(int(
self.number))
self.issue.get_commits()
self.issue.get_files()
self.issue.get_review_comments()
# do the work
self.process()
else:
# need to get the PRs
print('Getting ALL pullrequests')
pullrequests = self.repo.get_pullrequests(since=None)
# iterate
for idp, pr in enumerate(pullrequests):
# get the issue and make a wrapper
issue = self.repo.get_issue(int(pr.number))
self.issue = IssueWrapper(repo=self.repo,
issue=issue,
cachedir=self.cachedir)
self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS
self.issue.valid_assignees = self.valid_assignees
self.issue.get_events()
self.issue.get_comments()
# get the PR and it's properties
self.issue.pullrequest = pr
self.issue.get_commits()
self.issue.get_files()
self.issue.get_review_comments()
# do the work
self.process()
class TriageV3(DefaultTriager):
def __init__(self, args):
self.args = args
self.last_run = None
self.daemonize = None
self.daemonize_interval = None
self.dry_run = False
self.force = False
self.gh_pass = None
self.github_pass = None
self.gh_token = None
self.github_token = None
self.gh_user = None
self.github_user = None
self.logfile = None
self.no_since = False
self.only_closed = False
self.only_issues = False
self.only_open = False
self.only_prs = False
self.pause = False
self.pr = False
self.repo = None
self.safe_force = False
self.skiprepo = []
self.start_at = False
self.verbose = False
# where to store junk
self.cachedir = '~/.ansibullbot/cache'
self.cachedir = os.path.expanduser(self.cachedir)
# repo objects
self.repos = {}
self.set_logger()
logging.info('starting bot')
logging.debug('setting bot attributes')
attribs = dir(self.args)
attribs = [x for x in attribs if not x.startswith('_')]
for x in attribs:
val = getattr(self.args, x)
setattr(self, x, val)
#import epdb; epdb.st()
if self.args.daemonize:
logging.info('starting daemonize loop')
self.loop()
else:
logging.info('starting single run')
self.run()
logging.info('stopping bot')
def set_logger(self):
logging.level = logging.INFO
logFormatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
rootLogger = logging.getLogger()
rootLogger.setLevel(logging.INFO)
fileHandler = logging.FileHandler("{0}/{1}".format(
os.path.dirname(self.args.logfile),
os.path.basename(self.args.logfile))
)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
def loop(self):
'''Call the run method in a defined interval'''
while True:
self.run()
interval = self.args.daemonize_interval
logging.info('sleep %ss (%sm)' % (interval, interval / 60))
time.sleep(interval)
def run(self):
'''Primary execution method'''
self.collect_issues()
#if self.last_run:
# wissues = self.get_updated_issues(since=self.last_run)
#else:
# wissues = self.get_updated_issues()
for item in self.repos.items():
repopath = item[0]
if repopath in self.skiprepo:
continue
repoobj = item[1]['repo']
issues = item[1]['issues']
for i_item in issues.items():
ik = i_item[0]
iw = i_item[1]
logging.info(iw)
if iw.state == 'closed':
logging.info(iw + ' is closed, skipping')
continue
#hcache = os.path.join(self.cachedir, iw.repo_full_name, 'issues')
hcache = os.path.join(self.cachedir, iw.repo_full_name)
action_meta = None
if iw.repo_full_name in MREPOS:
if iw.created_at >= REPOMERGEDATE:
# close new module issues+prs immediately
logging.info('module issue created -after- merge')
action_meta = self.close_module_issue_with_message(iw)
else:
# process history
# - check if message was given, comment if not
# - if X days after message, close PRs, move issues.
logging.info('module issue created -before- merge')
logging.info('build history')
hw = self.get_history(iw, usecache=True, cachedir=hcache)
logging.info('history built')
lc = hw.last_date_for_boilerplate('repomerge')
if lc:
lcdelta = (datetime.datetime.now() - lc).days
else:
lcdelta = None
kwargs = {}
# missing the comment?
if lc:
kwargs['bp'] = 'repomerge'
else:
kwargs['bp'] = None
# should it be closed or not?
if iw.is_pullrequest():
if lc and lcdelta > MREPO_CLOSE_WINDOW:
kwargs['close'] = True
action_meta = self.close_module_issue_with_message(iw, **kwargs)
elif not lc:
# add the comment
self.add_repomerge_comment(iw)
else:
# do nothing
pass
else:
kwargs['close'] = False
if lc and lcdelta > MREPO_CLOSE_WINDOW:
# move it for them
self.move_issue(iw)
elif not lc:
# add the comment
self.add_repomerge_comment(iw)
else:
# do nothing
pass
else:
# ansible/ansible triage
pass
pprint(action_meta)
#import epdb; epdb.st()
def move_issue(self, issue):
pass
def add_repomerge_comment(self, issue, bp='repomerge'):
'''Add the comment without closing'''
self.actions = {}
self.actions['close'] = False
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
# stubs for the comment templater
self.module_maintainers = []
self.module = None
self.issue = issue
self.template_data = {}
self.github_repo = issue.repo_full_name
self.match = {}
comment = self.render_comment(boilerplate=bp)
self.actions['comments'] = [comment]
pprint(self.actions)
action_meta = self.apply_actions()
return action_meta
def close_module_issue_with_message(self, issue, bp='repomerge_new'):
'''After the repomerge, new issues+prs in the module repos should be closed'''
self.actions = {}
self.actions['close'] = True
self.actions['comments'] = []
self.actions['newlabel'] = []
self.actions['unlabel'] = []
# stubs for the comment templater
self.module_maintainers = []
self.module = None
self.issue = issue
self.template_data = {}
self.github_repo = issue.repo_full_name
self.match = {}
comment = self.render_comment(boilerplate=bp)
self.actions['comments'] = [comment]
pprint(self.actions)
action_meta = self.apply_actions()
return action_meta
def collect_issues(self):
'''Populate the local cache of issues'''
# this should do a few things:
# 1. collect all issues (open+closed) via a webcrawler
# 2. index the issues into a sqllite database so we can query on update times
# 3. set an abstracted object that takes in queries
logging.info('start collecting issues')
logging.debug('creating github connection object')
self.gh = self._connect()
logging.info('creating github connection wrapper')
self.ghw = GithubWrapper(self.gh)
for repo in REPOS:
if repo in self.skiprepo:
continue
#import epdb; epdb.st()
logging.info('getting repo obj for %s' % repo)
cachedir = os.path.join(self.cachedir, repo)
self.repos[repo] = {}
self.repos[repo]['repo'] = self.ghw.get_repo(repo, verbose=False)
self.repos[repo]['issues'] = {}
logging.info('getting issue objs for %s' % repo)
issues = self.repos[repo]['repo'].get_issues()
for issue in issues:
iw = IssueWrapper(
repo=self.repos[repo]['repo'],
issue=issue,
cachedir=cachedir
)
self.repos[repo]['issues'][iw.number] = iw
logging.info('getting issue objs for %s complete' % repo)
logging.info('finished collecting issues')
def get_updated_issues(self, since=None):
'''Get issues to work on'''
# this should return a list of issueids that have changed since the last run
logging.info('start querying updated issues')
# these need to be tuples (namespace, repo, number)
issueids = []
logging.info('finished querying updated issues')
return issueids
def get_history(self, issue, usecache=True, cachedir=None):
history = HistoryWrapper(issue, usecache=usecache, cachedir=cachedir)
return history
def run(self, useapiwrapper=True):
# how many issues have been processed
self.icount = 0
# Create the api connection
if not useapiwrapper:
# use the default non-caching connection
self.repo = self._connect().get_repo(self._get_repo_path())
else:
# make use of the special caching wrapper for the api
self.gh = self._connect()
self.ghw = GithubWrapper(self.gh)
self.repo = self.ghw.get_repo(self._get_repo_path())
# make a list of valid assignees
print('Getting valid assignees')
self.valid_assignees = [x.login for x in self.repo.get_assignees()]
# extend the ignored labels by repo
if hasattr(self, 'IGNORE_LABELS_ADD'):
self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD)
if self.number:
# get the issue
issue = self.repo.get_issue(int(self.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS
self.issue.valid_assignees = self.valid_assignees
self.issue.get_events()
self.issue.get_comments()
# get the PR and it's properties
self.issue.pullrequest = self.repo.get_pullrequest(int(self.number))
self.issue.get_commits()
self.issue.get_files()
self.issue.get_review_comments()
# do the work
self.process()
else:
# need to get the PRs
print('Getting ALL pullrequests')
pullrequests = self.repo.get_pullrequests(since=None)
# iterate
for idp,pr in enumerate(pullrequests):
# get the issue and make a wrapper
issue = self.repo.get_issue(int(pr.number))
self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir)
self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS
self.issue.valid_assignees = self.valid_assignees
self.issue.get_events()
self.issue.get_comments()
# get the PR and it's properties
self.issue.pullrequest = pr
self.issue.get_commits()
self.issue.get_files()
self.issue.get_review_comments()
# do the work
self.process()
