def get_valid_labels(self): # use the repo wrapper to enable caching+updating self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) vlabels = [] for vl in self.repo.get_labels(): vlabels.append(vl.name) return vlabels
def collect_issues(self): '''Populate the local cache of issues''' # this should do a few things: # 1. collect all issues (open+closed) via a webcrawler # 2. index the issues into a sqllite database so we can query on update times # 3. set an abstracted object that takes in queries logging.info('start collecting issues') logging.debug('creating github connection object') self.gh = self._connect() logging.info('creating github connection wrapper') self.ghw = GithubWrapper(self.gh) for repo in REPOS: if repo in self.skiprepo: continue #import epdb; epdb.st() logging.info('getting repo obj for %s' % repo) cachedir = os.path.join(self.cachedir, repo) self.repos[repo] = {} self.repos[repo]['repo'] = self.ghw.get_repo(repo, verbose=False) self.repos[repo]['issues'] = {} logging.info('getting issue objs for %s' % repo) issues = self.repos[repo]['repo'].get_issues() for issue in issues: iw = IssueWrapper( repo=self.repos[repo]['repo'], issue=issue, cachedir=cachedir ) self.repos[repo]['issues'][iw.number] = iw logging.info('getting issue objs for %s complete' % repo) logging.info('finished collecting issues')
def get_valid_labels(self, repo=None): # use the repo wrapper to enable caching+updating if not self.ghw: self.gh = self._connect() self.ghw = GithubWrapper(self.gh) if not repo: # OLD workflow self.repo = self.ghw.get_repo(self._get_repo_path()) vlabels = [] for vl in self.repo.get_labels(): vlabels.append(vl.name) else: # v3 workflow rw = self.ghw.get_repo(repo) vlabels = [] for vl in rw.get_labels(): vlabels.append(vl.name) return vlabels
def run(self, useapiwrapper=True): """Starts a triage run""" # how many issues have been processed self.icount = 0 # Create the api connection if not useapiwrapper: # use the default non-caching connection self.repo = self._connect().get_repo(self._get_repo_path()) else: # make use of the special caching wrapper for the api self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) # extend the ignored labels by repo if hasattr(self, 'IGNORE_LABELS_ADD'): self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD) if self.number: issue = self.repo.get_issue(int(self.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() self.process() else: last_run = None now = self.get_current_time() last_run_file = '~/.ansibullbot/cache' if self.github_repo == 'ansible': last_run_file += '/ansible/ansible/' else: last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo last_run_file += 'issues/last_run.pickle' last_run_file = os.path.expanduser(last_run_file) if os.path.isfile(last_run_file): try: with open(last_run_file, 'rb') as f: last_run = pickle.load(f) except Exception as e: print(e) if last_run and not self.no_since: self.debug('Getting issues updated/created since %s' % last_run) issues = self.repo.get_issues(since=last_run) else: self.debug('Getting ALL issues') issues = self.repo.get_issues() for issue in issues: self.icount += 1 if self.start_at and issue.number > self.start_at: continue if self.is_pr(issue): continue self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if action_res: while action_res['REDO']: issue = self.repo.get_issue(int(issue.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if not action_res: action_res = {'REDO': False} # save this run time with open(last_run_file, 'wb') as f: pickle.dump(now, f)
class TriageIssues(DefaultTriager): VALID_COMMANDS = ['needs_info', '!needs_info', 'notabug', 'bot_broken', 'bot_skip', 'wontfix', 'bug_resolved', 'resolved_by_pr', 'needs_contributor', 'duplicate_of'] CLOSURE_COMMANDS = [ 'notabug', 'wontfix', 'bug_resolved', 'resolved_by_pr', 'duplicate_of' ] # re-notify interval for maintainers RENOTIFY_INTERVAL = 14 # max limit for needs_info notifications RENOTIFY_EXPIRE = 56 # re-notify interval by this number for features FEATURE_RENOTIFY_INTERVAL = 60 # the max comments per week before ansibot becomes a "spammer" MAX_BOT_COMMENTS_PER_WEEK = 5 def run(self, useapiwrapper=True): """Starts a triage run""" # how many issues have been processed self.icount = 0 # Create the api connection if not useapiwrapper: # use the default non-caching connection self.repo = self._connect().get_repo(self._get_repo_path()) else: # make use of the special caching wrapper for the api self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) # extend the ignored labels by repo if hasattr(self, 'IGNORE_LABELS_ADD'): self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD) if self.number: issue = self.repo.get_issue(int(self.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() self.process() else: last_run = None now = self.get_current_time() last_run_file = '~/.ansibullbot/cache' if self.github_repo == 'ansible': last_run_file += '/ansible/ansible/' else: last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo last_run_file += 'issues/last_run.pickle' last_run_file = os.path.expanduser(last_run_file) if os.path.isfile(last_run_file): try: with open(last_run_file, 'rb') as f: last_run = pickle.load(f) except Exception as e: print(e) if last_run and not self.no_since: self.debug('Getting issues updated/created since %s' % last_run) issues = self.repo.get_issues(since=last_run) else: self.debug('Getting ALL issues') issues = self.repo.get_issues() for issue in issues: self.icount += 1 if self.start_at and issue.number > self.start_at: continue if self.is_pr(issue): continue self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if action_res: while action_res['REDO']: issue = self.repo.get_issue(int(issue.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if not action_res: action_res = {'REDO': False} # save this run time with open(last_run_file, 'wb') as f: pickle.dump(now, f) @RateLimited def process(self, usecache=True): """Processes the Issue""" # basic processing self._process() # who maintains this? maintainers = [] if self.meta.get('component_valid', False): correct_repo = self.match.get('repository') if correct_repo != self.github_repo: self.meta['correct_repo'] = False else: self.meta['correct_repo'] = True maintainers = self.get_module_maintainers() if not maintainers: #issue_module_no_maintainer #import epdb; epdb.st() pass ########################################################### # Enumerate Actions ########################################################### self.keep_current_main_labels() self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.process_history(usecache=usecache) self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.create_actions() self.debug(msg='desired_comments: %s' % self.issue.desired_comments) print("Module: %s" % self.module) if self.match: print("MModule: %s" % self.match['name']) else: print("MModule: %s" % self.match) print("Maintainer(s): %s" \ % ', '.join(self.get_module_maintainers(expand=False))) print("Submitter: %s" % self.issue.get_submitter()) print("Total Comments: %s" % len(self.issue.current_comments)) self.print_comment_list() print("Current Labels: %s" % ', '.join(sorted(self.issue.current_labels))) # invoke the wizard import pprint; pprint.pprint(self.actions) action_meta = self.apply_actions() return action_meta def create_actions(self): """Create actions from the desired label/unlabel/comment actions""" # do nothing for bot_skip if self.meta['bot_skip']: return if 'bot_broken' in self.issue.desired_labels: # If the bot is broken, do nothing other than set the broken label self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] self.actions['close'] = False if not 'bot_broken' in self.issue.current_labels: self.actions['newlabel'] = ['bot_broken'] return # add the version labels self.create_label_version_actions() if self.issue.desired_state != self.issue.instance.state: if self.issue.desired_state == 'closed': # close the issue ... self.actions['close'] = True # We only want up to 1 comment when an issue is closed if 'issue_closure' in self.issue.desired_comments: self.issue.desired_comments = ['issue_closure'] elif 'issue_deprecated_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_deprecated_module'] else: self.issue.desired_comments = [] if self.issue.desired_comments: comment = self.render_comment( boilerplate=self.issue.desired_comments[0] ) self.actions['comments'].append(comment) return resolved_desired_labels = [] for desired_label in self.issue.desired_labels: resolved_desired_label = self.issue.resolve_desired_labels( desired_label ) if desired_label != resolved_desired_label: resolved_desired_labels.append(resolved_desired_label) if (resolved_desired_label not in self.issue.get_current_labels()): self.issue.add_desired_comment(desired_label) self.actions['newlabel'].append(resolved_desired_label) else: resolved_desired_labels.append(desired_label) if desired_label not in self.issue.get_current_labels(): self.actions['newlabel'].append(desired_label) ''' if os.path.exists("templates/" + 'issue_' + desired_label + ".j2"): self.issue.add_desired_comment('issue_' + desired_label) ''' for current_label in self.issue.get_current_labels(): if current_label in self.IGNORE_LABELS: continue if current_label not in resolved_desired_labels: self.actions['unlabel'].append(current_label) # should only make one comment at a time if len(self.issue.desired_comments) > 1: if 'issue_wrong_repo' in self.issue.desired_comments: self.issue.desired_comments = ['issue_wrong_repo'] elif 'issue_invalid_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_invalid_module'] elif 'issue_module_no_maintainer' in self.issue.desired_comments: self.issue.desired_comments = ['issue_module_no_maintainer'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' in self.issue.desired_labels: self.issue.desired_comments = ['issue_needs_info'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' not in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') else: import epdb; epdb.st() # Do not comment needs_info if it's not a label if 'issue_needs_info' in self.issue.desired_comments \ and not 'needs_info' in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') # render the comments for boilerplate in self.issue.desired_comments: comment = self.render_comment(boilerplate=boilerplate) self.debug(msg=boilerplate) self.actions['comments'].append(comment) # do not re-comment for idx,comment in enumerate(self.actions['comments']): if self.issue.current_comments: if self.issue.current_comments[-1].body == comment: self.debug(msg="Removing repeat comment from actions") self.actions['comments'].remove(comment) #import epdb; epdb.st() def create_label_version_actions(self): if not self.ansible_label_version: return expected = 'affects_%s' % self.ansible_label_version if expected not in self.valid_labels: print("NEED NEW LABEL: %s" % expected) import epdb; epdb.st() candidates = [x for x in self.issue.current_labels] candidates = [x for x in candidates if x.startswith('affects_')] candidates = [x for x in candidates if x != expected] if len(candidates) > 0: #for cand in candidates: # self.issue.pop_desired_label(name=cand) #return pass else: if expected not in self.issue.current_labels \ or expected not in self.issue.desired_labels: self.issue.add_desired_label(name=expected) #import epdb; epdb.st() def create_commment_actions(self): '''Render desired comment templates''' # should only make one comment at a time if len(self.issue.desired_comments) > 1: if 'issue_wrong_repo' in self.issue.desired_comments: self.issue.desired_comments = ['issue_wrong_repo'] elif 'issue_invalid_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_invalid_module'] elif 'issue_module_no_maintainer' in self.issue.desired_comments: self.issue.desired_comments = ['issue_module_no_maintainer'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' in self.issue.desired_labels: self.issue.desired_comments = ['issue_needs_info'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' not in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') else: import epdb; epdb.st() # Do not comment needs_info if it's not a label if 'issue_needs_info' in self.issue.desired_comments \ and not 'needs_info' in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') # render the comments for boilerplate in self.issue.desired_comments: comment = self.render_comment(boilerplate=boilerplate) self.debug(msg=boilerplate) self.actions['comments'].append(comment) # do not re-comment for idx,comment in enumerate(self.actions['comments']): if self.issue.current_comments: if self.issue.current_comments[-1].body == comment: self.debug(msg="Removing repeat comment from actions") self.actions['comments'].remove(comment) def create_label_actions(self): """Create actions from the desired label/unlabel/comment actions""" if 'bot_broken' in self.issue.desired_labels: # If the bot is broken, do nothing other than set the broken label self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] self.actions['close'] = False if not 'bot_broken' in self.issue.current_labels: self.actions['newlabel'] = ['bot_broken'] return if self.meta['bot_skip']: return resolved_desired_labels = [] for desired_label in self.issue.desired_labels: resolved_desired_label = self.issue.resolve_desired_labels( desired_label ) if desired_label != resolved_desired_label: resolved_desired_labels.append(resolved_desired_label) if (resolved_desired_label not in self.issue.get_current_labels()): self.issue.add_desired_comment(desired_label) self.actions['newlabel'].append(resolved_desired_label) else: resolved_desired_labels.append(desired_label) if desired_label not in self.issue.get_current_labels(): self.actions['newlabel'].append(desired_label) for current_label in self.issue.get_current_labels(): if current_label in self.IGNORE_LABELS: continue if current_label not in resolved_desired_labels: self.actions['unlabel'].append(current_label) def process_history(self, usecache=True): '''Steps through all known meta about the issue and decides what to do''' self.meta.update(self.get_facts()) self.add_desired_labels_by_issue_type() self.add_desired_labels_by_ansible_version() self.add_desired_labels_by_namespace() ################################################# # FINAL LOGIC LOOP ################################################# if self.meta['bot_broken']: self.debug(msg='broken bot stanza') self.issue.add_desired_label('bot_broken') elif self.meta['bot_skip']: self.debug(msg='bot skip stanza') # clear out all actions and do nothing for k,v in self.actions.iteritems(): if type(v) == list: self.actions[k] = [] self.actions['close'] = False elif self.meta['bot_spam']: self.debug(msg='bot spam stanza') # clear out all actions and do nothing for k,v in self.actions.iteritems(): if type(v) == list: self.actions[k] = [] self.actions['close'] = False # do we mark this somehow? self.issue.add_desired_label('bot_broken') elif self.match and self.match.get('deprecated', False) \ and 'feature_idea' in self.issue.desired_labels: self.debug(msg='deprecated module stanza') # Make the deprecated comment self.issue.desired_comments = ['issue_deprecated_module'] # Close the issue ... self.issue.set_desired_state('closed') elif not self.meta['maintainers_known'] and self.meta['valid_module']: self.debug(msg='unknown maintainer stanza') self.issue.desired_comments = ['issue_module_no_maintainer'] elif self.meta['maintainer_closure']: self.debug(msg='maintainer closure stanza') # Need to close the issue ... self.issue.set_desired_state('closed') elif self.meta['new_module_request']: self.debug(msg='new module request stanza') self.issue.desired_comments = [] for label in self.issue.current_labels: if not label in self.issue.desired_labels: self.issue.desired_labels.append(label) elif not self.meta['correct_repo']: self.debug(msg='wrong repo stanza') self.issue.desired_comments = ['issue_wrong_repo'] self.actions['close'] = True elif not self.meta['valid_module'] and \ not self.meta['maintainer_command_needsinfo']: self.debug(msg='invalid module stanza') self.issue.add_desired_label('needs_info') if 'issue_invalid_module' not in self.issue.current_bot_comments \ and not 'issue_needs_info' in self.issue.current_bot_comments: self.issue.desired_comments = ['issue_invalid_module'] elif not self.meta['notification_maintainers'] and \ not self.meta['maintainer_command_needsinfo']: self.debug(msg='no maintainer stanza') self.issue.add_desired_label('waiting_on_maintainer') self.issue.add_desired_comment("issue_module_no_maintainer") elif self.meta['maintainer_command'] == 'needs_contributor': # maintainer can't or won't fix this, but would like someone else to self.debug(msg='maintainer needs contributor stanza') self.issue.add_desired_label('waiting_on_contributor') elif self.meta['maintainer_waiting_on']: self.debug(msg='maintainer wait stanza') self.issue.add_desired_label('waiting_on_maintainer') if len(self.issue.current_comments) == 0: # new issue if self.meta['issue_type']: if self.meta['submitter'] not in self.meta['notification_maintainers']: # ping the maintainer self.issue.add_desired_comment('issue_new') else: # do not send intial ping to maintainer if also submitter if 'issue_new' in self.issue.desired_comments: self.issue.desired_comments.remove('issue_new') else: # old issue -- renotify if not self.match['deprecated'] and self.meta['notification_maintainers']: if self.meta['maintainer_to_ping']: self.issue.add_desired_comment("issue_notify_maintainer") elif self.meta['maintainer_to_reping']: self.issue.add_desired_comment("issue_renotify_maintainer") elif self.meta['submitter_waiting_on']: self.debug(msg='submitter wait stanza') if 'waiting_on_maintainer' in self.issue.desired_labels: self.issue.desired_labels.remove('waiting_on_maintainer') if (self.meta['needsinfo_add'] or self.meta['missing_sections']) \ or (not self.meta['needsinfo_remove'] and self.meta['missing_sections']) \ or (self.meta['needsinfo_add'] and not self.meta['missing_sections']): #import epdb; epdb.st() self.issue.add_desired_label('needs_info') if len(self.issue.current_comments) == 0 or \ not self.meta['maintainer_commented']: if self.issue.current_bot_comments: if 'issue_needs_info' not in self.issue.current_bot_comments: self.issue.add_desired_comment("issue_needs_info") else: self.issue.add_desired_comment("issue_needs_info") # needs_info: warn if stale, close if expired elif self.meta['needsinfo_expired']: self.issue.add_desired_comment("issue_closure") self.issue.set_desired_state('closed') elif self.meta['needsinfo_stale'] \ and (self.meta['submitter_to_ping'] or self.meta['submitter_to_reping']): self.issue.add_desired_comment("issue_pending_closure") def get_history_facts(self, usecache=True): return self.get_facts(usecache=usecache) def get_facts(self, usecache=True): '''Only used by the ansible/ansible triager at the moment''' hfacts = {} today = self.get_current_time() self.history = HistoryWrapper( self.issue, usecache=usecache, cachedir=self.cachedir ) # what was the last commment? bot_broken = False if self.issue.current_comments: for comment in self.issue.current_comments: if 'bot_broken' in comment.body: bot_broken = True # did someone from ansible want this issue skipped? bot_skip = False if self.issue.current_comments: for comment in self.issue.current_comments: if 'bot_skip' in comment.body and comment.user.login in self.ansible_members: bot_skip = True ######################################################### # Facts for new "triage" workflow ######################################################### # these logins don't count for the next set of facts exclude_logins = ['ansibot', 'gregdek', self.issue.instance.user.login] # Has a human (not ansibot + not submitter) ever changed labels? lchanges = [x for x in self.history.history if x['event'] in ['labeled', 'unlabeled'] and x['actor'] not in exclude_logins] if lchanges: hfacts['human_labeled'] = True else: hfacts['human_labeled'] = False # Has a human (not ansibot + not submitter) ever changed the title? tchanges = [x for x in self.history.history if x['event'] in ['renamed'] and x['actor'] not in exclude_logins] if tchanges: hfacts['human_retitled'] = True else: hfacts['human_retitled'] = False # Has a human (not ansibot + not submitter) ever changed the description? # ... github doesn't record an event for this. Lame. # Has a human (not ansibot + not submitter) ever set a milestone? mchanges = [x for x in self.history.history if x['event'] in ['milestoned', 'demilestoned'] and x['actor'] not in exclude_logins] if mchanges: hfacts['human_milestoned'] = True else: hfacts['human_milestoned'] = False # Has a human (not ansibot + not submitter) ever set/unset assignees? achanges = [x for x in self.history.history if x['event'] in ['assigned', 'unassigned'] and x['actor'] not in exclude_logins] if achanges: hfacts['human_assigned'] = True else: hfacts['human_assigned'] = False ######################################################### # Has the bot been overzealous with comments? hfacts['bot_spam'] = False bcg = self.history.get_user_comments_groupby('ansibot', groupby='w') for k,v in bcg.iteritems(): if v >= self.MAX_BOT_COMMENTS_PER_WEEK: hfacts['bot_spam'] = True # Is this a new module? hfacts['new_module_request'] = False if 'feature_idea' in self.issue.desired_labels: if self.template_data['component name'] == 'new': hfacts['new_module_request'] = True # who made this and when did they last comment? submitter = self.issue.get_submitter() submitter_last_commented = self.history.last_commented_at(submitter) if not submitter_last_commented: submitter_last_commented = self.issue.instance.created_at #import epdb; epdb.st() submitter_last_comment = self.history.last_comment(submitter) submitter_last_notified = self.history.last_notified(submitter) # what did they not provide? missing_sections = self.issue.get_missing_sections() # Is this a valid module? if self.match: self.meta['valid_module'] = True else: self.meta['valid_module'] = False # Filed in the right place? if self.meta['valid_module']: if self.match['repository'] != self.github_repo: hfacts['correct_repo'] = False else: hfacts['correct_repo'] = True else: hfacts['correct_repo'] = True # DEBUG + FIXME - speeds up bulk triage if 'component name' in missing_sections \ and (self.match or self.github_repo == 'ansible'): missing_sections.remove('component name') #import epdb; epdb.st() # Who are the maintainers? maintainers = [x for x in self.get_module_maintainers()] #hfacts['maintainers'] = maintainers #import epdb; epdb.st() # Set a fact to indicate that we know the maintainer self.meta['maintainers_known'] = False if maintainers: self.meta['maintainers_known'] = True if 'ansible' in maintainers: maintainers.remove('ansible') maintainers.extend(self.ansible_members) if 'ansibot' in maintainers: maintainers.remove('ansibot') if submitter in maintainers: maintainers.remove(submitter) maintainers = sorted(set(maintainers)) # Has maintainer been notified? When? notification_maintainers = [x for x in self.get_module_maintainers()] if 'ansible' in notification_maintainers: notification_maintainers.extend(self.ansible_members) if 'ansibot' in notification_maintainers: notification_maintainers.remove('ansibot') hfacts['notification_maintainers'] = notification_maintainers maintainer_last_notified = self.history.\ last_notified(notification_maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) maintainer_last_viewed = self.history.last_viewed_at(maintainers) #import epdb; epdb.st() # Has maintainer been mentioned? maintainer_mentioned = self.history.is_mentioned(maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) # Has the maintainer ever responded? maintainer_commented = self.history.has_commented(maintainers) maintainer_last_commented = self.history.last_commented_at(maintainers) maintainer_last_comment = self.history.last_comment(maintainers) maintainer_comments = self.history.get_user_comments(maintainers) #import epdb; epdb.st() # Was the maintainer the last commentor? last_commentor_ismaintainer = False last_commentor_issubmitter = False last_commentor = self.history.last_commentor() if last_commentor in maintainers and last_commentor != self.github_user: last_commentor_ismaintainer = True elif last_commentor == submitter: last_commentor_issubmitter = True # Did the maintainer issue a command? maintainer_commands = self.history.get_commands(maintainers, self.VALID_COMMANDS) # Keep all commands hfacts['maintainer_commands'] = maintainer_commands # Set a bit for the last command given if hfacts['maintainer_commands']: hfacts['maintainer_command'] = hfacts['maintainer_commands'][-1] else: hfacts['maintainer_command'] = None # Is the last command a closure command? if hfacts['maintainer_command'] in self.CLOSURE_COMMANDS: hfacts['maintainer_closure'] = True else: hfacts['maintainer_closure'] = False # handle resolved_by_pr ... if 'resolved_by_pr' in maintainer_commands: maintainer_comments = self.history.get_user_comments(maintainers) maintainer_comments = [x for x in reversed(maintainer_comments) \ if 'resolved_by_pr' in x] for comment in maintainer_comments: pr_number = extract_pr_number_from_comment(comment) hfacts['resolved_by_pr'] = { 'number': pr_number, 'merged': self.is_pr_merged(pr_number), } if not hfacts['resolved_by_pr']['merged']: hfacts['maintainer_closure'] = False break # needs_info toggles ni_commands = [x for x in maintainer_commands if 'needs_info' in x] # Has the maintainer ever subscribed? maintainer_subscribed = self.history.has_subscribed(maintainers) # Was it ever needs_info? was_needs_info = self.history.was_labeled(label='needs_info') needsinfo_last_applied = self.history.label_last_applied('needs_info') needsinfo_last_removed = self.history.label_last_removed('needs_info') # Still needs_info? needsinfo_add = False needsinfo_remove = False if 'needs_info' in self.issue.current_labels: if not needsinfo_last_applied or not submitter_last_commented: import epdb; epdb.st() if submitter_last_commented > needsinfo_last_applied: needsinfo_add = False needsinfo_remove = True #if 'needs_info' in maintainer_commands and maintainer_last_commented: if ni_commands and maintainer_last_commented: if ni_commands[-1] == 'needs_info': #import epdb; epdb.st() if submitter_last_commented and maintainer_last_commented: if submitter_last_commented > maintainer_last_commented: needsinfo_add = False needsinfo_remove = True else: needsinfo_add = True needsinfo_remove = False else: needsinfo_add = False needsinfo_remove = True # Save existing needs_info if not time to remove ... if 'needs_info' in self.issue.current_labels \ and not needsinfo_add \ and not needsinfo_remove: needsinfo_add = True if ni_commands and maintainer_last_commented: if maintainer_last_commented > submitter_last_commented: if ni_commands[-1] == 'needs_info': needsinfo_add = True needsinfo_remove = False else: needsinfo_add = False needsinfo_remove = True #import epdb; epdb.st() # Is needs_info stale or expired? needsinfo_age = None needsinfo_stale = False needsinfo_expired = False if 'needs_info' in self.issue.current_labels: time_delta = today - needsinfo_last_applied needsinfo_age = time_delta.days if needsinfo_age > self.RENOTIFY_INTERVAL: needsinfo_stale = True if needsinfo_age > self.RENOTIFY_EXPIRE: needsinfo_expired = True # Should we be in waiting_on_maintainer mode? maintainer_waiting_on = False if (needsinfo_remove or not needsinfo_add) \ or not was_needs_info \ and not missing_sections: maintainer_waiting_on = True # Should we [re]notify the submitter? submitter_waiting_on = False submitter_to_ping = False submitter_to_reping = False if not maintainer_waiting_on: submitter_waiting_on = True if missing_sections: submitter_waiting_on = True maintainer_waiting_on = False # use [!]needs_info to set final state if ni_commands: if ni_commands[-1] == '!needs_info': submitter_waiting_on = False maintainer_waiting_on = True elif ni_commands[-1] == 'needs_info': submitter_waiting_on = True maintainer_waiting_on = False # Time to [re]ping maintainer? maintainer_to_ping = False maintainer_to_reping = False if maintainer_waiting_on: # if feature idea, extend the notification interval interval = self.RENOTIFY_INTERVAL if self.meta.get('issue_type', None) == 'feature idea' \ or 'feature_idea' in self.issue.current_labels: interval = self.FEATURE_RENOTIFY_INTERVAL if maintainer_viewed and not maintainer_last_notified: time_delta = today - maintainer_last_viewed view_age = time_delta.days if view_age > interval: maintainer_to_reping = True elif maintainer_last_notified: time_delta = today - maintainer_last_notified ping_age = time_delta.days if ping_age > interval: maintainer_to_reping = True else: maintainer_to_ping = True # Time to [re]ping the submitter? if submitter_waiting_on: if submitter_last_notified: time_delta = today - submitter_last_notified notification_age = time_delta.days if notification_age > self.RENOTIFY_INTERVAL: submitter_to_reping = True else: submitter_to_reping = False submitter_to_ping = False else: submitter_to_ping = True submitter_to_reping = False # needs_contributor ... hfacts['needs_contributor'] = False for command in maintainer_commands: if command == 'needs_contributor': hfacts['needs_contributor'] = True elif command == '!needs_contributor': hfacts['needs_contributor'] = False hfacts['bot_broken'] = bot_broken hfacts['bot_skip'] = bot_skip hfacts['missing_sections'] = missing_sections hfacts['was_needsinfo'] = was_needs_info hfacts['needsinfo_age'] = needsinfo_age hfacts['needsinfo_stale'] = needsinfo_stale hfacts['needsinfo_expired'] = needsinfo_expired hfacts['needsinfo_add'] = needsinfo_add hfacts['needsinfo_remove'] = needsinfo_remove hfacts['notification_maintainers'] = self.get_module_maintainers() or 'ansible' hfacts['maintainer_last_notified'] = maintainer_last_notified hfacts['maintainer_commented'] = maintainer_commented hfacts['maintainer_viewed'] = maintainer_viewed hfacts['maintainer_subscribed'] = maintainer_subscribed hfacts['maintainer_command_needsinfo'] = 'needs_info' in maintainer_commands hfacts['maintainer_command_not_needsinfo'] = '!needs_info' in maintainer_commands hfacts['maintainer_waiting_on'] = maintainer_waiting_on hfacts['maintainer_to_ping'] = maintainer_to_ping hfacts['maintainer_to_reping'] = maintainer_to_reping hfacts['submitter'] = submitter hfacts['submitter_waiting_on'] = submitter_waiting_on hfacts['submitter_to_ping'] = submitter_to_ping hfacts['submitter_to_reping'] = submitter_to_reping hfacts['last_commentor_ismaintainer'] = last_commentor_ismaintainer hfacts['last_commentor_issubmitter'] = last_commentor_issubmitter hfacts['last_commentor'] = last_commentor return hfacts
class DefaultTriager(object): BOTLIST = ['gregdek', 'robynbergeron', 'ansibot'] VALID_ISSUE_TYPES = ['bug report', 'feature idea', 'documentation report'] IGNORE_LABELS = [ "aws","azure","cloud", "feature_pull_request", "feature_idea", "bugfix_pull_request", "bug_report", "docs_pull_request", "docs_report", "in progress", "docs_pull_request", "easyfix", "pending_action", "gce", "python3", "P1","P2","P3","P4", ] def __init__(self, verbose=None, github_user=None, github_pass=None, github_token=None, github_repo=None, number=None, start_at=None, always_pause=False, force=False, safe_force=False, dry_run=False, no_since=False): self.verbose = verbose self.github_user = github_user self.github_pass = github_pass self.github_token = github_token self.github_repo = github_repo self.number = number self.start_at = start_at self.always_pause = always_pause self.force = force self.safe_force = safe_force self.dry_run = dry_run self.no_since = no_since self.issue = None self.maintainers = {} self.module_maintainers = [] self.actions = { 'newlabel': [], 'unlabel': [], 'comments': [], 'close': False, } # set the cache dir self.cachedir = '~/.ansibullbot/cache' if self.github_repo == 'ansible': self.cachedir += '/ansible/ansible/' else: self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo self.cachedir += 'issues' self.cachedir = os.path.expanduser(self.cachedir) if not os.path.isdir(self.cachedir): os.makedirs(self.cachedir) print("Initializing AnsibleVersionIndexer") self.version_indexer = AnsibleVersionIndexer() #import epdb; epdb.st() print("Initializing ModuleIndexer") self.module_indexer = ModuleIndexer() self.module_indexer.get_ansible_modules() print("Initializing FileIndexer") self.file_indexer = FileIndexer() self.file_indexer.get_files() print("Getting ansible members") self.ansible_members = self.get_ansible_members() print("Getting valid labels") self.valid_labels = self.get_valid_labels() # processed metadata self.meta = {} def _process(self, usecache=True): '''Do some initial processing of the issue''' # clear all actions self.actions = { 'newlabel': [], 'unlabel': [], 'comments': [], 'close': False, } # clear module maintainers self.module_maintainers = [] # print some general info about the Issue to be processed print("\n") print("Issue #%s [%s]: %s" % (self.issue.number, self.icount, (self.issue.instance.title).encode('ascii','ignore'))) print("%s" % self.issue.instance.html_url) print("Created at %s" % self.issue.instance.created_at) print("Updated at %s" % self.issue.instance.updated_at) # get the template data self.template_data = self.issue.get_template_data() # was the issue type defined correctly? issue_type_defined = False issue_type_valid = False issue_type = False if 'issue type' in self.template_data: issue_type_defined = True issue_type = self.template_data['issue type'] if issue_type.lower() in self.VALID_ISSUE_TYPES: issue_type_valid = True self.meta['issue_type_defined'] = issue_type_defined self.meta['issue_type_valid'] = issue_type_valid self.meta['issue_type'] = issue_type if self.meta['issue_type_valid']: self.meta['issue_type_label'] = self.issue_type_to_label(issue_type) else: self.meta['issue_type_label'] = None # What is the ansible version? self.ansible_version = self.get_ansible_version() if not isinstance(self.debug, bool): self.debug('version: %s' % self.ansible_version) self.ansible_label_version = self.get_ansible_version_major_minor() if not isinstance(self.debug, bool): self.debug('lversion: %s' % self.ansible_label_version) # was component specified? component_defined = 'component name' in self.template_data self.meta['component_defined'] = component_defined # extract the component component = self.template_data.get('component name', None) # save the real name if self.github_repo != 'ansible': self.match = self.module_indexer.find_match(component) or {} else: self.match = self.module_indexer.find_match(component, exact=True) or {} self.module = self.match.get('name', None) # check if component is a known module component_isvalid = self.module_indexer.is_valid(component) self.meta['component_valid'] = component_isvalid # smart match modules (only on module repos) if not component_isvalid and self.github_repo != 'ansible' and not self.match: #smatch = self.smart_match_module() if hasattr(self, 'meta'): self.meta['fuzzy_match_called'] = True kwargs = dict( repo=self.github_repo, title=self.issue.instance.title, component=self.template_data.get('component name') ) smatch = self.module_indexer.fuzzy_match(**kwargs) if self.module_indexer.is_valid(smatch): self.module = smatch component = smatch self.match = self.module_indexer.find_match(smatch) component_isvalid = self.module_indexer.is_valid(component) self.meta['component_valid'] = component_isvalid # Allow globs for module groups # https://github.com/ansible/ansible-modules-core/issues/3831 craw = self.template_data.get('component_raw') if self.module_indexer.is_multi(craw): self.meta['multiple_components'] = True # get all of the matches self.matches = self.module_indexer.multi_match(craw) if self.matches: # get maintainers for all of the matches mmap = {} for match in self.matches: key = match['filename'] mmap[key] = self.get_maintainers_by_match(match) # is there a match that represents all included maintainers? mtuples = [x[1] for x in mmap.items()] umtuples = [list(x) for x in set(tuple(x) for x in mtuples)] all_maintainers = [] for mtup in umtuples: for x in mtup: if x not in all_maintainers: all_maintainers.append(x) best_match = None for k,v in mmap.iteritems(): if sorted(set(v)) == sorted(set(all_maintainers)): best_match = k break if best_match: self.match = self.module_indexer.find_match(best_match) else: # there's no good match that would include all maintainers # just skip multi-module processing for now since the rest # of the code doesn't know what to do with it. if not isinstance(self.debug, bool): self.debug('multi-match maintainers: %s' % umtuples) #print(craw) #import epdb; epdb.st() pass else: self.meta['multiple_components'] = False # set the maintainer(s) self.module_maintainer = [x for x in self.get_module_maintainers()] self.meta['module_maintainers'] = self.module_maintainer # fixme: too many places where the module is set if self.match: self.module = self.match['name'] # Helper to fix issue descriptions ... DF = DescriptionFixer(self.issue, self.module_indexer, self.match) self.issue.new_description = DF.new_description def _connect(self): """Connects to GitHub's API""" return Github(login_or_token=self.github_token or self.github_user, password=self.github_pass) #return GithubWrapper( # login_or_token=self.github_token or self.github_user, # password=self.github_pass # ) def _get_repo_path(self): if self.github_repo in ['core', 'extras']: return "ansible/ansible-modules-%s" % self.github_repo else: return "ansible/%s" % self.github_repo def is_pr(self, issue): if '/pull/' in issue.html_url: return True else: return False def is_issue(self, issue): return not self.is_pr(issue) @ratecheck() def get_ansible_members(self): ansible_members = [] update = False write_cache = False now = self.get_current_time() org = self._connect().get_organization("ansible") cachedir = self.cachedir if cachedir.endswith('/issues'): cachedir = os.path.dirname(cachedir) cachefile = os.path.join(cachedir, 'members.pickle') if not os.path.isdir(cachedir): os.makedirs(cachedir) if os.path.isfile(cachefile): with open(cachefile, 'rb') as f: mdata = pickle.load(f) ansible_members = mdata[1] if mdata[0] < org.updated_at: update = True else: update = True write_cache = True if update: members = org.get_members() ansible_members = [x.login for x in members] # save the data if write_cache: mdata = [now, ansible_members] with open(cachefile, 'wb') as f: pickle.dump(mdata, f) #import epdb; epdb.st() return ansible_members @ratecheck() def get_valid_labels(self): # use the repo wrapper to enable caching+updating self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) vlabels = [] for vl in self.repo.get_labels(): vlabels.append(vl.name) return vlabels def _get_maintainers(self, usecache=True): """Reads all known maintainers from files and their owner namespace""" if not self.maintainers or not usecache: for repo in ['core', 'extras']: f = open(MAINTAINERS_FILES[repo]) for line in f: owner_space = (line.split(': ')[0]).strip() maintainers_string = (line.split(': ')[-1]).strip() self.maintainers[owner_space] = maintainers_string.split(' ') f.close() # meta is special self.maintainers['meta'] = ['ansible'] return self.maintainers def debug(self, msg=""): """Prints debug message if verbosity is given""" if self.verbose: print("Debug: " + msg) def get_ansible_version(self): aversion = None rawdata = self.template_data.get('ansible version', '') if rawdata: aversion = self.version_indexer.strip_ansible_version(rawdata) if not aversion or aversion == 'devel': aversion = self.version_indexer.ansible_version_by_date(self.issue.instance.created_at) if aversion: if aversion.endswith('.'): aversion += '0' # re-run for versions ending with .x if aversion: if aversion.endswith('.x'): aversion = self.version_indexer.strip_ansible_version(aversion) #import epdb; epdb.st() if self.version_indexer.is_valid_version(aversion) and aversion != None: return aversion else: # try to go through the submitter's comments and look for the # first one that specifies a valid version cversion = None for comment in self.issue.current_comments: if comment.user.login != self.issue.instance.user.login: continue xver = self.version_indexer.strip_ansible_version(comment.body) if self.version_indexer.is_valid_version(xver): cversion = xver break # use the comment version aversion = cversion return aversion def get_ansible_version_major_minor(self): return self.version_indexer.get_major_minor(self.ansible_version) def get_maintainers_by_match(self, match): module_maintainers = [] maintainers = self._get_maintainers() if match['name'] in maintainers: module_maintainers = maintainers[match['name']] elif match['repo_filename'] in maintainers: module_maintainers = maintainers[match['repo_filename']] elif (match['deprecated_filename']) in maintainers: module_maintainers = maintainers[match['deprecated_filename']] elif match['namespaced_module'] in maintainers: module_maintainers = maintainers[match['namespaced_module']] elif match['fulltopic'] in maintainers: module_maintainers = maintainers[match['fulltopic']] elif (match['topic'] + '/') in maintainers: module_maintainers = maintainers[match['topic'] + '/'] else: pass # Fallback to using the module author(s) if not module_maintainers and self.match: if self.match['authors']: module_maintainers = [x for x in self.match['authors']] #import epdb; epdb.st() return module_maintainers def get_module_maintainers(self, expand=True, usecache=True): """Returns the list of maintainers for the current module""" # expand=False means don't use cache and don't expand the 'ansible' group if self.module_maintainers and usecache: return self.module_maintainers module_maintainers = [] module = self.module if not module: return module_maintainers if not self.module_indexer.is_valid(module): return module_maintainers if self.match: mdata = self.match else: mdata = self.module_indexer.find_match(module) if mdata['repository'] != self.github_repo: # this was detected and handled in the process loop pass # get cached or non-cached maintainers list if not expand: maintainers = self._get_maintainers(usecache=False) else: maintainers = self._get_maintainers() if mdata['name'] in maintainers: module_maintainers = maintainers[mdata['name']] elif mdata['repo_filename'] in maintainers: module_maintainers = maintainers[mdata['repo_filename']] elif (mdata['deprecated_filename']) in maintainers: module_maintainers = maintainers[mdata['deprecated_filename']] elif mdata['namespaced_module'] in maintainers: module_maintainers = maintainers[mdata['namespaced_module']] elif mdata['fulltopic'] in maintainers: module_maintainers = maintainers[mdata['fulltopic']] elif (mdata['topic'] + '/') in maintainers: module_maintainers = maintainers[mdata['topic'] + '/'] else: pass # Fallback to using the module author(s) if not module_maintainers and self.match: if self.match['authors']: module_maintainers = [x for x in self.match['authors']] # need to set the no maintainer template or assume ansible? if not module_maintainers and self.module and self.match: #import epdb; epdb.st() pass #import epdb; epdb.st() return module_maintainers def get_current_labels(self): """Pull the list of labels on this Issue""" if not self.current_labels: labels = self.issue.instance.labels for label in labels: self.current_labels.append(label.name) return self.current_labels def run(self): pass def create_actions(self): pass def component_from_comments(self): """Extracts a component name from special comments""" # https://github.com/ansible/ansible-modules/core/issues/2618 # comments like: [module: packaging/os/zypper.py] ... ? component = None for idx, x in enumerate(self.issue.current_comments): if '[' in x.body and ']' in x.body and ('module' in x.body or 'component' in x.body or 'plugin' in x.body): if x.user.login in BOTLIST: component = x.body.split()[-1] component = component.replace('[', '') return component def has_maintainer_commented(self): """Has the maintainer -ever- commented on the issue?""" commented = False if self.module_maintainers: for comment in self.issue.current_comments: # ignore comments from submitter if comment.user.login == self.issue.get_submitter(): continue # "ansible" is special ... if 'ansible' in self.module_maintainers \ and comment.user.login in self.ansible_members: commented = True elif comment.user.login in self.module_maintainers: commented = True return commented def is_maintainer_mentioned(self): mentioned = False if self.module_maintainers: for comment in self.issue.current_comments: # "ansible" is special ... if 'ansible' in self.module_maintainers: for x in self.ansible_members: if ('@%s' % x) in comment.body: mentioned = True break else: for x in self.module_maintainers: if ('@%s' % x) in comment.body: mentioned = True break return mentioned def get_current_time(self): #now = datetime.now() now = datetime.utcnow() #now = datetime.now(pytz.timezone('US/Pacific')) #import epdb; epdb.st() return now def age_of_last_maintainer_comment(self): """How long ago did the maintainer comment?""" last_comment = None if self.module_maintainers: for idx,comment in enumerate(self.issue.current_comments): # "ansible" is special ... is_maintainer = False if 'ansible' in self.module_maintainers \ and comment.user.login in self.ansible_members: is_maintainer = True elif comment.user.login in self.module_maintainers: is_maintainer = True if is_maintainer: last_comment = comment break if not last_comment: return -1 else: now = self.get_current_time() diff = now - last_comment.created_at age = diff.days return age def is_waiting_on_maintainer(self): """Is the issue waiting on the maintainer to comment?""" waiting = False if self.module_maintainers: if not self.issue.current_comments: return True creator_last_index = -1 maintainer_last_index = -1 for idx,comment in enumerate(self.issue.current_comments): if comment.user.login == self.issue.get_submitter(): if creator_last_index == -1 or idx < creator_last_index: creator_last_index = idx # "ansible" is special ... is_maintainer = False if 'ansible' in self.module_maintainers \ and comment.user.login in self.ansible_members: is_maintainer = True elif comment.user.login in self.module_maintainers: is_maintainer = True if is_maintainer and \ (maintainer_last_index == -1 or idx < maintainer_last_index): maintainer_last_index = idx if creator_last_index == -1 and maintainer_last_index == -1: waiting = True elif creator_last_index == -1 and maintainer_last_index > -1: waiting = False elif creator_last_index < maintainer_last_index: waiting = True return waiting def keep_current_main_labels(self): current_labels = self.issue.get_current_labels() for current_label in current_labels: if current_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS: self.issue.add_desired_label(name=current_label) def add_desired_labels_by_issue_type(self, comments=True): """Adds labels by defined issue type""" issue_type = self.template_data.get('issue type', False) if issue_type is False: self.issue.add_desired_label('needs_info') return if not issue_type.lower() in self.VALID_ISSUE_TYPES: # special handling for PRs if self.issue.instance.pull_request: mel = [x for x in self.issue.current_labels if x in self.MUTUALLY_EXCLUSIVE_LABELS] if not mel: # if only adding new files, assume it is a feature if self.patch_contains_only_new_files(): issue_type = 'feature pull request' else: if not isinstance(self.debug, bool): self.debug('"%s" was not a valid issue type, adding "needs_info"' % issue_type) self.issue.add_desired_label('needs_info') return else: if not isinstance(self.debug, bool): self.debug('"%s" was not a valid issue type, adding "needs_info"' % issue_type) self.issue.add_desired_label('needs_info') return desired_label = issue_type.replace(' ', '_') desired_label = desired_label.lower() desired_label = desired_label.replace('documentation', 'docs') # FIXME - shouldn't have to do this if desired_label == 'test_pull_request': desired_label = 'test_pull_requests' #import epdb; epdb.st() # is there a mutually exclusive label already? if desired_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS: mel = [x for x in self.issue.MUTUALLY_EXCLUSIVE_LABELS \ if x in self.issue.current_labels] if len(mel) > 0: return if desired_label not in self.issue.get_current_labels(): self.issue.add_desired_label(name=desired_label) if len(self.issue.current_comments) == 0 and comments: # only set this if no other comments self.issue.add_desired_comment(boilerplate='issue_new') def patch_contains_only_new_files(self): '''Does the PR edit any existing files?''' oldfiles = False for x in self.issue.files: if x.filename.encode('ascii', 'ignore') in self.file_indexer.files: if not isinstance(self.debug, bool): self.debug('old file match on %s' % x.filename.encode('ascii', 'ignore')) oldfiles = True break return not oldfiles def add_desired_labels_by_ansible_version(self): if not 'ansible version' in self.template_data: if not isinstance(self.debug, bool): self.debug(msg="no ansible version section") self.issue.add_desired_label(name="needs_info") #self.issue.add_desired_comment( # boilerplate="issue_missing_data" #) return if not self.template_data['ansible version']: if not isinstance(self.debug, bool): self.debug(msg="no ansible version defined") self.issue.add_desired_label(name="needs_info") #self.issue.add_desired_comment( # boilerplate="issue_missing_data" #) return def add_desired_labels_by_namespace(self): """Adds labels regarding module namespaces""" SKIPTOPICS = ['network/basics/'] if not self.match: return False ''' if 'component name' in self.template_data and self.match: if self.match['repository'] != self.github_repo: self.issue.add_desired_comment(boilerplate='issue_wrong_repo') ''' for key in ['topic', 'subtopic']: # ignore networking/basics if self.match[key] and not self.match['fulltopic'] in SKIPTOPICS: thislabel = self.issue.TOPIC_MAP.\ get(self.match[key], self.match[key]) if thislabel in self.valid_labels: self.issue.add_desired_label(thislabel) def render_comment(self, boilerplate=None): """Renders templates into comments using the boilerplate as filename""" maintainers = self.get_module_maintainers(expand=False) if not maintainers: maintainers = ['NO_MAINTAINER_FOUND'] #FIXME - why? submitter = self.issue.get_submitter() missing_sections = [x for x in self.issue.REQUIRED_SECTIONS \ if not x in self.template_data \ or not self.template_data.get(x)] if not self.match and missing_sections: # be lenient on component name for ansible/ansible if self.github_repo == 'ansible' and 'component name' in missing_sections: missing_sections.remove('component name') #if missing_sections: # import epdb; epdb.st() issue_type = self.template_data.get('issue type', None) if issue_type: issue_type = issue_type.lower() correct_repo = self.match.get('repository', None) template = environment.get_template('%s.j2' % boilerplate) comment = template.render(maintainers=maintainers, submitter=submitter, issue_type=issue_type, correct_repo=correct_repo, component_name=self.template_data.get('component name', 'NULL'), missing_sections=missing_sections) #import epdb; epdb.st() return comment def process_comments(self): """ Processes ISSUE comments for matching criteria to add labels""" if not self.github_user in self.BOTLIST: self.BOTLIST.append(self.github_user) module_maintainers = self.get_module_maintainers() comments = self.issue.get_comments() today = datetime.today() if not isinstance(self.debug, bool): self.debug(msg="--- START Processing Comments:") for idc,comment in enumerate(comments): if comment.user.login in self.BOTLIST: if not isinstance(self.debug, bool): self.debug(msg="%s is in botlist: " % comment.user.login) time_delta = today - comment.created_at comment_days_old = time_delta.days if not isinstance(self.debug, bool): self.debug(msg="Days since last bot comment: %s" % comment_days_old) if comment_days_old > 14: labels = self.issue.desired_labels if 'pending' not in comment.body: if self.issue.is_labeled_for_interaction(): if not isinstance(self.debug, bool): self.debug(msg="submitter_first_warning") self.issue.add_desired_comment( boilerplate="submitter_first_warning" ) break if "maintainer_review" not in labels: if not isinstance(self.debug, bool): self.debug(msg="maintainer_first_warning") self.issue.add_desired_comment( boilerplate="maintainer_first_warning" ) break # pending in comment.body else: if self.issue.is_labeled_for_interaction(): if not isinstance(self.debug, bool): self.debug(msg="submitter_second_warning") self.issue.add_desired_comment( boilerplate="submitter_second_warning" ) break if "maintainer_review" in labels: if not isinstance(self.debug, bool): self.debug(msg="maintainer_second_warning") self.issue.add_desired_comment( boilerplate="maintainer_second_warning" ) break if not isinstance(self.debug, bool): self.debug(msg="STATUS: no useful state change since last pass" "( %s )" % comment.user.login) break if comment.user.login in module_maintainers \ or comment.user.login.lower() in module_maintainers\ or ('ansible' in module_maintainers and comment.user.login in self.ansible_members): if not isinstance(self.debug, bool): self.debug(msg="%s is module maintainer commented on %s." % (comment.user.login, comment.created_at)) if 'needs_info' in comment.body: if not isinstance(self.debug, bool): self.debug(msg="...said needs_info!") self.issue.add_desired_label(name="needs_info") elif "close_me" in comment.body: if not isinstance(self.debug, bool): self.debug(msg="...said close_me!") self.issue.add_desired_label(name="pending_action_close_me") break if comment.user.login == self.issue.get_submitter(): if not isinstance(self.debug, bool): self.debug(msg="submitter %s, commented on %s." % (comment.user.login, comment.created_at)) if comment.user.login not in self.BOTLIST and comment.user.login in self.ansible_members: if not isinstance(self.debug, bool): self.debug(msg="%s is a ansible member" % comment.user.login) if not isinstance(self.debug, bool): self.debug(msg="--- END Processing Comments") def issue_type_to_label(self, issue_type): if issue_type: issue_type = issue_type.lower() issue_type = issue_type.replace(' ', '_') issue_type = issue_type.replace('documentation', 'docs') return issue_type def check_safe_match(self): """ Turn force on or off depending on match characteristics """ safe_match = False if self.action_count() == 0: safe_match = True elif not self.actions['close'] and not self.actions['unlabel']: if len(self.actions['newlabel']) == 1: if self.actions['newlabel'][0].startswith('affects_'): safe_match = True else: safe_match = False if self.module: if self.module in self.issue.instance.title.lower(): safe_match = True # be more lenient on re-notifications if not safe_match: if not self.actions['close'] and \ not self.actions['unlabel'] and \ not self.actions['newlabel']: if len(self.actions['comments']) == 1: if 'still waiting' in self.actions['comments'][0]: safe_match = True #import epdb; epdb.st() if safe_match: self.force = True else: self.force = False def action_count(self): """ Return the number of actions that are to be performed """ count = 0 for k,v in self.actions.iteritems(): if k == 'close' and v: count += 1 elif k != 'close': count += len(v) return count def apply_actions(self): action_meta = {'REDO': False} if self.safe_force: self.check_safe_match() if self.action_count() > 0: if self.dry_run: print("Dry-run specified, skipping execution of actions") else: if self.force: print("Running actions non-interactive as you forced.") self.execute_actions() return cont = raw_input("Take recommended actions (y/N/a/R/T)? ") if cont in ('a', 'A'): sys.exit(0) if cont in ('Y', 'y'): self.execute_actions() if cont == 'T': self.template_wizard() action_meta['REDO'] = True if cont == 'r' or cont == 'R': action_meta['REDO'] = True if cont == 'DEBUG': import epdb; epdb.st() elif self.always_pause: print("Skipping, but pause.") cont = raw_input("Continue (Y/n/a/R/T)? ") if cont in ('a', 'A', 'n', 'N'): sys.exit(0) if cont == 'T': self.template_wizard() action_meta['REDO'] = True elif cont == 'REDO': action_meta['REDO'] = True elif cont == 'DEBUG': import epdb; epdb.st() action_meta['REDO'] = True else: print("Skipping.") # let the upper level code redo this issue return action_meta def template_wizard(self): print('################################################') print(self.issue.new_description) print('################################################') cont = raw_input("Apply this new description? (Y/N)") if cont == 'Y': self.issue.set_description(self.issue.new_description) #import epdb; epdb.st() def execute_actions(self): """Turns the actions into API calls""" #time.sleep(1) for comment in self.actions['comments']: #import epdb; epdb.st() if not isinstance(self.debug, bool): self.debug(msg="API Call comment: " + comment) self.issue.add_comment(comment=comment) if self.actions['close']: # https://github.com/PyGithub/PyGithub/blob/master/github/Issue.py#L263 self.issue.instance.edit(state='closed') return for unlabel in self.actions['unlabel']: if not isinstance(self.debug, bool): self.debug(msg="API Call unlabel: " + unlabel) self.issue.remove_label(label=unlabel) for newlabel in self.actions['newlabel']: if not isinstance(self.debug, bool): self.debug(msg="API Call newlabel: " + newlabel) self.issue.add_label(label=newlabel) if 'assign' in self.actions: for user in self.actions['assign']: self.issue.assign_user(user) if 'unassign' in self.actions: for user in self.actions['unassign']: self.issue.unassign_user(user) def smart_match_module(self): '''Fuzzy matching for modules''' if hasattr(self, 'meta'): self.meta['smart_match_module_called'] = True match = None known_modules = [] for k,v in self.module_indexer.modules.iteritems(): known_modules.append(v['name']) title = self.issue.instance.title.lower() title = title.replace(':', '') title_matches = [x for x in known_modules if x + ' module' in title] if not title_matches: title_matches = [x for x in known_modules if title.startswith(x + ' ')] if not title_matches: title_matches = [x for x in known_modules if ' ' + x + ' ' in title] cmatches = None if self.template_data.get('component name'): component = self.template_data.get('component name') cmatches = [x for x in known_modules if x in component] cmatches = [x for x in cmatches if not '_' + x in component] import epdb; epdb.st() # use title ... ? if title_matches: cmatches = [x for x in cmatches if x in title_matches] if cmatches: if len(cmatches) >= 1: match = cmatches[0] if not match: if 'docs.ansible.com' in component: #import epdb; epdb.st() pass else: #import epdb; epdb.st() pass #import epdb; epdb.st() if not match: if len(title_matches) == 1: match = title_matches[0] else: print("module - title matches: %s" % title_matches) print("module - component matches: %s" % cmatches) return match def cache_issue(self, issue): iid = issue.instance.number fpath = os.path.join(self.cachedir, str(iid)) if not os.path.isdir(fpath): os.makedirs(fpath) fpath = os.path.join(fpath, 'iwrapper.pickle') with open(fpath, 'wb') as f: pickle.dump(issue, f) #import epdb; epdb.st() def load_cached_issues(self, state='open'): issues = [] idirs = glob.glob('%s/*' % self.cachedir) idirs = [x for x in idirs if not x.endswith('.pickle')] for idir in idirs: wfile = os.path.join(idir, 'iwrapper.pickle') if os.path.isfile(wfile): with open(wfile, 'rb') as f: wrapper = pickle.load(f) issues.append(wrapper.instance) return issues def wait_for_rate_limit(self): gh = self._connect() GithubWrapper.wait_for_rate_limit(githubobj=gh) @ratecheck() def is_pr_merged(self, number): '''Check if a PR# has been merged or not''' merged = False pr = None try: pr = self.repo.get_pullrequest(number) except Exception as e: print(e) if pr: merged = pr.merged return merged def print_comment_list(self): """Print comment creators and the commands they used""" for x in self.issue.current_comments: command = None if x.user.login != 'ansibot': command = [y for y in self.VALID_COMMANDS if y in x.body \ and not '!' + y in x.body] command = ', '.join(command) else: # What template did ansibot use? try: command = x.body.split('\n')[-1].split()[-2] except: pass if command: print("\t%s %s (%s)" % (x.created_at.isoformat(), x.user.login, command)) else: print("\t%s %s" % (x.created_at.isoformat(), x.user.login))
def wait_for_rate_limit(self): gh = self._connect() GithubWrapper.wait_for_rate_limit(githubobj=gh)
class DefaultTriager(object): BOTLIST = ['gregdek', 'robynbergeron', 'ansibot'] VALID_ISSUE_TYPES = ['bug report', 'feature idea', 'documentation report'] IGNORE_LABELS = [ "aws", "azure", "cloud", "feature_pull_request", "feature_idea", "bugfix_pull_request", "bug_report", "docs_pull_request", "docs_report", "in progress", "docs_pull_request", "easyfix", "pending_action", "gce", "python3", "P1", "P2", "P3", "P4", ] def __init__(self, verbose=None, github_user=None, github_pass=None, github_token=None, github_repo=None, number=None, start_at=None, always_pause=False, force=False, safe_force=False, dry_run=False, no_since=False): self.verbose = verbose self.github_user = github_user self.github_pass = github_pass self.github_token = github_token self.github_repo = github_repo self.number = number self.start_at = start_at self.always_pause = always_pause self.force = force self.safe_force = safe_force self.dry_run = dry_run self.no_since = no_since self.issue = None self.maintainers = {} self.module_maintainers = [] self.actions = { 'newlabel': [], 'unlabel': [], 'comments': [], 'close': False, } # set the cache dir self.cachedir = '~/.ansibullbot/cache' if self.github_repo == 'ansible': self.cachedir += '/ansible/ansible/' else: self.cachedir += '/ansible/ansible-modules-%s/' % self.github_repo self.cachedir += 'issues' self.cachedir = os.path.expanduser(self.cachedir) if not os.path.isdir(self.cachedir): os.makedirs(self.cachedir) print("Initializing AnsibleVersionIndexer") self.version_indexer = AnsibleVersionIndexer() #import epdb; epdb.st() print("Initializing ModuleIndexer") self.module_indexer = ModuleIndexer() self.module_indexer.get_ansible_modules() print("Initializing FileIndexer") self.file_indexer = FileIndexer() self.file_indexer.get_files() print("Getting ansible members") self.ansible_members = self.get_ansible_members() print("Getting valid labels") self.valid_labels = self.get_valid_labels() # processed metadata self.meta = {} def _process(self, usecache=True): '''Do some initial processing of the issue''' # clear all actions self.actions = { 'newlabel': [], 'unlabel': [], 'comments': [], 'close': False, } # clear module maintainers self.module_maintainers = [] # print some general info about the Issue to be processed print("\n") print("Issue #%s [%s]: %s" % (self.issue.number, self.icount, self.issue.instance.title.encode('ascii', 'ignore'))) print("%s" % self.issue.instance.html_url) print("Created at %s" % self.issue.instance.created_at) print("Updated at %s" % self.issue.instance.updated_at) # get the template data self.template_data = self.issue.get_template_data() # was the issue type defined correctly? issue_type_defined = False issue_type_valid = False issue_type = False if 'issue type' in self.template_data: issue_type_defined = True issue_type = self.template_data['issue type'] if issue_type.lower() in self.VALID_ISSUE_TYPES: issue_type_valid = True self.meta['issue_type_defined'] = issue_type_defined self.meta['issue_type_valid'] = issue_type_valid self.meta['issue_type'] = issue_type if self.meta['issue_type_valid']: self.meta['issue_type_label'] = self.issue_type_to_label( issue_type) else: self.meta['issue_type_label'] = None # What is the ansible version? self.ansible_version = self.get_ansible_version() if not isinstance(self.debug, bool): self.debug('version: %s' % self.ansible_version) self.ansible_label_version = self.get_ansible_version_major_minor() if not isinstance(self.debug, bool): self.debug('lversion: %s' % self.ansible_label_version) # was component specified? component_defined = 'component name' in self.template_data self.meta['component_defined'] = component_defined # extract the component component = self.template_data.get('component name', None) # save the real name if self.github_repo != 'ansible': self.match = self.module_indexer.find_match(component) or {} else: self.match = \ self.module_indexer.find_match(component, exact=True) or {} self.module = self.match.get('name', None) # check if component is a known module component_isvalid = self.module_indexer.is_valid(component) self.meta['component_valid'] = component_isvalid # smart match modules (only on module repos) if not component_isvalid and \ self.github_repo != 'ansible' and \ not self.match: if hasattr(self, 'meta'): self.meta['fuzzy_match_called'] = True kwargs = dict(repo=self.github_repo, title=self.issue.instance.title, component=self.template_data.get('component name')) smatch = self.module_indexer.fuzzy_match(**kwargs) if self.module_indexer.is_valid(smatch): self.module = smatch component = smatch self.match = self.module_indexer.find_match(smatch) component_isvalid = self.module_indexer.is_valid(component) self.meta['component_valid'] = component_isvalid # Allow globs for module groups # https://github.com/ansible/ansible-modules-core/issues/3831 craw = self.template_data.get('component_raw') if self.module_indexer.is_multi(craw): self.meta['multiple_components'] = True # get all of the matches self.matches = self.module_indexer.multi_match(craw) if self.matches: # get maintainers for all of the matches mmap = {} for match in self.matches: key = match['filename'] mmap[key] = self.get_maintainers_by_match(match) # is there a match that represents all included maintainers? mtuples = [x[1] for x in mmap.items()] umtuples = [list(x) for x in set(tuple(x) for x in mtuples)] all_maintainers = [] for mtup in umtuples: for x in mtup: if x not in all_maintainers: all_maintainers.append(x) best_match = None for k, v in mmap.iteritems(): if sorted(set(v)) == sorted(set(all_maintainers)): best_match = k break if best_match: self.match = self.module_indexer.find_match(best_match) else: # there's no good match that would include all maintainers # just skip multi-module processing for now since the rest # of the code doesn't know what to do with it. if not isinstance(self.debug, bool): self.debug('multi-match maintainers: %s' % umtuples) #print(craw) #import epdb; epdb.st() pass else: self.meta['multiple_components'] = False # set the maintainer(s) self.module_maintainer = [x for x in self.get_module_maintainers()] self.meta['module_maintainers'] = self.module_maintainer # fixme: too many places where the module is set if self.match: self.module = self.match['name'] # Helper to fix issue descriptions ... DF = DescriptionFixer(self.issue, self.module_indexer, self.match) self.issue.new_description = DF.new_description @RateLimited def _connect(self): """Connects to GitHub's API""" if self.github_token != 'False': return Github(login_or_token=self.github_token) else: return Github(login_or_token=self.github_user, password=self.github_pass) def _get_repo_path(self): if self.github_repo in ['core', 'extras']: return "ansible/ansible-modules-%s" % self.github_repo else: return "ansible/%s" % self.github_repo def is_pr(self, issue): if '/pull/' in issue.html_url: return True else: return False def is_issue(self, issue): return not self.is_pr(issue) @RateLimited def get_ansible_members(self): ansible_members = [] update = False write_cache = False now = self.get_current_time() org = self._connect().get_organization("ansible") cachedir = self.cachedir if cachedir.endswith('/issues'): cachedir = os.path.dirname(cachedir) cachefile = os.path.join(cachedir, 'members.pickle') if not os.path.isdir(cachedir): os.makedirs(cachedir) if os.path.isfile(cachefile): with open(cachefile, 'rb') as f: mdata = pickle.load(f) ansible_members = mdata[1] if mdata[0] < org.updated_at: update = True else: update = True write_cache = True if update: members = org.get_members() ansible_members = [x.login for x in members] # save the data if write_cache: mdata = [now, ansible_members] with open(cachefile, 'wb') as f: pickle.dump(mdata, f) #import epdb; epdb.st() return ansible_members @RateLimited def get_ansible_core_team(self): teamlist = [ 'ansible-commit', 'ansible-community', 'ansible-commit-external' ] teams = [] ansible_members = [] conn = self._connect() org = conn.get_organization('ansible') for x in org.get_teams(): if x.name in teamlist: teams.append(x) for x in teams: for y in x.get_members(): ansible_members.append(y.login) ansible_members = sorted(set(ansible_members)) return ansible_members #@RateLimited def get_valid_labels(self, repo=None): # use the repo wrapper to enable caching+updating if not self.ghw: self.gh = self._connect() self.ghw = GithubWrapper(self.gh) if not repo: # OLD workflow self.repo = self.ghw.get_repo(self._get_repo_path()) vlabels = [] for vl in self.repo.get_labels(): vlabels.append(vl.name) else: # v3 workflow rw = self.ghw.get_repo(repo) vlabels = [] for vl in rw.get_labels(): vlabels.append(vl.name) return vlabels def _get_maintainers(self, usecache=True): """Reads all known maintainers from files and their owner namespace""" if not self.maintainers or not usecache: for repo in ['core', 'extras']: f = open(MAINTAINERS_FILES[repo]) for line in f: owner_space = (line.split(': ')[0]).strip() maintainers_string = (line.split(': ')[-1]).strip() self.maintainers[owner_space] = \ maintainers_string.split(' ') f.close() # meta is special self.maintainers['meta'] = ['ansible'] return self.maintainers def debug(self, msg=""): """Prints debug message if verbosity is given""" if self.verbose: print("Debug: " + msg) def get_ansible_version(self): aversion = None rawdata = self.template_data.get('ansible version', '') if rawdata: aversion = self.version_indexer.strip_ansible_version(rawdata) if not aversion or aversion == 'devel': aversion = \ self.version_indexer.ansible_version_by_date( self.issue.instance.created_at ) if aversion: if aversion.endswith('.'): aversion += '0' # re-run for versions ending with .x if aversion: if aversion.endswith('.x'): aversion = self.version_indexer.strip_ansible_version(aversion) #import epdb; epdb.st() if self.version_indexer.is_valid_version(aversion) and \ aversion is not None: return aversion else: # try to go through the submitter's comments and look for the # first one that specifies a valid version cversion = None for comment in self.issue.current_comments: if comment.user.login != self.issue.instance.user.login: continue xver = self.version_indexer.strip_ansible_version(comment.body) if self.version_indexer.is_valid_version(xver): cversion = xver break # use the comment version aversion = cversion return aversion def get_ansible_version_by_issue(self, issuewrapper): iw = issuewrapper aversion = None rawdata = iw.get_template_data().get('ansible version', '') if rawdata: aversion = self.version_indexer.strip_ansible_version(rawdata) if not aversion or aversion == 'devel': aversion = self.version_indexer.ansible_version_by_date( self.issue.instance.created_at) if aversion: if aversion.endswith('.'): aversion += '0' # re-run for versions ending with .x if aversion: if aversion.endswith('.x'): aversion = self.version_indexer.strip_ansible_version(aversion) #import epdb; epdb.st() if self.version_indexer.is_valid_version(aversion) and \ aversion is not None: return aversion else: # try to go through the submitter's comments and look for the # first one that specifies a valid version cversion = None for comment in self.issue.current_comments: if comment.user.login != self.issue.instance.user.login: continue xver = self.version_indexer.strip_ansible_version(comment.body) if self.version_indexer.is_valid_version(xver): cversion = xver break # use the comment version aversion = cversion return aversion def get_ansible_version_major_minor(self, version=None): if not version: # old workflow if not hasattr(self, 'ansible_version'): logging.debug('breakpoint!') import epdb epdb.st() return self.version_indexer.get_major_minor(self.ansible_version) else: # v3 workflow return self.version_indexer.get_major_minor(version) def get_maintainers_by_match(self, match): module_maintainers = [] maintainers = self._get_maintainers() if match['name'] in maintainers: module_maintainers = maintainers[match['name']] elif match['repo_filename'] in maintainers: module_maintainers = maintainers[match['repo_filename']] elif (match['deprecated_filename']) in maintainers: module_maintainers = maintainers[match['deprecated_filename']] elif match['namespaced_module'] in maintainers: module_maintainers = maintainers[match['namespaced_module']] elif match['fulltopic'] in maintainers: module_maintainers = maintainers[match['fulltopic']] elif (match['topic'] + '/') in maintainers: module_maintainers = maintainers[match['topic'] + '/'] else: pass # Fallback to using the module author(s) if not module_maintainers and self.match: if self.match['authors']: module_maintainers = [x for x in self.match['authors']] #import epdb; epdb.st() return module_maintainers def get_module_maintainers(self, expand=True, usecache=True): """Returns the list of maintainers for the current module""" # expand=False ... ? if self.module_maintainers and usecache: return self.module_maintainers module_maintainers = [] module = self.module if not module: return module_maintainers if not self.module_indexer.is_valid(module): return module_maintainers if self.match: mdata = self.match else: mdata = self.module_indexer.find_match(module) if mdata['repository'] != self.github_repo: # this was detected and handled in the process loop pass # get cached or non-cached maintainers list if not expand: maintainers = self._get_maintainers(usecache=False) else: maintainers = self._get_maintainers() if mdata['name'] in maintainers: module_maintainers = maintainers[mdata['name']] elif mdata['repo_filename'] in maintainers: module_maintainers = maintainers[mdata['repo_filename']] elif (mdata['deprecated_filename']) in maintainers: module_maintainers = maintainers[mdata['deprecated_filename']] elif mdata['namespaced_module'] in maintainers: module_maintainers = maintainers[mdata['namespaced_module']] elif mdata['fulltopic'] in maintainers: module_maintainers = maintainers[mdata['fulltopic']] elif (mdata['topic'] + '/') in maintainers: module_maintainers = maintainers[mdata['topic'] + '/'] else: pass # Fallback to using the module author(s) if not module_maintainers and self.match: if self.match['authors']: module_maintainers = [x for x in self.match['authors']] # need to set the no maintainer template or assume ansible? if not module_maintainers and self.module and self.match: #import epdb; epdb.st() pass #import epdb; epdb.st() return module_maintainers def get_current_labels(self): """Pull the list of labels on this Issue""" if not self.current_labels: labels = self.issue.instance.labels for label in labels: self.current_labels.append(label.name) return self.current_labels def run(self): pass def create_actions(self): pass def component_from_comments(self): """Extracts a component name from special comments""" # https://github.com/ansible/ansible-modules/core/issues/2618 # comments like: [module: packaging/os/zypper.py] ... ? component = None for idx, x in enumerate(self.issue.current_comments): if '[' in x.body and \ ']' in x.body and \ ('module' in x.body or 'component' in x.body or 'plugin' in x.body): if x.user.login in BOTLIST: component = x.body.split()[-1] component = component.replace('[', '') return component def has_maintainer_commented(self): """Has the maintainer -ever- commented on the issue?""" commented = False if self.module_maintainers: for comment in self.issue.current_comments: # ignore comments from submitter if comment.user.login == self.issue.get_submitter(): continue # "ansible" is special ... if 'ansible' in self.module_maintainers \ and comment.user.login in self.ansible_members: commented = True elif comment.user.login in self.module_maintainers: commented = True return commented def is_maintainer_mentioned(self): mentioned = False if self.module_maintainers: for comment in self.issue.current_comments: # "ansible" is special ... if 'ansible' in self.module_maintainers: for x in self.ansible_members: if ('@%s' % x) in comment.body: mentioned = True break else: for x in self.module_maintainers: if ('@%s' % x) in comment.body: mentioned = True break return mentioned def get_current_time(self): #now = datetime.now() now = datetime.utcnow() #now = datetime.now(pytz.timezone('US/Pacific')) #import epdb; epdb.st() return now def age_of_last_maintainer_comment(self): """How long ago did the maintainer comment?""" last_comment = None if self.module_maintainers: for idx, comment in enumerate(self.issue.current_comments): # "ansible" is special ... is_maintainer = False if 'ansible' in self.module_maintainers \ and comment.user.login in self.ansible_members: is_maintainer = True elif comment.user.login in self.module_maintainers: is_maintainer = True if is_maintainer: last_comment = comment break if not last_comment: return -1 else: now = self.get_current_time() diff = now - last_comment.created_at age = diff.days return age def is_waiting_on_maintainer(self): """Is the issue waiting on the maintainer to comment?""" waiting = False if self.module_maintainers: if not self.issue.current_comments: return True creator_last_index = -1 maintainer_last_index = -1 for idx, comment in enumerate(self.issue.current_comments): if comment.user.login == self.issue.get_submitter(): if creator_last_index == -1 or idx < creator_last_index: creator_last_index = idx # "ansible" is special ... is_maintainer = False if 'ansible' in self.module_maintainers \ and comment.user.login in self.ansible_members: is_maintainer = True elif comment.user.login in self.module_maintainers: is_maintainer = True if is_maintainer and \ (maintainer_last_index == -1 or idx < maintainer_last_index): maintainer_last_index = idx if creator_last_index == -1 and maintainer_last_index == -1: waiting = True elif creator_last_index == -1 and maintainer_last_index > -1: waiting = False elif creator_last_index < maintainer_last_index: waiting = True return waiting def keep_current_main_labels(self): current_labels = self.issue.get_current_labels() for current_label in current_labels: if current_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS: self.issue.add_desired_label(name=current_label) def add_desired_labels_by_issue_type(self, comments=True): """Adds labels by defined issue type""" issue_type = self.template_data.get('issue type', False) if issue_type is False: self.issue.add_desired_label('needs_info') return if not issue_type.lower() in self.VALID_ISSUE_TYPES: # special handling for PRs if self.issue.instance.pull_request: mel = [ x for x in self.issue.current_labels if x in self.MUTUALLY_EXCLUSIVE_LABELS ] if not mel: # if only adding new files, assume it is a feature if self.patch_contains_only_new_files(): issue_type = 'feature pull request' else: if not isinstance(self.debug, bool): msg = '"%s"' % issue_type msg += ' was not a valid issue type' msg += ', adding "needs_info"' self.debug(msg) self.issue.add_desired_label('needs_info') return else: if not isinstance(self.debug, bool): msg = '"%s"' % issue_type msg += ' was not a valid issue type' msg += ', adding "needs_info"' self.debug(msg) self.issue.add_desired_label('needs_info') return desired_label = issue_type.replace(' ', '_') desired_label = desired_label.lower() desired_label = desired_label.replace('documentation', 'docs') # FIXME - shouldn't have to do this if desired_label == 'test_pull_request': desired_label = 'test_pull_requests' #import epdb; epdb.st() # is there a mutually exclusive label already? if desired_label in self.issue.MUTUALLY_EXCLUSIVE_LABELS: mel = [ x for x in self.issue.MUTUALLY_EXCLUSIVE_LABELS if x in self.issue.current_labels ] if len(mel) > 0: return if desired_label not in self.issue.get_current_labels(): self.issue.add_desired_label(name=desired_label) if len(self.issue.current_comments) == 0 and comments: # only set this if no other comments self.issue.add_desired_comment(boilerplate='issue_new') def patch_contains_only_new_files(self): '''Does the PR edit any existing files?''' oldfiles = False for x in self.issue.files: if x.filename.encode('ascii', 'ignore') in self.file_indexer.files: if not isinstance(self.debug, bool): msg = 'old file match on' msg += ' %s' % x.filename.encode('ascii', 'ignore') self.debug(msg) oldfiles = True break return not oldfiles def add_desired_labels_by_ansible_version(self): if 'ansible version' not in self.template_data: if not isinstance(self.debug, bool): self.debug(msg="no ansible version section") self.issue.add_desired_label(name="needs_info") #self.issue.add_desired_comment( # boilerplate="issue_missing_data" #) return if not self.template_data['ansible version']: if not isinstance(self.debug, bool): self.debug(msg="no ansible version defined") self.issue.add_desired_label(name="needs_info") #self.issue.add_desired_comment( # boilerplate="issue_missing_data" #) return def add_desired_labels_by_namespace(self): """Adds labels regarding module namespaces""" SKIPTOPICS = ['network/basics/'] if not self.match: return False ''' if 'component name' in self.template_data and self.match: if self.match['repository'] != self.github_repo: self.issue.add_desired_comment(boilerplate='issue_wrong_repo') ''' for key in ['topic', 'subtopic']: # ignore networking/basics if self.match[key] and not self.match['fulltopic'] in SKIPTOPICS: thislabel = self.issue.TOPIC_MAP.\ get(self.match[key], self.match[key]) if thislabel in self.valid_labels: self.issue.add_desired_label(thislabel) def render_boilerplate(self, tvars, boilerplate=None): template = environment.get_template('%s.j2' % boilerplate) comment = template.render(**tvars) return comment def render_comment(self, boilerplate=None): """Renders templates into comments using the boilerplate as filename""" maintainers = self.get_module_maintainers(expand=False) if not maintainers: # FIXME - why? maintainers = ['NO_MAINTAINER_FOUND'] submitter = self.issue.get_submitter() missing_sections = [ x for x in self.issue.REQUIRED_SECTIONS if x not in self.template_data or not self.template_data.get(x) ] if not self.match and missing_sections: # be lenient on component name for ansible/ansible if self.github_repo == 'ansible' and \ 'component name' in missing_sections: missing_sections.remove('component name') #if missing_sections: # import epdb; epdb.st() issue_type = self.template_data.get('issue type', None) if issue_type: issue_type = issue_type.lower() correct_repo = self.match.get('repository', None) template = environment.get_template('%s.j2' % boilerplate) component_name = self.template_data.get('component name', 'NULL'), comment = template.render(maintainers=maintainers, submitter=submitter, issue_type=issue_type, correct_repo=correct_repo, component_name=component_name, missing_sections=missing_sections) return comment def process_comments(self): """ Processes ISSUE comments for matching criteria to add labels""" if self.github_user not in self.BOTLIST: self.BOTLIST.append(self.github_user) module_maintainers = self.get_module_maintainers() comments = self.issue.get_comments() today = datetime.today() if not isinstance(self.debug, bool): self.debug(msg="--- START Processing Comments:") for idc, comment in enumerate(comments): if comment.user.login in self.BOTLIST: if not isinstance(self.debug, bool): self.debug(msg="%s is in botlist: " % comment.user.login) time_delta = today - comment.created_at comment_days_old = time_delta.days if not isinstance(self.debug, bool): msg = "Days since last bot comment: %s" % comment_days_old self.debug(msg=msg) if comment_days_old > 14: labels = self.issue.desired_labels if 'pending' not in comment.body: if self.issue.is_labeled_for_interaction(): if not isinstance(self.debug, bool): self.debug(msg="submitter_first_warning") self.issue.add_desired_comment( boilerplate="submitter_first_warning") break if "maintainer_review" not in labels: if not isinstance(self.debug, bool): self.debug(msg="maintainer_first_warning") self.issue.add_desired_comment( boilerplate="maintainer_first_warning") break # pending in comment.body else: if self.issue.is_labeled_for_interaction(): if not isinstance(self.debug, bool): self.debug(msg="submitter_second_warning") self.issue.add_desired_comment( boilerplate="submitter_second_warning") break if "maintainer_review" in labels: if not isinstance(self.debug, bool): self.debug(msg="maintainer_second_warning") self.issue.add_desired_comment( boilerplate="maintainer_second_warning") break if not isinstance(self.debug, bool): msg = "STATUS: no useful state change since last pass" msg += "( %s )" % comment.user.login self.debug(msg=msg) break if comment.user.login in module_maintainers \ or comment.user.login.lower() in module_maintainers\ or ('ansible' in module_maintainers and comment.user.login in self.ansible_members): if not isinstance(self.debug, bool): msg = "%s" % comment.user.login msg = " is module maintainer commented on" msg += "%s." % comment.created_at self.debug(msg=msg) if 'needs_info' in comment.body: if not isinstance(self.debug, bool): self.debug(msg="...said needs_info!") self.issue.add_desired_label(name="needs_info") elif "close_me" in comment.body: if not isinstance(self.debug, bool): self.debug(msg="...said close_me!") self.issue.add_desired_label( name="pending_action_close_me") break if comment.user.login == self.issue.get_submitter(): if not isinstance(self.debug, bool): msg = "submitter %s" % comment.user.login msg += ", commented on %s." % comment.created_at self.debug(msg=msg) if comment.user.login not in self.BOTLIST and \ comment.user.login in self.ansible_members: if not isinstance(self.debug, bool): self.debug(msg="%s is a ansible member" % comment.user.login) if not isinstance(self.debug, bool): self.debug(msg="--- END Processing Comments") def issue_type_to_label(self, issue_type): if issue_type: issue_type = issue_type.lower() issue_type = issue_type.replace(' ', '_') issue_type = issue_type.replace('documentation', 'docs') return issue_type def check_safe_match(self): """ Turn force on or off depending on match characteristics """ import epdb epdb.st() safe_match = False if self.action_count() == 0: safe_match = True elif not self.actions['close'] and not self.actions['unlabel']: if len(self.actions['newlabel']) == 1: if self.actions['newlabel'][0].startswith('affects_'): safe_match = True else: safe_match = False if self.module: if self.module in self.issue.instance.title.lower(): safe_match = True # be more lenient on re-notifications if not safe_match: if not self.actions['close'] and \ not self.actions['unlabel'] and \ not self.actions['newlabel']: if len(self.actions['comments']) == 1: if 'still waiting' in self.actions['comments'][0]: safe_match = True #import epdb; epdb.st() if safe_match: self.force = True else: self.force = False def action_count(self): """ Return the number of actions that are to be performed """ count = 0 for k, v in self.actions.iteritems(): if k in ['close', 'open', 'merge', 'close_migrated'] and v: count += 1 elif k != 'close' and k != 'open' and \ k != 'merge' and k != 'close_migrated': count += len(v) return count def apply_actions(self): action_meta = {'REDO': False} if self.safe_force: self.check_safe_match() if self.action_count() > 0: if self.dry_run: print("Dry-run specified, skipping execution of actions") else: if self.force: print("Running actions non-interactive as you forced.") self.execute_actions() return action_meta cont = raw_input( "Take recommended actions (y/N/a/R/T/DEBUG)? ") if cont in ('a', 'A'): sys.exit(0) if cont in ('Y', 'y'): self.execute_actions() if cont == 'T': self.template_wizard() action_meta['REDO'] = True if cont == 'r' or cont == 'R': action_meta['REDO'] = True if cont == 'DEBUG': # put the user into a breakpoint to do live debug action_meta['REDO'] = True import epdb epdb.st() elif self.always_pause: print("Skipping, but pause.") cont = raw_input("Continue (Y/n/a/R/T/DEBUG)? ") if cont in ('a', 'A', 'n', 'N'): sys.exit(0) if cont == 'T': self.template_wizard() action_meta['REDO'] = True elif cont == 'REDO': action_meta['REDO'] = True elif cont == 'DEBUG': # put the user into a breakpoint to do live debug import epdb epdb.st() action_meta['REDO'] = True else: print("Skipping.") # let the upper level code redo this issue return action_meta def template_wizard(self): DF = DescriptionFixer(self.issue, self.meta) print('################################################') print(DF.new_description) print('################################################') cont = raw_input("Apply this new description? (Y/N)") if cont == 'Y': self.issue.set_description(DF.new_description) def execute_actions(self): """Turns the actions into API calls""" #time.sleep(1) for comment in self.actions['comments']: logging.info("acton: comment - " + comment) self.issue.add_comment(comment=comment) if self.actions['close']: # https://github.com/PyGithub/PyGithub/blob/master/github/Issue.py#L263 logging.info('action: close') self.issue.instance.edit(state='closed') return if self.actions['close_migrated']: mi = self.get_issue_by_repopath_and_number( self.meta['migrated_issue_repo_path'], self.meta['migrated_issue_number']) logging.info('close migrated: %s' % mi.html_url) mi.instance.edit(state='closed') #import epdb; epdb.st() for unlabel in self.actions['unlabel']: logging.info('action: unlabel - ' + unlabel) self.issue.remove_label(label=unlabel) for newlabel in self.actions['newlabel']: logging.info('action: label - ' + newlabel) self.issue.add_label(label=newlabel) if 'assign' in self.actions: for user in self.actions['assign']: logging.info('action: assign - ' + user) self.issue.assign_user(user) if 'unassign' in self.actions: for user in self.actions['unassign']: logging.info('action: unassign - ' + user) self.issue.unassign_user(user) if 'merge' in self.actions: if self.actions['merge']: self.issue.merge() def smart_match_module(self): '''Fuzzy matching for modules''' if hasattr(self, 'meta'): self.meta['smart_match_module_called'] = True match = None known_modules = [] for k, v in self.module_indexer.modules.iteritems(): known_modules.append(v['name']) title = self.issue.instance.title.lower() title = title.replace(':', '') title_matches = [x for x in known_modules if x + ' module' in title] if not title_matches: title_matches = [ x for x in known_modules if title.startswith(x + ' ') ] if not title_matches: title_matches = [ x for x in known_modules if ' ' + x + ' ' in title ] cmatches = None if self.template_data.get('component name'): component = self.template_data.get('component name') cmatches = [x for x in known_modules if x in component] cmatches = [x for x in cmatches if not '_' + x in component] # use title ... ? if title_matches: cmatches = [x for x in cmatches if x in title_matches] if cmatches: if len(cmatches) >= 1: match = cmatches[0] if not match: if 'docs.ansible.com' in component: pass else: pass if not match: if len(title_matches) == 1: match = title_matches[0] else: print("module - title matches: %s" % title_matches) print("module - component matches: %s" % cmatches) return match def cache_issue(self, issue): iid = issue.instance.number fpath = os.path.join(self.cachedir, str(iid)) if not os.path.isdir(fpath): os.makedirs(fpath) fpath = os.path.join(fpath, 'iwrapper.pickle') with open(fpath, 'wb') as f: pickle.dump(issue, f) #import epdb; epdb.st() def load_cached_issues(self, state='open'): issues = [] idirs = glob.glob('%s/*' % self.cachedir) idirs = [x for x in idirs if not x.endswith('.pickle')] for idir in idirs: wfile = os.path.join(idir, 'iwrapper.pickle') if os.path.isfile(wfile): with open(wfile, 'rb') as f: wrapper = pickle.load(f) issues.append(wrapper.instance) return issues def wait_for_rate_limit(self): gh = self._connect() GithubWrapper.wait_for_rate_limit(githubobj=gh) @RateLimited def is_pr_merged(self, number, repo=None): '''Check if a PR# has been merged or not''' merged = False pr = None try: if not repo: pr = self.repo.get_pullrequest(number) else: pr = repo.get_pullrequest(number) except Exception as e: print(e) if pr: merged = pr.merged return merged def print_comment_list(self): """Print comment creators and the commands they used""" for x in self.issue.current_comments: command = None if x.user.login != 'ansibot': command = [ y for y in self.VALID_COMMANDS if y in x.body and not '!' + y in x.body ] command = ', '.join(command) else: # What template did ansibot use? try: command = x.body.split('\n')[-1].split()[-2] except: pass if command: print("\t%s %s (%s)" % (x.created_at.isoformat(), x.user.login, command)) else: print("\t%s %s" % (x.created_at.isoformat(), x.user.login))
class TriageIssues(DefaultTriager): VALID_COMMANDS = ['needs_info', '!needs_info', 'notabug', 'bot_broken', 'bot_skip', 'wontfix', 'bug_resolved', 'resolved_by_pr', 'needs_contributor', 'duplicate_of'] CLOSURE_COMMANDS = [ 'notabug', 'wontfix', 'bug_resolved', 'resolved_by_pr', 'duplicate_of' ] # re-notify interval for maintainers RENOTIFY_INTERVAL = 14 # max limit for needs_info notifications RENOTIFY_EXPIRE = 56 # re-notify interval by this number for features FEATURE_RENOTIFY_INTERVAL = 60 # the max comments per week before ansibot becomes a "spammer" MAX_BOT_COMMENTS_PER_WEEK = 5 def run(self, useapiwrapper=True): """Starts a triage run""" # how many issues have been processed self.icount = 0 # Create the api connection if not useapiwrapper: # use the default non-caching connection self.repo = self._connect().get_repo(self._get_repo_path()) else: # make use of the special caching wrapper for the api self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) # extend the ignored labels by repo if hasattr(self, 'IGNORE_LABELS_ADD'): self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD) if self.number: issue = self.repo.get_issue(int(self.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() self.process() else: last_run = None now = self.get_current_time() last_run_file = '~/.ansibullbot/cache' if self.github_repo == 'ansible': last_run_file += '/ansible/ansible/' else: last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo last_run_file += 'issues/last_run.pickle' last_run_file = os.path.expanduser(last_run_file) if os.path.isfile(last_run_file): try: with open(last_run_file, 'rb') as f: last_run = pickle.load(f) except Exception as e: print(e) if last_run and not self.no_since: self.debug('Getting issues updated/created since %s' % last_run) issues = self.repo.get_issues(since=last_run) else: self.debug('Getting ALL issues') issues = self.repo.get_issues() for issue in issues: self.icount += 1 if self.start_at and issue.number > self.start_at: continue if self.is_pr(issue): continue self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if action_res: while action_res['REDO']: issue = self.repo.get_issue(int(issue.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if not action_res: action_res = {'REDO': False} # save this run time with open(last_run_file, 'wb') as f: pickle.dump(now, f) @RateLimited def process(self, usecache=True): """Processes the Issue""" # basic processing self._process() # who maintains this? maintainers = [] if self.meta.get('component_valid', False): correct_repo = self.match.get('repository') if correct_repo != self.github_repo: self.meta['correct_repo'] = False else: self.meta['correct_repo'] = True maintainers = self.get_module_maintainers() if not maintainers: #issue_module_no_maintainer #import epdb; epdb.st() pass ########################################################### # Enumerate Actions ########################################################### self.keep_current_main_labels() self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.process_history(usecache=usecache) self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.create_actions() self.debug(msg='desired_comments: %s' % self.issue.desired_comments) print("Module: %s" % self.module) if self.match: print("MModule: %s" % self.match['name']) else: print("MModule: %s" % self.match) print("Maintainer(s): %s" \ % ', '.join(self.get_module_maintainers(expand=False))) print("Submitter: %s" % self.issue.get_submitter()) print("Total Comments: %s" % len(self.issue.current_comments)) self.print_comment_list() print("Current Labels: %s" % ', '.join(sorted(self.issue.current_labels))) # invoke the wizard import pprint; pprint.pprint(self.actions) action_meta = self.apply_actions() return action_meta def create_actions(self): """Create actions from the desired label/unlabel/comment actions""" # do nothing for bot_skip if self.meta['bot_skip']: return if 'bot_broken' in self.issue.desired_labels: # If the bot is broken, do nothing other than set the broken label self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] self.actions['close'] = False if not 'bot_broken' in self.issue.current_labels: self.actions['newlabel'] = ['bot_broken'] return # add the version labels self.create_label_version_actions() if self.issue.desired_state != self.issue.instance.state: if self.issue.desired_state == 'closed': # close the issue ... self.actions['close'] = True # We only want up to 1 comment when an issue is closed if 'issue_closure' in self.issue.desired_comments: self.issue.desired_comments = ['issue_closure'] elif 'issue_deprecated_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_deprecated_module'] else: self.issue.desired_comments = [] if self.issue.desired_comments: comment = self.render_comment( boilerplate=self.issue.desired_comments[0] ) self.actions['comments'].append(comment) return resolved_desired_labels = [] for desired_label in self.issue.desired_labels: resolved_desired_label = self.issue.resolve_desired_labels( desired_label ) if desired_label != resolved_desired_label: resolved_desired_labels.append(resolved_desired_label) if (resolved_desired_label not in self.issue.get_current_labels()): self.issue.add_desired_comment(desired_label) self.actions['newlabel'].append(resolved_desired_label) else: resolved_desired_labels.append(desired_label) if desired_label not in self.issue.get_current_labels(): self.actions['newlabel'].append(desired_label) ''' if os.path.exists("templates/" + 'issue_' + desired_label + ".j2"): self.issue.add_desired_comment('issue_' + desired_label) ''' for current_label in self.issue.get_current_labels(): if current_label in self.IGNORE_LABELS: continue if current_label not in resolved_desired_labels: self.actions['unlabel'].append(current_label) # should only make one comment at a time if len(self.issue.desired_comments) > 1: if 'issue_wrong_repo' in self.issue.desired_comments: self.issue.desired_comments = ['issue_wrong_repo'] elif 'issue_invalid_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_invalid_module'] elif 'issue_module_no_maintainer' in self.issue.desired_comments: self.issue.desired_comments = ['issue_module_no_maintainer'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' in self.issue.desired_labels: self.issue.desired_comments = ['issue_needs_info'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' not in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') else: import epdb; epdb.st() # Do not comment needs_info if it's not a label if 'issue_needs_info' in self.issue.desired_comments \ and not 'needs_info' in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') # render the comments for boilerplate in self.issue.desired_comments: comment = self.render_comment(boilerplate=boilerplate) self.debug(msg=boilerplate) self.actions['comments'].append(comment) # do not re-comment for idx,comment in enumerate(self.actions['comments']): if self.issue.current_comments: if self.issue.current_comments[-1].body == comment: self.debug(msg="Removing repeat comment from actions") self.actions['comments'].remove(comment) #import epdb; epdb.st() def create_label_version_actions(self): if not self.ansible_label_version: return expected = 'affects_%s' % self.ansible_label_version if expected not in self.valid_labels: print("NEED NEW LABEL: %s" % expected) import epdb; epdb.st() candidates = [x for x in self.issue.current_labels] candidates = [x for x in candidates if x.startswith('affects_')] candidates = [x for x in candidates if x != expected] if len(candidates) > 0: #for cand in candidates: # self.issue.pop_desired_label(name=cand) #return pass else: if expected not in self.issue.current_labels \ or expected not in self.issue.desired_labels: self.issue.add_desired_label(name=expected) #import epdb; epdb.st() def create_commment_actions(self): '''Render desired comment templates''' # should only make one comment at a time if len(self.issue.desired_comments) > 1: if 'issue_wrong_repo' in self.issue.desired_comments: self.issue.desired_comments = ['issue_wrong_repo'] elif 'issue_invalid_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_invalid_module'] elif 'issue_module_no_maintainer' in self.issue.desired_comments: self.issue.desired_comments = ['issue_module_no_maintainer'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' in self.issue.desired_labels: self.issue.desired_comments = ['issue_needs_info'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' not in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') else: import epdb; epdb.st() # Do not comment needs_info if it's not a label if 'issue_needs_info' in self.issue.desired_comments \ and not 'needs_info' in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') # render the comments for boilerplate in self.issue.desired_comments: comment = self.render_comment(boilerplate=boilerplate) self.debug(msg=boilerplate) self.actions['comments'].append(comment) # do not re-comment for idx,comment in enumerate(self.actions['comments']): if self.issue.current_comments: if self.issue.current_comments[-1].body == comment: self.debug(msg="Removing repeat comment from actions") self.actions['comments'].remove(comment) def create_label_actions(self): """Create actions from the desired label/unlabel/comment actions""" if 'bot_broken' in self.issue.desired_labels: # If the bot is broken, do nothing other than set the broken label self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] self.actions['close'] = False if not 'bot_broken' in self.issue.current_labels: self.actions['newlabel'] = ['bot_broken'] return if self.meta['bot_skip']: return resolved_desired_labels = [] for desired_label in self.issue.desired_labels: resolved_desired_label = self.issue.resolve_desired_labels( desired_label ) if desired_label != resolved_desired_label: resolved_desired_labels.append(resolved_desired_label) if (resolved_desired_label not in self.issue.get_current_labels()): self.issue.add_desired_comment(desired_label) self.actions['newlabel'].append(resolved_desired_label) else: resolved_desired_labels.append(desired_label) if desired_label not in self.issue.get_current_labels(): self.actions['newlabel'].append(desired_label) for current_label in self.issue.get_current_labels(): if current_label in self.IGNORE_LABELS: continue if current_label not in resolved_desired_labels: self.actions['unlabel'].append(current_label) def process_history(self, usecache=True): '''Steps through all known meta about the issue and decides what to do''' self.meta.update(self.get_facts()) self.add_desired_labels_by_issue_type() self.add_desired_labels_by_ansible_version() self.add_desired_labels_by_namespace() ################################################# # FINAL LOGIC LOOP ################################################# if self.meta['bot_broken']: self.debug(msg='broken bot stanza') self.issue.add_desired_label('bot_broken') elif self.meta['bot_skip']: self.debug(msg='bot skip stanza') # clear out all actions and do nothing for k,v in six.iteritems(self.actions): if type(v) == list: self.actions[k] = [] self.actions['close'] = False elif self.meta['bot_spam']: self.debug(msg='bot spam stanza') # clear out all actions and do nothing for k,v in six.iteritems(self.actions): if type(v) == list: self.actions[k] = [] self.actions['close'] = False # do we mark this somehow? self.issue.add_desired_label('bot_broken') elif self.match and self.match.get('deprecated', False) \ and 'feature_idea' in self.issue.desired_labels: self.debug(msg='deprecated module stanza') # Make the deprecated comment self.issue.desired_comments = ['issue_deprecated_module'] # Close the issue ... self.issue.set_desired_state('closed') elif not self.meta['maintainers_known'] and self.meta['valid_module']: self.debug(msg='unknown maintainer stanza') self.issue.desired_comments = ['issue_module_no_maintainer'] elif self.meta['maintainer_closure']: self.debug(msg='maintainer closure stanza') # Need to close the issue ... self.issue.set_desired_state('closed') elif self.meta['new_module_request']: self.debug(msg='new module request stanza') self.issue.desired_comments = [] for label in self.issue.current_labels: if not label in self.issue.desired_labels: self.issue.desired_labels.append(label) elif not self.meta['correct_repo']: self.debug(msg='wrong repo stanza') self.issue.desired_comments = ['issue_wrong_repo'] self.actions['close'] = True elif not self.meta['valid_module'] and \ not self.meta['maintainer_command_needsinfo']: self.debug(msg='invalid module stanza') self.issue.add_desired_label('needs_info') if 'issue_invalid_module' not in self.issue.current_bot_comments \ and not 'issue_needs_info' in self.issue.current_bot_comments: self.issue.desired_comments = ['issue_invalid_module'] elif not self.meta['notification_maintainers'] and \ not self.meta['maintainer_command_needsinfo']: self.debug(msg='no maintainer stanza') self.issue.add_desired_label('waiting_on_maintainer') self.issue.add_desired_comment("issue_module_no_maintainer") elif self.meta['maintainer_command'] == 'needs_contributor': # maintainer can't or won't fix this, but would like someone else to self.debug(msg='maintainer needs contributor stanza') self.issue.add_desired_label('waiting_on_contributor') elif self.meta['maintainer_waiting_on']: self.debug(msg='maintainer wait stanza') self.issue.add_desired_label('waiting_on_maintainer') if len(self.issue.current_comments) == 0: # new issue if self.meta['issue_type']: if self.meta['submitter'] not in self.meta['notification_maintainers']: # ping the maintainer self.issue.add_desired_comment('issue_new') else: # do not send intial ping to maintainer if also submitter if 'issue_new' in self.issue.desired_comments: self.issue.desired_comments.remove('issue_new') else: # old issue -- renotify if not self.match['deprecated'] and self.meta['notification_maintainers']: if self.meta['maintainer_to_ping']: self.issue.add_desired_comment("issue_notify_maintainer") elif self.meta['maintainer_to_reping']: self.issue.add_desired_comment("issue_renotify_maintainer") elif self.meta['submitter_waiting_on']: self.debug(msg='submitter wait stanza') if 'waiting_on_maintainer' in self.issue.desired_labels: self.issue.desired_labels.remove('waiting_on_maintainer') if (self.meta['needsinfo_add'] or self.meta['missing_sections']) \ or (not self.meta['needsinfo_remove'] and self.meta['missing_sections']) \ or (self.meta['needsinfo_add'] and not self.meta['missing_sections']): #import epdb; epdb.st() self.issue.add_desired_label('needs_info') if len(self.issue.current_comments) == 0 or \ not self.meta['maintainer_commented']: if self.issue.current_bot_comments: if 'issue_needs_info' not in self.issue.current_bot_comments: self.issue.add_desired_comment("issue_needs_info") else: self.issue.add_desired_comment("issue_needs_info") # needs_info: warn if stale, close if expired elif self.meta['needsinfo_expired']: self.issue.add_desired_comment("issue_closure") self.issue.set_desired_state('closed') elif self.meta['needsinfo_stale'] \ and (self.meta['submitter_to_ping'] or self.meta['submitter_to_reping']): self.issue.add_desired_comment("issue_pending_closure") def get_history_facts(self, usecache=True): return self.get_facts(usecache=usecache) def get_facts(self, usecache=True): '''Only used by the ansible/ansible triager at the moment''' hfacts = {} today = self.get_current_time() self.history = HistoryWrapper( self.issue, usecache=usecache, cachedir=self.cachedir ) # what was the last commment? bot_broken = False if self.issue.current_comments: for comment in self.issue.current_comments: if 'bot_broken' in comment.body: bot_broken = True # did someone from ansible want this issue skipped? bot_skip = False if self.issue.current_comments: for comment in self.issue.current_comments: if 'bot_skip' in comment.body and comment.user.login in self.ansible_members: bot_skip = True ######################################################### # Facts for new "triage" workflow ######################################################### # these logins don't count for the next set of facts exclude_logins = ['ansibot', 'gregdek', self.issue.instance.user.login] # Has a human (not ansibot + not submitter) ever changed labels? lchanges = [x for x in self.history.history if x['event'] in ['labeled', 'unlabeled'] and x['actor'] not in exclude_logins] if lchanges: hfacts['human_labeled'] = True else: hfacts['human_labeled'] = False # Has a human (not ansibot + not submitter) ever changed the title? tchanges = [x for x in self.history.history if x['event'] in ['renamed'] and x['actor'] not in exclude_logins] if tchanges: hfacts['human_retitled'] = True else: hfacts['human_retitled'] = False # Has a human (not ansibot + not submitter) ever changed the description? # ... github doesn't record an event for this. Lame. # Has a human (not ansibot + not submitter) ever set a milestone? mchanges = [x for x in self.history.history if x['event'] in ['milestoned', 'demilestoned'] and x['actor'] not in exclude_logins] if mchanges: hfacts['human_milestoned'] = True else: hfacts['human_milestoned'] = False # Has a human (not ansibot + not submitter) ever set/unset assignees? achanges = [x for x in self.history.history if x['event'] in ['assigned', 'unassigned'] and x['actor'] not in exclude_logins] if achanges: hfacts['human_assigned'] = True else: hfacts['human_assigned'] = False ######################################################### # Has the bot been overzealous with comments? hfacts['bot_spam'] = False bcg = self.history.get_user_comments_groupby('ansibot', groupby='w') for k,v in six.iteritems(bcg): if v >= self.MAX_BOT_COMMENTS_PER_WEEK: hfacts['bot_spam'] = True # Is this a new module? hfacts['new_module_request'] = False if 'feature_idea' in self.issue.desired_labels: if self.template_data['component name'] == 'new': hfacts['new_module_request'] = True # who made this and when did they last comment? submitter = self.issue.get_submitter() submitter_last_commented = self.history.last_commented_at(submitter) if not submitter_last_commented: submitter_last_commented = self.issue.instance.created_at #import epdb; epdb.st() submitter_last_comment = self.history.last_comment(submitter) submitter_last_notified = self.history.last_notified(submitter) # what did they not provide? missing_sections = self.issue.get_missing_sections() # Is this a valid module? if self.match: self.meta['valid_module'] = True else: self.meta['valid_module'] = False # Filed in the right place? if self.meta['valid_module']: if self.match['repository'] != self.github_repo: hfacts['correct_repo'] = False else: hfacts['correct_repo'] = True else: hfacts['correct_repo'] = True # DEBUG + FIXME - speeds up bulk triage if 'component name' in missing_sections \ and (self.match or self.github_repo == 'ansible'): missing_sections.remove('component name') #import epdb; epdb.st() # Who are the maintainers? maintainers = [x for x in self.get_module_maintainers()] #hfacts['maintainers'] = maintainers #import epdb; epdb.st() # Set a fact to indicate that we know the maintainer self.meta['maintainers_known'] = False if maintainers: self.meta['maintainers_known'] = True if 'ansible' in maintainers: maintainers.remove('ansible') maintainers.extend(self.ansible_members) if 'ansibot' in maintainers: maintainers.remove('ansibot') if submitter in maintainers: maintainers.remove(submitter) maintainers = sorted(set(maintainers)) # Has maintainer been notified? When? notification_maintainers = [x for x in self.get_module_maintainers()] if 'ansible' in notification_maintainers: notification_maintainers.extend(self.ansible_members) if 'ansibot' in notification_maintainers: notification_maintainers.remove('ansibot') hfacts['notification_maintainers'] = notification_maintainers maintainer_last_notified = self.history.\ last_notified(notification_maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) maintainer_last_viewed = self.history.last_viewed_at(maintainers) #import epdb; epdb.st() # Has maintainer been mentioned? maintainer_mentioned = self.history.is_mentioned(maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) # Has the maintainer ever responded? maintainer_commented = self.history.has_commented(maintainers) maintainer_last_commented = self.history.last_commented_at(maintainers) maintainer_last_comment = self.history.last_comment(maintainers) maintainer_comments = self.history.get_user_comments(maintainers) #import epdb; epdb.st() # Was the maintainer the last commentor? last_commentor_ismaintainer = False last_commentor_issubmitter = False last_commentor = self.history.last_commentor() if last_commentor in maintainers and last_commentor != self.github_user: last_commentor_ismaintainer = True elif last_commentor == submitter: last_commentor_issubmitter = True # Did the maintainer issue a command? maintainer_commands = self.history.get_commands(maintainers, self.VALID_COMMANDS) # Keep all commands hfacts['maintainer_commands'] = maintainer_commands # Set a bit for the last command given if hfacts['maintainer_commands']: hfacts['maintainer_command'] = hfacts['maintainer_commands'][-1] else: hfacts['maintainer_command'] = None # Is the last command a closure command? if hfacts['maintainer_command'] in self.CLOSURE_COMMANDS: hfacts['maintainer_closure'] = True else: hfacts['maintainer_closure'] = False # handle resolved_by_pr ... if 'resolved_by_pr' in maintainer_commands: maintainer_comments = self.history.get_user_comments(maintainers) maintainer_comments = [x for x in reversed(maintainer_comments) \ if 'resolved_by_pr' in x] for comment in maintainer_comments: pr_number = extract_pr_number_from_comment(comment) hfacts['resolved_by_pr'] = { 'number': pr_number, 'merged': self.is_pr_merged(pr_number), } if not hfacts['resolved_by_pr']['merged']: hfacts['maintainer_closure'] = False break # needs_info toggles ni_commands = [x for x in maintainer_commands if 'needs_info' in x] # Has the maintainer ever subscribed? maintainer_subscribed = self.history.has_subscribed(maintainers) # Was it ever needs_info? was_needs_info = self.history.was_labeled(label='needs_info') needsinfo_last_applied = self.history.label_last_applied('needs_info') needsinfo_last_removed = self.history.label_last_removed('needs_info') # Still needs_info? needsinfo_add = False needsinfo_remove = False if 'needs_info' in self.issue.current_labels: if not needsinfo_last_applied or not submitter_last_commented: import epdb; epdb.st() if submitter_last_commented > needsinfo_last_applied: needsinfo_add = False needsinfo_remove = True #if 'needs_info' in maintainer_commands and maintainer_last_commented: if ni_commands and maintainer_last_commented: if ni_commands[-1] == 'needs_info': #import epdb; epdb.st() if submitter_last_commented and maintainer_last_commented: if submitter_last_commented > maintainer_last_commented: needsinfo_add = False needsinfo_remove = True else: needsinfo_add = True needsinfo_remove = False else: needsinfo_add = False needsinfo_remove = True # Save existing needs_info if not time to remove ... if 'needs_info' in self.issue.current_labels \ and not needsinfo_add \ and not needsinfo_remove: needsinfo_add = True if ni_commands and maintainer_last_commented: if maintainer_last_commented > submitter_last_commented: if ni_commands[-1] == 'needs_info': needsinfo_add = True needsinfo_remove = False else: needsinfo_add = False needsinfo_remove = True #import epdb; epdb.st() # Is needs_info stale or expired? needsinfo_age = None needsinfo_stale = False needsinfo_expired = False if 'needs_info' in self.issue.current_labels: time_delta = today - needsinfo_last_applied needsinfo_age = time_delta.days if needsinfo_age > self.RENOTIFY_INTERVAL: needsinfo_stale = True if needsinfo_age > self.RENOTIFY_EXPIRE: needsinfo_expired = True # Should we be in waiting_on_maintainer mode? maintainer_waiting_on = False if (needsinfo_remove or not needsinfo_add) \ or not was_needs_info \ and not missing_sections: maintainer_waiting_on = True # Should we [re]notify the submitter? submitter_waiting_on = False submitter_to_ping = False submitter_to_reping = False if not maintainer_waiting_on: submitter_waiting_on = True if missing_sections: submitter_waiting_on = True maintainer_waiting_on = False # use [!]needs_info to set final state if ni_commands: if ni_commands[-1] == '!needs_info': submitter_waiting_on = False maintainer_waiting_on = True elif ni_commands[-1] == 'needs_info': submitter_waiting_on = True maintainer_waiting_on = False # Time to [re]ping maintainer? maintainer_to_ping = False maintainer_to_reping = False if maintainer_waiting_on: # if feature idea, extend the notification interval interval = self.RENOTIFY_INTERVAL if self.meta.get('issue_type', None) == 'feature idea' \ or 'feature_idea' in self.issue.current_labels: interval = self.FEATURE_RENOTIFY_INTERVAL if maintainer_viewed and not maintainer_last_notified: time_delta = today - maintainer_last_viewed view_age = time_delta.days if view_age > interval: maintainer_to_reping = True elif maintainer_last_notified: time_delta = today - maintainer_last_notified ping_age = time_delta.days if ping_age > interval: maintainer_to_reping = True else: maintainer_to_ping = True # Time to [re]ping the submitter? if submitter_waiting_on: if submitter_last_notified: time_delta = today - submitter_last_notified notification_age = time_delta.days if notification_age > self.RENOTIFY_INTERVAL: submitter_to_reping = True else: submitter_to_reping = False submitter_to_ping = False else: submitter_to_ping = True submitter_to_reping = False # needs_contributor ... hfacts['needs_contributor'] = False for command in maintainer_commands: if command == 'needs_contributor': hfacts['needs_contributor'] = True elif command == '!needs_contributor': hfacts['needs_contributor'] = False hfacts['bot_broken'] = bot_broken hfacts['bot_skip'] = bot_skip hfacts['missing_sections'] = missing_sections hfacts['was_needsinfo'] = was_needs_info hfacts['needsinfo_age'] = needsinfo_age hfacts['needsinfo_stale'] = needsinfo_stale hfacts['needsinfo_expired'] = needsinfo_expired hfacts['needsinfo_add'] = needsinfo_add hfacts['needsinfo_remove'] = needsinfo_remove hfacts['notification_maintainers'] = self.get_module_maintainers() or 'ansible' hfacts['maintainer_last_notified'] = maintainer_last_notified hfacts['maintainer_commented'] = maintainer_commented hfacts['maintainer_viewed'] = maintainer_viewed hfacts['maintainer_subscribed'] = maintainer_subscribed hfacts['maintainer_command_needsinfo'] = 'needs_info' in maintainer_commands hfacts['maintainer_command_not_needsinfo'] = '!needs_info' in maintainer_commands hfacts['maintainer_waiting_on'] = maintainer_waiting_on hfacts['maintainer_to_ping'] = maintainer_to_ping hfacts['maintainer_to_reping'] = maintainer_to_reping hfacts['submitter'] = submitter hfacts['submitter_waiting_on'] = submitter_waiting_on hfacts['submitter_to_ping'] = submitter_to_ping hfacts['submitter_to_reping'] = submitter_to_reping hfacts['last_commentor_ismaintainer'] = last_commentor_ismaintainer hfacts['last_commentor_issubmitter'] = last_commentor_issubmitter hfacts['last_commentor'] = last_commentor return hfacts
class TriageIssues(DefaultTriager): VALID_COMMANDS = [ 'needs_info', '!needs_info', 'notabug', 'wontfix', 'bug_resolved', 'resolved_by_pr', 'needs_contributor', 'duplicate_of' ] def run(self, useapiwrapper=True): """Starts a triage run""" # how many issues have been processed self.icount = 0 # Create the api connection if not useapiwrapper: # use the default non-caching connection self.repo = self._connect().get_repo(self._get_repo_path()) else: # make use of the special caching wrapper for the api self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) # extend the ignored labels by repo if hasattr(self, 'IGNORE_LABELS_ADD'): self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD) if self.number: issue = self.repo.get_issue(int(self.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() self.process() else: last_run = None now = self.get_current_time() last_run_file = '~/.ansibullbot/cache' if self.github_repo == 'ansible': last_run_file += '/ansible/ansible/' else: last_run_file += '/ansible/ansible-modules-%s/' % self.github_repo last_run_file += 'issues/last_run.pickle' last_run_file = os.path.expanduser(last_run_file) if os.path.isfile(last_run_file): try: with open(last_run_file, 'rb') as f: last_run = pickle.load(f) except Exception as e: print(e) if last_run and not self.no_since: self.debug('Getting issues updated/created since %s' % last_run) issues = self.repo.get_issues(since=last_run) else: self.debug('Getting ALL issues') issues = self.repo.get_issues() for issue in issues: self.icount += 1 if self.start_at and issue.number > self.start_at: continue if self.is_pr(issue): continue self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if action_res: while action_res['REDO']: issue = self.repo.get_issue(int(issue.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.get_events() self.issue.get_comments() action_res = self.process() if not action_res: action_res = {'REDO': False} # save this run time with open(last_run_file, 'wb') as f: pickle.dump(now, f) #import epdb; epdb.st() @ratecheck() def process(self, usecache=True): """Processes the Issue""" # basic processing self._process() # filed under the correct repository? this_repo = False correct_repo = None # who maintains this? maintainers = [] component_isvalid = self.meta.get('component_isvalid', False) if not component_isvalid: pass else: correct_repo = self.module_indexer.\ get_repository_for_module(component) if correct_repo == self.github_repo: this_repo = True maintainers = self.get_module_maintainers() # Has the maintainer -ever- commented? maintainer_commented = False if component_isvalid: maintainer_commented = self.has_maintainer_commented() waiting_on_maintainer = False if component_isvalid: waiting_on_maintainer = self.is_waiting_on_maintainer() # How long ago did the maintainer last comment? maintainer_last_comment_age = -1 if component_isvalid: maintainer_last_comment_age = self.age_of_last_maintainer_comment() ########################################################### # Enumerate Actions ########################################################### self.keep_current_main_labels() self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.process_history(usecache=usecache) self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.create_actions() self.debug(msg='desired_comments: %s' % self.issue.desired_comments) print("Module: %s" % self.module) if self.match: print("MModule: %s" % self.match['name']) else: print("MModule: %s" % self.match) print("Maintainer(s): %s" \ % ', '.join(self.get_module_maintainers(expand=False))) print("Submitter: %s" % self.issue.get_submitter()) print("Total Comments: %s" % len(self.issue.current_comments)) for x in self.issue.current_comments: print("\t%s %s %s" % (x.created_at.isoformat(), x.user.login, [y for y in self.VALID_COMMANDS if y in x.body \ and not '!' + y in x.body] or '')) print("Current Labels: %s" % ', '.join(sorted(self.issue.current_labels))) import pprint pprint.pprint(self.actions) # invoke the wizard action_meta = self.apply_actions() return action_meta def create_actions(self): """Create actions from the desired label/unlabel/comment actions""" if 'bot_broken' in self.issue.desired_labels: # If the bot is broken, do nothing other than set the broken label self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] self.actions['close'] = False if not 'bot_broken' in self.issue.current_labels: self.actions['newlabel'] = ['bot_broken'] return # add the version labels self.create_label_version_actions() if self.issue.desired_state != self.issue.instance.state: if self.issue.desired_state == 'closed': # close the issue ... self.actions['close'] = True if 'issue_closure' in self.issue.desired_comments: comment = self.render_comment(boilerplate='issue_closure') self.actions['comments'].append(comment) return resolved_desired_labels = [] for desired_label in self.issue.desired_labels: resolved_desired_label = self.issue.resolve_desired_labels( desired_label) if desired_label != resolved_desired_label: resolved_desired_labels.append(resolved_desired_label) if (resolved_desired_label not in self.issue.get_current_labels()): self.issue.add_desired_comment(desired_label) self.actions['newlabel'].append(resolved_desired_label) else: resolved_desired_labels.append(desired_label) if desired_label not in self.issue.get_current_labels(): self.actions['newlabel'].append(desired_label) ''' if os.path.exists("templates/" + 'issue_' + desired_label + ".j2"): self.issue.add_desired_comment('issue_' + desired_label) ''' for current_label in self.issue.get_current_labels(): if current_label in self.IGNORE_LABELS: continue if current_label not in resolved_desired_labels: self.actions['unlabel'].append(current_label) # should only make one comment at a time if len(self.issue.desired_comments) > 1: if 'issue_wrong_repo' in self.issue.desired_comments: self.issue.desired_comments = ['issue_wrong_repo'] elif 'issue_invalid_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_invalid_module'] elif 'issue_module_no_maintainer' in self.issue.desired_comments: self.issue.desired_comments = ['issue_module_no_maintainer'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' in self.issue.desired_labels: self.issue.desired_comments = ['issue_needs_info'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' not in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') else: import epdb epdb.st() # Do not comment needs_info if it's not a label if 'issue_needs_info' in self.issue.desired_comments \ and not 'needs_info' in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') # render the comments for boilerplate in self.issue.desired_comments: comment = self.render_comment(boilerplate=boilerplate) self.debug(msg=boilerplate) self.actions['comments'].append(comment) # do not re-comment for idx, comment in enumerate(self.actions['comments']): if self.issue.current_comments: if self.issue.current_comments[-1].body == comment: self.debug(msg="Removing repeat comment from actions") self.actions['comments'].remove(comment) #import epdb; epdb.st() def create_label_version_actions(self): if not self.ansible_label_version: return expected = 'affects_%s' % self.ansible_label_version if expected not in self.valid_labels: print("NEED NEW LABEL: %s" % expected) import epdb epdb.st() candidates = [x for x in self.issue.current_labels] candidates = [x for x in candidates if x.startswith('affects_')] candidates = [x for x in candidates if x != expected] if len(candidates) > 0: #for cand in candidates: # self.issue.pop_desired_label(name=cand) #return pass else: if expected not in self.issue.current_labels \ or expected not in self.issue.desired_labels: self.issue.add_desired_label(name=expected) #import epdb; epdb.st() def create_commment_actions(self): '''Render desired comment templates''' # should only make one comment at a time if len(self.issue.desired_comments) > 1: if 'issue_wrong_repo' in self.issue.desired_comments: self.issue.desired_comments = ['issue_wrong_repo'] elif 'issue_invalid_module' in self.issue.desired_comments: self.issue.desired_comments = ['issue_invalid_module'] elif 'issue_module_no_maintainer' in self.issue.desired_comments: self.issue.desired_comments = ['issue_module_no_maintainer'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' in self.issue.desired_labels: self.issue.desired_comments = ['issue_needs_info'] elif 'issue_needs_info' in self.issue.desired_comments \ and 'needs_info' not in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') else: import epdb epdb.st() # Do not comment needs_info if it's not a label if 'issue_needs_info' in self.issue.desired_comments \ and not 'needs_info' in self.issue.desired_labels: self.issue.desired_comments.remove('issue_needs_info') # render the comments for boilerplate in self.issue.desired_comments: comment = self.render_comment(boilerplate=boilerplate) self.debug(msg=boilerplate) self.actions['comments'].append(comment) # do not re-comment for idx, comment in enumerate(self.actions['comments']): if self.issue.current_comments: if self.issue.current_comments[-1].body == comment: self.debug(msg="Removing repeat comment from actions") self.actions['comments'].remove(comment) def create_label_actions(self): """Create actions from the desired label/unlabel/comment actions""" if 'bot_broken' in self.issue.desired_labels: # If the bot is broken, do nothing other than set the broken label self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] self.actions['close'] = False if not 'bot_broken' in self.issue.current_labels: self.actions['newlabel'] = ['bot_broken'] return resolved_desired_labels = [] for desired_label in self.issue.desired_labels: resolved_desired_label = self.issue.resolve_desired_labels( desired_label) if desired_label != resolved_desired_label: resolved_desired_labels.append(resolved_desired_label) if (resolved_desired_label not in self.issue.get_current_labels()): self.issue.add_desired_comment(desired_label) self.actions['newlabel'].append(resolved_desired_label) else: resolved_desired_labels.append(desired_label) if desired_label not in self.issue.get_current_labels(): self.actions['newlabel'].append(desired_label) for current_label in self.issue.get_current_labels(): if current_label in self.IGNORE_LABELS: continue if current_label not in resolved_desired_labels: self.actions['unlabel'].append(current_label) def process_history(self, usecache=True): self.meta = {} today = self.get_current_time() # Build the history self.debug(msg="Building event history ...") self.history = HistoryWrapper(self.issue, usecache=usecache, cachedir=self.cachedir) # what was the last commment? bot_broken = False if self.issue.current_comments: for comment in self.issue.current_comments: if 'bot_broken' in comment.body: bot_broken = True # who made this and when did they last comment? submitter = self.issue.get_submitter() submitter_last_commented = self.history.last_commented_at(submitter) submitter_last_comment = self.history.last_comment(submitter) submitter_last_notified = self.history.last_notified(submitter) # what did they not provide? missing_sections = self.issue.get_missing_sections() if 'ansible version' in missing_sections: missing_sections.remove('ansible version') # DEBUG + FIXME - speeds up bulk triage if 'component name' in missing_sections and self.match: missing_sections.remove('component name') #import epdb; epdb.st() # Is this a valid module? valid_module = False correct_repo = True if self.match: valid_module = True if self.match['repository'] != self.github_repo: correct_repo = False #import epdb; epdb.st() # Do these after evaluating the module self.add_desired_labels_by_issue_type() if 'ansible version' in missing_sections: #self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.add_desired_labels_by_ansible_version() #self.debug(msg='desired_comments: %s' % self.issue.desired_comments) self.add_desired_labels_by_namespace() #self.debug(msg='desired_comments: %s' % self.issue.desired_comments) # Who are the maintainers? maintainers = [x for x in self.get_module_maintainers()] #if 'ansible' in maintainers: # maintainers.remove('ansible') # maintainers += self.ansible_members #print("MAINTAINERS: %s" % maintainers) if 'ansible' in maintainers: maintainers.remove('ansible') maintainers.extend(self.ansible_members) if 'ansibot' in maintainers: maintainers.remove('ansibot') if submitter in maintainers: maintainers.remove(submitter) maintainers = sorted(set(maintainers)) # Has maintainer been notified? When? notification_maintainers = self.get_module_maintainers() if 'ansible' in notification_maintainers: notification_maintainers.extend(self.ansible_members) if 'ansibot' in notification_maintainers: notification_maintainers.remove('ansibot') maintainer_last_notified = self.history.\ last_notified(notification_maintainers) #import epdb; epdb.st() # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) maintainer_last_viewed = self.history.last_viewed_at(maintainers) #import epdb; epdb.st() # Has maintainer been mentioned? maintainer_mentioned = self.history.is_mentioned(maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) # Has the maintainer ever responded? maintainer_commented = self.history.has_commented(maintainers) maintainer_last_commented = self.history.last_commented_at(maintainers) maintainer_last_comment = self.history.last_comment(maintainers) maintainer_comments = self.history.get_user_comments(maintainers) #import epdb; epdb.st() # Was the maintainer the last commentor? last_commentor_ismaintainer = False last_commentor_issubmitter = False last_commentor = self.history.last_commentor() if last_commentor in maintainers and last_commentor != self.github_user: last_commentor_ismaintainer = True elif last_commentor == submitter: last_commentor_issubmitter = True # Did the maintainer issue a command? maintainer_commands = self.history.get_commands( maintainers, self.VALID_COMMANDS) #import epdb; epdb.st() maintainer_command_close = False maintainer_command_needsinfo = False maintainer_command_not_needsinfo = False maintainer_command_notabug = False maintainer_command_wontfix = False maintainer_command_resolved_bug = False maintainer_command_resolved_pr = False maintainer_command_needscontributor = False maintainer_command_duplicateof = False if maintainer_commented and not maintainer_last_comment: print('ERROR: should have a comment from maintainer') import epdb epdb.st() elif maintainer_last_comment and last_commentor_ismaintainer: maintainer_last_comment = maintainer_last_comment.strip() if 'needs_info' in maintainer_last_comment \ and not '!needs_info' in maintainer_last_comment: maintainer_command_needsinfo = True elif '!needs_info' in maintainer_last_comment: maintainer_command_not_needsinfo = True elif 'notabug' in maintainer_last_comment: maintainer_command_notabug = True maintainer_command_close = True elif 'wontfix' in maintainer_last_comment: maintainer_command_wontfix = True maintainer_command_close = True elif 'bug_resolved' in maintainer_last_comment: maintainer_command_resolved_bug = True maintainer_command_close = True elif 'resolved_by_pr' in maintainer_last_comment: maintainer_command_resolved_pr = True maintainer_command_close = True elif 'needs_contributor' in maintainer_last_comment: maintainer_command_needscontributor = True elif 'duplicate_of' in maintainer_last_comment: maintainer_command_duplicateof = True maintainer_command_close = True elif maintainer_commands: # are there any persistant commands? if 'needs_contributor' in maintainer_commands: maintainer_command_needscontributor = True elif not missing_sections and not submitter_last_commented and maintainer_commands[ -1] == 'needs_info': maintainer_command_needsinfo = True elif not missing_sections and not submitter_last_commented and maintainer_commands[ -1] == '!needs_info': maintainer_command_not_needsinfo = True #import epdb; epdb.st() #import epdb; epdb.st() # Has the maintainer ever subscribed? maintainer_subscribed = self.history.has_subscribed(maintainers) # Was it ever needs_info? was_needs_info = self.history.was_labeled(label='needs_info') needsinfo_last_applied = self.history.label_last_applied('needs_info') needsinfo_last_removed = self.history.label_last_removed('needs_info') #import epdb; epdb.st() # Still needs_info? needsinfo_add = False needsinfo_remove = False if 'needs_info' in self.issue.current_labels: if submitter_last_commented and needsinfo_last_applied: if submitter_last_commented > needsinfo_last_applied \ and not missing_sections: needsinfo_remove = True if maintainer_command_needsinfo and maintainer_last_commented: if submitter_last_commented and maintainer_last_commented: if submitter_last_commented > maintainer_last_commented: needsinfo_add = False needsinfo_remove = True else: needsinfo_add = True needsinfo_remove = False # Is needs_info stale or expired? needsinfo_age = None needsinfo_stale = False needsinfo_expired = False if 'needs_info' in self.issue.current_labels: time_delta = today - needsinfo_last_applied needsinfo_age = time_delta.days if needsinfo_age > 14: needsinfo_stale = True if needsinfo_age > 56: needsinfo_expired = True # Should we be in waiting_on_maintainer mode? maintainer_waiting_on = False if (needsinfo_remove or not needsinfo_add) \ or not was_needs_info \ and not missing_sections: maintainer_waiting_on = True # Should we [re]notify the submitter? submitter_waiting_on = False submitter_to_ping = False submitter_to_reping = False if not maintainer_waiting_on: submitter_waiting_on = True if missing_sections: submitter_waiting_on = True maintainer_waiting_on = False else: if 'needs_info' in self.issue.current_labels \ and not maintainer_command_not_needsinfo: needsinfo_remove = True submitter_waiting_on = False maintainer_waiting_on = True if maintainer_command_not_needsinfo: submitter_waiting_on = False maintainer_waiting_on = True if maintainer_command_needsinfo: submitter_waiting_on = True maintainer_waiting_on = False needsinfo_add = True needsinfo_remove = False # Time to [re]ping maintainer? maintainer_to_ping = False maintainer_to_reping = False if maintainer_waiting_on: #import epdb; epdb.st() if maintainer_viewed and not maintainer_last_notified: time_delta = today - maintainer_last_viewed view_age = time_delta.days if view_age > 14: maintainer_to_reping = True elif maintainer_last_notified: time_delta = today - maintainer_last_notified ping_age = time_delta.days if ping_age > 14: maintainer_to_reping = True else: maintainer_to_ping = True # Time to [re]ping the submitter? if submitter_waiting_on: if submitter_last_notified: time_delta = today - submitter_last_notified notification_age = time_delta.days if notification_age > 14: submitter_to_reping = True else: submitter_to_reping = False submitter_to_ping = False else: submitter_to_ping = True submitter_to_reping = False issue_type = self.template_data.get('issue type', None) issue_type = self.issue_type_to_label(issue_type) self.meta['issue_type'] = issue_type # new module requests need to disable everything # https://github.com/ansible/ansible-modules-core/issues/4112 # https://github.com/ansible/ansible-modules-core/issues/2267 # https://github.com/ansible/ansible-modules-core/issues/2626 # https://github.com/ansible/ansible-modules-core/issues/645 new_module_request = False if 'feature_idea' in self.issue.desired_labels: if self.template_data['component name'] == 'new': new_module_request = True # reset the maintainers maintainers = self.get_module_maintainers() ################################################# # FINAL LOGIC LOOP ################################################# if bot_broken: self.debug(msg='broken bot stanza') self.issue.add_desired_label('bot_broken') elif maintainer_command_close: self.debug(msg='maintainer closure stanza') # Need to close the issue ... self.issue.set_desired_state('closed') elif new_module_request: self.debug(msg='new module request stanza') self.issue.desired_comments = [] for label in self.issue.current_labels: if not label in self.issue.desired_labels: self.issue.desired_labels.append(label) elif not correct_repo: self.debug(msg='wrong repo stanza') self.issue.desired_comments = ['issue_wrong_repo'] self.actions['close'] = True #import epdb; epdb.st() elif not valid_module and not maintainer_command_needsinfo: self.debug(msg='invalid module stanza') #import epdb; epdb.st() self.issue.add_desired_label('needs_info') if 'issue_invalid_module' not in self.issue.current_bot_comments \ and not 'issue_needs_info' in self.issue.current_bot_comments: self.issue.desired_comments = ['issue_invalid_module'] elif not maintainers and not maintainer_command_needsinfo: self.debug(msg='no maintainer stanza') self.issue.add_desired_label('waiting_on_maintainer') self.issue.add_desired_comment("issue_module_no_maintainer") elif maintainer_command_needscontributor: # maintainer can't or won't fix this, but would like someone else to self.debug(msg='maintainer needs contributor stanza') self.issue.add_desired_label('waiting_on_contributor') #import epdb; epdb.st() elif maintainer_waiting_on: self.debug(msg='maintainer wait stanza') self.issue.add_desired_label('waiting_on_maintainer') if len(self.issue.current_comments) == 0: if issue_type: if submitter not in self.module_maintainers: self.issue.add_desired_comment('issue_new') else: if 'issue_new' in self.issue.desired_comments: self.issue.desired_comments.remove('issue_new') #import epdb; epdb.st() else: if maintainers != ['DEPRECATED']: if maintainer_to_ping and maintainers: self.issue.add_desired_comment( "issue_notify_maintainer") elif maintainer_to_reping and maintainers: self.issue.add_desired_comment( "issue_renotify_maintainer") #import epdb; epdb.st() elif submitter_waiting_on: self.debug(msg='submitter wait stanza') #import epdb; epdb.st() if 'waiting_on_maintainer' in self.issue.desired_labels: self.issue.desired_labels.remove('waiting_on_maintainer') if (needsinfo_add or missing_sections) \ or (not needsinfo_remove and missing_sections) \ or (needsinfo_add and not missing_sections): self.issue.add_desired_label('needs_info') #import epdb; epdb.st() if len(self.issue.current_comments ) == 0 or not maintainer_commented: self.issue.add_desired_comment("issue_needs_info") # needs_info: warn if stale, close if expired elif needsinfo_expired: self.issue.add_desired_comment("issue_closure") self.issue.set_desired_state('closed') elif needsinfo_stale \ and (submitter_to_ping or submitter_to_reping): self.issue.add_desired_comment("issue_pending_closure") #import epdb; epdb.st() #import epdb; epdb.st() def get_history_facts(self, usecache=True): hfacts = {} today = self.get_current_time() self.history = HistoryWrapper(self.issue, usecache=usecache, cachedir=self.cachedir) # what was the last commment? bot_broken = False if self.issue.current_comments: for comment in self.issue.current_comments: if 'bot_broken' in comment.body: bot_broken = True # who made this and when did they last comment? submitter = self.issue.get_submitter() submitter_last_commented = self.history.last_commented_at(submitter) if not submitter_last_commented: submitter_last_commented = self.issue.instance.created_at #import epdb; epdb.st() submitter_last_comment = self.history.last_comment(submitter) submitter_last_notified = self.history.last_notified(submitter) # what did they not provide? missing_sections = self.issue.get_missing_sections() #if 'ansible version' in missing_sections: # missing_sections.remove('ansible version') # DEBUG + FIXME - speeds up bulk triage if 'component name' in missing_sections \ and (self.match or self.github_repo == 'ansible'): missing_sections.remove('component name') #import epdb; epdb.st() # Who are the maintainers? maintainers = [x for x in self.get_module_maintainers()] if 'ansible' in maintainers: maintainers.remove('ansible') maintainers.extend(self.ansible_members) if 'ansibot' in maintainers: maintainers.remove('ansibot') if submitter in maintainers: maintainers.remove(submitter) maintainers = sorted(set(maintainers)) # Has maintainer been notified? When? notification_maintainers = self.get_module_maintainers() if 'ansible' in notification_maintainers: notification_maintainers.extend(self.ansible_members) if 'ansibot' in notification_maintainers: notification_maintainers.remove('ansibot') maintainer_last_notified = self.history.\ last_notified(notification_maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) maintainer_last_viewed = self.history.last_viewed_at(maintainers) #import epdb; epdb.st() # Has maintainer been mentioned? maintainer_mentioned = self.history.is_mentioned(maintainers) # Has maintainer viewed issue? maintainer_viewed = self.history.has_viewed(maintainers) # Has the maintainer ever responded? maintainer_commented = self.history.has_commented(maintainers) maintainer_last_commented = self.history.last_commented_at(maintainers) maintainer_last_comment = self.history.last_comment(maintainers) maintainer_comments = self.history.get_user_comments(maintainers) #import epdb; epdb.st() # Was the maintainer the last commentor? last_commentor_ismaintainer = False last_commentor_issubmitter = False last_commentor = self.history.last_commentor() if last_commentor in maintainers and last_commentor != self.github_user: last_commentor_ismaintainer = True elif last_commentor == submitter: last_commentor_issubmitter = True # Did the maintainer issue a command? maintainer_commands = self.history.get_commands( maintainers, self.VALID_COMMANDS) # needs_info toggles ni_commands = [x for x in maintainer_commands if 'needs_info' in x] # Has the maintainer ever subscribed? maintainer_subscribed = self.history.has_subscribed(maintainers) # Was it ever needs_info? was_needs_info = self.history.was_labeled(label='needs_info') needsinfo_last_applied = self.history.label_last_applied('needs_info') needsinfo_last_removed = self.history.label_last_removed('needs_info') # Still needs_info? needsinfo_add = False needsinfo_remove = False if 'needs_info' in self.issue.current_labels: if not needsinfo_last_applied or not submitter_last_commented: import epdb epdb.st() if submitter_last_commented > needsinfo_last_applied: needsinfo_add = False needsinfo_remove = True #if 'needs_info' in maintainer_commands and maintainer_last_commented: if ni_commands and maintainer_last_commented: if ni_commands[-1] == 'needs_info': #import epdb; epdb.st() if submitter_last_commented and maintainer_last_commented: if submitter_last_commented > maintainer_last_commented: needsinfo_add = False needsinfo_remove = True else: needsinfo_add = True needsinfo_remove = False else: needsinfo_add = False needsinfo_remove = True # Save existing needs_info if not time to remove ... if 'needs_info' in self.issue.current_labels \ and not needsinfo_add \ and not needsinfo_remove: needsinfo_add = True if ni_commands and maintainer_last_commented: if maintainer_last_commented > submitter_last_commented: if ni_commands[-1] == 'needs_info': needsinfo_add = True needsinfo_remove = False else: needsinfo_add = False needsinfo_remove = True #import epdb; epdb.st() # Is needs_info stale or expired? needsinfo_age = None needsinfo_stale = False needsinfo_expired = False if 'needs_info' in self.issue.current_labels: time_delta = today - needsinfo_last_applied needsinfo_age = time_delta.days if needsinfo_age > 14: needsinfo_stale = True if needsinfo_age > 56: needsinfo_expired = True # Should we be in waiting_on_maintainer mode? maintainer_waiting_on = False if (needsinfo_remove or not needsinfo_add) \ or not was_needs_info \ and not missing_sections: maintainer_waiting_on = True # Should we [re]notify the submitter? submitter_waiting_on = False submitter_to_ping = False submitter_to_reping = False if not maintainer_waiting_on: submitter_waiting_on = True if missing_sections: submitter_waiting_on = True maintainer_waiting_on = False # use [!]needs_info to set final state if ni_commands: if ni_commands[-1] == '!needs_info': submitter_waiting_on = False maintainer_waiting_on = True elif ni_commands[-1] == 'needs_info': submitter_waiting_on = True maintainer_waiting_on = False # Time to [re]ping maintainer? maintainer_to_ping = False maintainer_to_reping = False if maintainer_waiting_on: #import epdb; epdb.st() if maintainer_viewed and not maintainer_last_notified: time_delta = today - maintainer_last_viewed view_age = time_delta.days if view_age > 14: maintainer_to_reping = True elif maintainer_last_notified: time_delta = today - maintainer_last_notified ping_age = time_delta.days if ping_age > 14: maintainer_to_reping = True else: maintainer_to_ping = True # Time to [re]ping the submitter? if submitter_waiting_on: if submitter_last_notified: time_delta = today - submitter_last_notified notification_age = time_delta.days if notification_age > 14: submitter_to_reping = True else: submitter_to_reping = False submitter_to_ping = False else: submitter_to_ping = True submitter_to_reping = False hfacts['bot_broken'] = bot_broken hfacts['missing_sections'] = missing_sections hfacts['was_needsinfo'] = was_needs_info hfacts['needsinfo_age'] = needsinfo_age hfacts['needsinfo_stale'] = needsinfo_stale hfacts['needsinfo_expired'] = needsinfo_expired hfacts['needsinfo_add'] = needsinfo_add hfacts['needsinfo_remove'] = needsinfo_remove hfacts['notification_maintainers'] = self.get_module_maintainers( ) or 'ansible' hfacts['maintainer_last_notified'] = maintainer_last_notified hfacts['maintainer_commented'] = maintainer_commented hfacts['maintainer_viewed'] = maintainer_viewed hfacts['maintainer_subscribed'] = maintainer_subscribed hfacts[ 'maintainer_command_needsinfo'] = 'needs_info' in maintainer_commands hfacts[ 'maintainer_command_not_needsinfo'] = '!needs_info' in maintainer_commands hfacts['maintainer_waiting_on'] = maintainer_waiting_on hfacts['maintainer_to_ping'] = maintainer_to_ping hfacts['maintainer_to_reping'] = maintainer_to_reping hfacts['submitter'] = submitter hfacts['submitter_waiting_on'] = submitter_waiting_on hfacts['submitter_to_ping'] = submitter_to_ping hfacts['submitter_to_reping'] = submitter_to_reping hfacts['last_commentor_ismaintainer'] = last_commentor_ismaintainer hfacts['last_commentor_issubmitter'] = last_commentor_issubmitter hfacts['last_commentor'] = last_commentor #import epdb; epdb.st() return hfacts
def run(self, useapiwrapper=True): # how many issues have been processed self.icount = 0 # Create the api connection if not useapiwrapper: # use the default non-caching connection self.repo = self._connect().get_repo(self._get_repo_path()) else: # make use of the special caching wrapper for the api self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) # make a list of valid assignees print('Getting valid assignees') self.valid_assignees = [x.login for x in self.repo.get_assignees()] # extend the ignored labels by repo if hasattr(self, 'IGNORE_LABELS_ADD'): self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD) if self.number: # get the issue issue = self.repo.get_issue(int(self.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS self.issue.valid_assignees = self.valid_assignees self.issue.get_events() self.issue.get_comments() # get the PR and it's properties self.issue.pullrequest = self.repo.get_pullrequest(int( self.number)) self.issue.get_commits() self.issue.get_files() self.issue.get_review_comments() # do the work self.process() else: # need to get the PRs print('Getting ALL pullrequests') pullrequests = self.repo.get_pullrequests(since=None) # iterate for idp, pr in enumerate(pullrequests): # get the issue and make a wrapper issue = self.repo.get_issue(int(pr.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS self.issue.valid_assignees = self.valid_assignees self.issue.get_events() self.issue.get_comments() # get the PR and it's properties self.issue.pullrequest = pr self.issue.get_commits() self.issue.get_files() self.issue.get_review_comments() # do the work self.process()
class TriageV3(DefaultTriager): def __init__(self, args): self.args = args self.last_run = None self.daemonize = None self.daemonize_interval = None self.dry_run = False self.force = False self.gh_pass = None self.github_pass = None self.gh_token = None self.github_token = None self.gh_user = None self.github_user = None self.logfile = None self.no_since = False self.only_closed = False self.only_issues = False self.only_open = False self.only_prs = False self.pause = False self.pr = False self.repo = None self.safe_force = False self.skiprepo = [] self.start_at = False self.verbose = False # where to store junk self.cachedir = '~/.ansibullbot/cache' self.cachedir = os.path.expanduser(self.cachedir) # repo objects self.repos = {} self.set_logger() logging.info('starting bot') logging.debug('setting bot attributes') attribs = dir(self.args) attribs = [x for x in attribs if not x.startswith('_')] for x in attribs: val = getattr(self.args, x) setattr(self, x, val) #import epdb; epdb.st() if self.args.daemonize: logging.info('starting daemonize loop') self.loop() else: logging.info('starting single run') self.run() logging.info('stopping bot') def set_logger(self): logging.level = logging.INFO logFormatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") rootLogger = logging.getLogger() rootLogger.setLevel(logging.INFO) fileHandler = logging.FileHandler("{0}/{1}".format( os.path.dirname(self.args.logfile), os.path.basename(self.args.logfile)) ) fileHandler.setFormatter(logFormatter) rootLogger.addHandler(fileHandler) consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(logFormatter) rootLogger.addHandler(consoleHandler) def loop(self): '''Call the run method in a defined interval''' while True: self.run() interval = self.args.daemonize_interval logging.info('sleep %ss (%sm)' % (interval, interval / 60)) time.sleep(interval) def run(self): '''Primary execution method''' self.collect_issues() #if self.last_run: # wissues = self.get_updated_issues(since=self.last_run) #else: # wissues = self.get_updated_issues() for item in self.repos.items(): repopath = item[0] if repopath in self.skiprepo: continue repoobj = item[1]['repo'] issues = item[1]['issues'] for i_item in issues.items(): ik = i_item[0] iw = i_item[1] logging.info(iw) if iw.state == 'closed': logging.info(iw + ' is closed, skipping') continue #hcache = os.path.join(self.cachedir, iw.repo_full_name, 'issues') hcache = os.path.join(self.cachedir, iw.repo_full_name) action_meta = None if iw.repo_full_name in MREPOS: if iw.created_at >= REPOMERGEDATE: # close new module issues+prs immediately logging.info('module issue created -after- merge') action_meta = self.close_module_issue_with_message(iw) else: # process history # - check if message was given, comment if not # - if X days after message, close PRs, move issues. logging.info('module issue created -before- merge') logging.info('build history') hw = self.get_history(iw, usecache=True, cachedir=hcache) logging.info('history built') lc = hw.last_date_for_boilerplate('repomerge') if lc: lcdelta = (datetime.datetime.now() - lc).days else: lcdelta = None kwargs = {} # missing the comment? if lc: kwargs['bp'] = 'repomerge' else: kwargs['bp'] = None # should it be closed or not? if iw.is_pullrequest(): if lc and lcdelta > MREPO_CLOSE_WINDOW: kwargs['close'] = True action_meta = self.close_module_issue_with_message(iw, **kwargs) elif not lc: # add the comment self.add_repomerge_comment(iw) else: # do nothing pass else: kwargs['close'] = False if lc and lcdelta > MREPO_CLOSE_WINDOW: # move it for them self.move_issue(iw) elif not lc: # add the comment self.add_repomerge_comment(iw) else: # do nothing pass else: # ansible/ansible triage pass pprint(action_meta) #import epdb; epdb.st() def move_issue(self, issue): pass def add_repomerge_comment(self, issue, bp='repomerge'): '''Add the comment without closing''' self.actions = {} self.actions['close'] = False self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] # stubs for the comment templater self.module_maintainers = [] self.module = None self.issue = issue self.template_data = {} self.github_repo = issue.repo_full_name self.match = {} comment = self.render_comment(boilerplate=bp) self.actions['comments'] = [comment] pprint(self.actions) action_meta = self.apply_actions() return action_meta def close_module_issue_with_message(self, issue, bp='repomerge_new'): '''After the repomerge, new issues+prs in the module repos should be closed''' self.actions = {} self.actions['close'] = True self.actions['comments'] = [] self.actions['newlabel'] = [] self.actions['unlabel'] = [] # stubs for the comment templater self.module_maintainers = [] self.module = None self.issue = issue self.template_data = {} self.github_repo = issue.repo_full_name self.match = {} comment = self.render_comment(boilerplate=bp) self.actions['comments'] = [comment] pprint(self.actions) action_meta = self.apply_actions() return action_meta def collect_issues(self): '''Populate the local cache of issues''' # this should do a few things: # 1. collect all issues (open+closed) via a webcrawler # 2. index the issues into a sqllite database so we can query on update times # 3. set an abstracted object that takes in queries logging.info('start collecting issues') logging.debug('creating github connection object') self.gh = self._connect() logging.info('creating github connection wrapper') self.ghw = GithubWrapper(self.gh) for repo in REPOS: if repo in self.skiprepo: continue #import epdb; epdb.st() logging.info('getting repo obj for %s' % repo) cachedir = os.path.join(self.cachedir, repo) self.repos[repo] = {} self.repos[repo]['repo'] = self.ghw.get_repo(repo, verbose=False) self.repos[repo]['issues'] = {} logging.info('getting issue objs for %s' % repo) issues = self.repos[repo]['repo'].get_issues() for issue in issues: iw = IssueWrapper( repo=self.repos[repo]['repo'], issue=issue, cachedir=cachedir ) self.repos[repo]['issues'][iw.number] = iw logging.info('getting issue objs for %s complete' % repo) logging.info('finished collecting issues') def get_updated_issues(self, since=None): '''Get issues to work on''' # this should return a list of issueids that have changed since the last run logging.info('start querying updated issues') # these need to be tuples (namespace, repo, number) issueids = [] logging.info('finished querying updated issues') return issueids def get_history(self, issue, usecache=True, cachedir=None): history = HistoryWrapper(issue, usecache=usecache, cachedir=cachedir) return history
def run(self, useapiwrapper=True): # how many issues have been processed self.icount = 0 # Create the api connection if not useapiwrapper: # use the default non-caching connection self.repo = self._connect().get_repo(self._get_repo_path()) else: # make use of the special caching wrapper for the api self.gh = self._connect() self.ghw = GithubWrapper(self.gh) self.repo = self.ghw.get_repo(self._get_repo_path()) # make a list of valid assignees print('Getting valid assignees') self.valid_assignees = [x.login for x in self.repo.get_assignees()] # extend the ignored labels by repo if hasattr(self, 'IGNORE_LABELS_ADD'): self.IGNORE_LABELS.extend(self.IGNORE_LABELS_ADD) if self.number: # get the issue issue = self.repo.get_issue(int(self.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS self.issue.valid_assignees = self.valid_assignees self.issue.get_events() self.issue.get_comments() # get the PR and it's properties self.issue.pullrequest = self.repo.get_pullrequest(int(self.number)) self.issue.get_commits() self.issue.get_files() self.issue.get_review_comments() # do the work self.process() else: # need to get the PRs print('Getting ALL pullrequests') pullrequests = self.repo.get_pullrequests(since=None) # iterate for idp,pr in enumerate(pullrequests): # get the issue and make a wrapper issue = self.repo.get_issue(int(pr.number)) self.issue = IssueWrapper(repo=self.repo, issue=issue, cachedir=self.cachedir) self.issue.MUTUALLY_EXCLUSIVE_LABELS = self.MUTUALLY_EXCLUSIVE_LABELS self.issue.valid_assignees = self.valid_assignees self.issue.get_events() self.issue.get_comments() # get the PR and it's properties self.issue.pullrequest = pr self.issue.get_commits() self.issue.get_files() self.issue.get_review_comments() # do the work self.process()