def test_get_no_component_commands(): iw = IssueWrapperMock() iw._comments = [ { 'id': 1, 'actor': 'jimi-c', 'body': 'unicorns are awesome', 'event': 'commented', 'created_at': datetime.datetime.utcnow(), } ] iw._events = iw._comments cachedir = tempfile.mkdtemp() hw = HistoryWrapper(iw, cachedir=cachedir, usecache=False) hw.BOTNAMES = [] events = hw._find_events_by_actor('commented', None) ccommands = hw.get_component_commands() assert len(events) == 1 assert len(ccommands) == 0
def test_ignore_events_without_dates_on_last_methods(): """With the addition of timeline events, we have a lot of missing keys that we would normally get from the events endpoint. This test asserts that the historywrapper filters those timeline events out when the necessary keys are missing.""" events = [ {'event': 'labeled', 'created_at': datetime.datetime.utcnow(), 'actor': 'bcoca', 'label': 'needs_info'}, {'event': 'labeled', 'created_at': datetime.datetime.utcnow(), 'actor': 'bcoca', 'label': 'needs_info'}, {'event': 'comment', 'created_at': datetime.datetime.utcnow(), 'actor': 'ansibot', 'body': 'foobar\n<!--- boilerplate: needs_info --->'}, {'event': 'comment', 'actor': 'ansibot', 'body': 'foobar\n<!--- boilerplate: needs_info --->'}, {'event': 'labeled', 'created_at': datetime.datetime.utcnow(), 'actor': 'ansibot', 'label': 'needs_info'}, {'event': 'labeled', 'actor': 'ansibot', 'label': 'needs_info'}, {'event': 'comment', 'created_at': datetime.datetime.utcnow(), 'actor': 'jimi-c', 'body': 'unicorns are awesome'}, {'event': 'comment', 'actor': 'jimi-c', 'body': 'unicorns are awesome'}, {'event': 'unlabeled', 'created_at': datetime.datetime.utcnow(), 'actor': 'ansibot', 'label': 'needs_info'}, {'event': 'unlabeled', 'actor': 'ansibot', 'label': 'needs_info'}, ] iw = IssueWrapperMock() for event in events: if event['event'] == 'comment': iw._comments.append(event) iw._events.append(event) cachedir = tempfile.mkdtemp() hw = HistoryWrapper(iw, cachedir=cachedir, usecache=False) hw.BOTNAMES = ['ansibot'] res = [] res.append(hw.label_last_applied('needs_info')) res.append(hw.label_last_removed('needs_info')) res.append(hw.last_date_for_boilerplate('needs_info')) res.append(hw.was_labeled('needs_info')) res.append(hw.was_unlabeled('needs_info')) assert not [x for x in res if x is None]
def history(self): if self._history is False: self._history = HistoryWrapper(self, cachedir=self.cachedir) return self._history
def history(self): if self._history is False: self._history = \ HistoryWrapper(self, cachedir=self.cachedir, usecache=True) return self._history
class IssueWrapper: def __init__(self, github=None, repo=None, issue=None, cachedir=None, gitrepo=None): self.github = github self.repo = repo self.instance = issue self.cachedir = cachedir self.gitrepo = gitrepo self.meta = {} self._assignees = UnsetValue self._committer_emails = False self._committer_logins = False self._commits = False self._events = UnsetValue self._history = UnsetValue self._labels = False self._merge_commits = False self._pr = False self._pr_reviews = False self._repo_full_name = False self._template_data = None self._pull_raw = None self._pr_files = None self.full_cachedir = os.path.join(self.cachedir, 'issues', str(self.number)) self._renamed_files = None self._pullrequest_check_runs = None @property def url(self): return self.instance.url @property def comments(self): return [x for x in self.history.history if x['event'] == 'commented'] @property def events(self): if self._events is UnsetValue: self._events = self._parse_events(self._get_timeline()) return self._events def _parse_events(self, events): processed_events = [] for event_no, dd in enumerate(events): if dd['event'] == 'committed': # FIXME # commits are added through HistoryWrapper.merge_commits() continue # reviews do not have created_at keys if not dd.get('created_at') and dd.get('submitted_at'): dd['created_at'] = dd['submitted_at'] # commits do not have created_at keys if not dd.get('created_at') and dd.get('author'): dd['created_at'] = dd['author']['date'] # commit comments do not have created_at keys if not dd.get('created_at') and dd.get('comments'): dd['created_at'] = dd['comments'][0]['created_at'] if not dd.get('created_at'): raise AssertionError(dd) # commits do not have actors if not dd.get('actor'): dd['actor'] = {'login': None} # fix commits with no message if dd['event'] == 'committed' and 'message' not in dd: dd['message'] = '' if not dd.get('id'): # set id as graphql node_id OR make one up if 'node_id' in dd: dd['id'] = dd['node_id'] else: dd['id'] = '%s/%s/%s/%s' % ( self.repo_full_name, self.number, 'timeline', event_no) event = { 'id': dd['id'], 'actor': dd['actor']['login'], 'event': dd['event'], } if isinstance(dd['created_at'], str): dd['created_at'] = strip_time_safely(dd['created_at']) event['created_at'] = pytz.utc.localize(dd['created_at']) if dd['event'] in ['labeled', 'unlabeled']: event['label'] = dd.get('label', {}).get('name', None) elif dd['event'] == 'referenced': event['commit_id'] = dd['commit_id'] elif dd['event'] == 'assigned': event['assignee'] = dd['assignee']['login'] event['assigner'] = event['actor'] elif dd['event'] == 'commented': event['body'] = dd['body'] elif dd['event'] == 'cross-referenced': event['source'] = dd['source'] processed_events.append(event) return sorted(processed_events, key=lambda x: x['created_at']) def _get_timeline(self): '''Use python-requests instead of pygithub''' data = None cache_data = os.path.join(self.full_cachedir, 'timeline_data.json') cache_meta = os.path.join(self.full_cachedir, 'timeline_meta.json') logging.debug(cache_data) if not os.path.exists(self.full_cachedir): os.makedirs(self.full_cachedir) meta = {} fetch = False if not os.path.exists(cache_data): fetch = True else: with open(cache_meta) as f: meta = json.loads(f.read()) if not fetch and (not meta or meta.get('updated_at', 0) < self.updated_at.isoformat()): fetch = True # validate the data is not infected by ratelimit errors if not fetch: with open(cache_data) as f: data = json.loads(f.read()) if isinstance(data, list): bad_events = [x for x in data if not isinstance(x, dict)] if bad_events: fetch = True else: fetch = True if data is None: fetch = True if fetch: url = self.url + '/timeline' data = self.github.get_request(url) with open(cache_meta, 'w') as f: f.write( json.dumps({ 'updated_at': self.updated_at.isoformat(), 'url': url })) with open(cache_data, 'w') as f: f.write(json.dumps(data)) return data @RateLimited def load_update_fetch_files(self): edata = None events = [] updated = None update = False write_cache = False pfile = os.path.join(self.full_cachedir, 'files.pickle') pdir = os.path.dirname(pfile) logging.debug(pfile) if not os.path.isdir(pdir): os.makedirs(pdir) if os.path.isfile(pfile): try: with open(pfile, 'rb') as f: edata = pickle.load(f) except Exception: update = True write_cache = True # check the timestamp on the cache if edata: updated = edata[0] events = edata[1] if updated < self.instance.updated_at: update = True write_cache = True # pull all events if timestamp is behind or no events cached if update or not events: write_cache = True updated = datetime.datetime.utcnow() events = [x for x in self.pullrequest.get_files()] if C.DEFAULT_PICKLE_ISSUES: if write_cache or not os.path.isfile(pfile): # need to dump the pickle back to disk edata = [updated, events] with open(pfile, 'wb') as f: pickle.dump(edata, f) return events @RateLimited def get_labels(self): """Pull the list of labels on this Issue""" labels = [] for label in self.instance.labels: labels.append(label.name) return labels @property def template_data(self): if self._template_data is None: self._template_data = get_template_data(self) return self._template_data @RateLimited def add_label(self, label=None): """Adds a label to the Issue using the GitHub API""" self.instance.add_to_labels(label) @RateLimited def remove_label(self, label=None): """Removes a label from the Issue using the GitHub API""" self.instance.remove_from_labels(label) @RateLimited def add_comment(self, comment=None): """Adds a comment to the Issue using the GitHub API""" self.instance.create_comment(comment) @RateLimited def remove_comment_by_id(self, commentid): if not isinstance(commentid, int): raise Exception("commentIds must be integers!") comment_url = os.path.join(C.DEFAULT_GITHUB_URL, 'repos', self.repo_full_name, 'issues', 'comments', str(commentid)) current_data = self.github.get_request(comment_url) if current_data and current_data.get('message') != 'Not Found': ok = self.github.delete_request(comment_url) if not ok: raise Exception("failed to delete commentid %s for %s" % (commentid, self.html_url)) @property def assignees(self): if self._assignees is UnsetValue: self._assignees = [x.login for x in self.instance.assignees] return self._assignees def is_pullrequest(self): return self.github_type == 'pullrequest' def is_issue(self): return self.github_type == 'issue' @property def age(self): created = self.created_at now = datetime.datetime.utcnow() age = now - created return age @property def title(self): return self.instance.title @property def repo_full_name(self): '''return the <org>/<repo> string''' # prefer regex over making GET calls if self._repo_full_name is False: try: url = self.url full_name = re.search(r'repos\/\w+\/\w+\/', url).group() full_name = full_name.replace('repos/', '') full_name = full_name.strip('/') except Exception: full_name = self.repo.repo.full_name self._repo_full_name = full_name return self._repo_full_name @property def html_url(self): return self.instance.html_url @property def created_at(self): return self.instance.created_at @property def updated_at(self): # this is a hack to fix unit tests if self.instance is not None: if self.instance.updated_at is not None: return self.instance.updated_at return datetime.datetime.utcnow() @property def closed_at(self): return self.instance.closed_at @property def merged_at(self): return self.instance.merged_at @property def state(self): return self.instance.state @property def github_type(self): if '/pull/' in self.html_url: return 'pullrequest' else: return 'issue' @property def number(self): return self.instance.number @property def submitter(self): # auto-migrated issue by ansibot{-dev} # figure out the original submitter if self.instance.user.login in C.DEFAULT_BOT_NAMES: m = re.match('From @(.*) on', self.instance.body) if m: return m.group(1) return self.instance.user.login @property def pullrequest(self): if not self._pr: logging.debug('@pullrequest.get_pullrequest #%s' % self.number) self._pr = self.repo.get_pullrequest(self.number) return self._pr def update_pullrequest(self): if self.is_pullrequest(): # the underlying call is wrapper with ratelimited ... self._pr = self.repo.get_pullrequest(self.number) self._pr_reviews = False self._merge_commits = False self._committer_emails = False @property @RateLimited def pullrequest_check_runs(self): if self._pullrequest_check_runs is None: logging.info('fetching pull request check runs') self._pullrequest_check_runs = self.commits[-1].get_check_runs() return self._pullrequest_check_runs @property @RateLimited def pullrequest_raw_data(self): if not self._pull_raw: logging.info('@pullrequest_raw_data') self._pull_raw = self.pullrequest.raw_data return self._pull_raw @property def pr_files(self): if self._pr_files is None: self._pr_files = self.load_update_fetch_files() return self._pr_files @property def files(self): if self.is_issue(): return None return [x.filename for x in self.pr_files] @property def new_files(self): new_files = [x for x in self.files if x not in self.gitrepo.files] new_files = [x for x in new_files if not self.gitrepo.existed(x)] return new_files @property def new_modules(self): new_modules = self.new_files new_modules = [ x for x in new_modules if x.startswith('lib/ansible/modules') ] new_modules = [ x for x in new_modules if not os.path.basename(x) == '__init__.py' ] new_modules = [ x for x in new_modules if not os.path.basename(x).startswith('_') ] new_modules = [ x for x in new_modules if not os.path.basename(x).endswith('.ps1') ] return new_modules @property def body(self): return self.instance.body @property def labels(self): if self._labels is False: self._labels = [x for x in self.get_labels()] return self._labels @property def reviews(self): if self._pr_reviews is False: self._pr_reviews = [ r.raw_data for r in self.pullrequest.get_reviews() ] # https://github.com/ansible/ansibullbot/issues/881 # https://github.com/ansible/ansibullbot/issues/883 for idx, x in enumerate(self._pr_reviews): if 'commit_id' not in x: self._pr_reviews[idx]['commit_id'] = None return self._pr_reviews @property def history(self): if self._history is UnsetValue: self._history = HistoryWrapper(self, cachedir=self.cachedir) if self.is_pullrequest(): self._history.merge_reviews(self.reviews) self._history.merge_commits(self.commits) return self._history @property def commits(self): if self._commits is False: self._commits = self.get_commits() return self._commits @RateLimited def get_commits(self): if not self.is_pullrequest(): return None commits = [x for x in self.pullrequest.get_commits()] return commits @property def mergeable(self): return self.pullrequest.mergeable @property def mergeable_state(self): if not self.is_pullrequest() or self.pullrequest.state == 'closed': return None # http://stackoverflow.com/a/30620973 fetchcount = 0 while self.pullrequest.mergeable_state == 'unknown': fetchcount += 1 if fetchcount >= 10: logging.warning('exceeded fetch threshold for mstate') return False logging.warning( 're-fetch[%s] PR#%s because mergeable state is unknown' % (fetchcount, self.number)) self.update_pullrequest() time.sleep(1) return self.pullrequest.mergeable_state @property def wip(self): return (self.title.startswith('WIP') or '[WIP]' in self.title or (self.is_pullrequest() and self.pullrequest.draft)) @property def incoming_repo_exists(self): return self.pullrequest.head.repo is not None @property def incoming_repo_slug(self): try: return self.pullrequest.head.repo.full_name except TypeError: return None @property def from_fork(self): if not self.incoming_repo_exists: return True return self.incoming_repo_slug != self.repo.repo.full_name @RateLimited def get_commit_files(self, commit): cdata = self.github.get_cached_request(commit.url) files = cdata.get('files', []) return files @RateLimited def get_commit_login(self, commit): cdata = self.github.get_cached_request(commit.url) # https://github.com/ansible/ansibullbot/issues/1265 # some commits are created from outside github and have no assocatied login if ('author' in cdata and cdata['author'] is None) or \ ('author' not in cdata): return '' login = cdata['author']['login'] return login @property @RateLimited def merge_commits(self): # https://api.github.com/repos/ansible/ansible/pulls/91/commits if self._merge_commits is False: self._merge_commits = [] for commit in self.commits: commit_data = self.github.get_cached_request(commit.url) if len(commit_data['parents']) > 1 or commit_data['commit'][ 'message'].startswith('Merge branch'): self._merge_commits.append(commit) return self._merge_commits @property def committer_emails(self): if self._committer_emails is False: self._committer_emails = [] for commit in self.commits: self.committer_emails.append(commit.commit.author.email) return self._committer_emails @property def committer_logins(self): if self._committer_logins is False: self._committer_logins = [] for commit in self.commits: self.committer_logins.append(self.get_commit_login(commit)) return self._committer_logins def merge(self): if self.merge_commits: return None # unique the lists so that we can tell how many people # have worked on this particular pullrequest emails = sorted(set(self.committer_emails)) logins = sorted(set(self.committer_logins)) if len(self.commits) == 1 or len(emails) == 1 or len(logins) == 1: # squash single committer PRs merge_method = 'squash' elif (len(self.commits) == len(emails)) and len(self.commits) <= 10: # rebase multi-committer PRs merge_method = 'rebase' else: logging.error('merge skipped for %s' % self.number) return merge_status = self.pullrequest.merge(merge_method=merge_method) if merge_status.merged: logging.info('merge successful for %s' % self.number) else: logging.error('merge failed on %s - %s' % (self.number, merge_status.messsage)) raise Exception('merge failed on %s - %s' % (self.number, merge_status.messsage)) @property def renamed_files(self): ''' A map of renamed files to prevent other code from thinking these are new files ''' if self._renamed_files is not None: return self._renamed_files self._renamed_files = {} if self.is_issue(): return self._renamed_files for x in self.commits: rd = x.raw_data for filed in rd.get('files', []): if filed.get('previous_filename'): src = filed['previous_filename'] dst = filed['filename'] self._renamed_files[dst] = src return self._renamed_files