def test_get_payload(self): sender = RepoPushWebhookSender() _ci = lambda x: MagicMock(webhook_info={'id': str(x)}, parent_ids=['0']) with patch.object(self.git.repo, 'commit', new=_ci): with h.push_config(c, app=self.git): result = sender.get_payload(commit_ids=['1', '2', '3'], ref='ref') expected_result = { 'size': 3, 'commits': [{ 'id': '1' }, { 'id': '2' }, { 'id': '3' }], 'ref': u'ref', 'after': u'1', 'before': u'0', 'repository': { 'full_name': u'/adobe/adobe-1/src/', 'name': u'Git', 'url': u'http://localhost/adobe/adobe-1/src/', }, } assert_equal(result, expected_result)
def test_send(self, send_webhook): sender = RepoPushWebhookSender() sender.get_payload = Mock() with h.push_config(c, app=self.git): sender.send(dict(arg1=1, arg2=2)) send_webhook.post.assert_called_once_with( self.wh._id, sender.get_payload.return_value)
def test_send_no_configured_webhooks(self, send_webhook): self.wh.delete() session(self.wh).flush(self.wh) sender = RepoPushWebhookSender() with h.push_config(c, app=self.git): sender.send(dict(arg1=1, arg2=2)) assert_equal(send_webhook.post.call_count, 0)
def test_webhook_payload(self): user = M.User.upsert('cory') email = user.claim_address('*****@*****.**') email.confirmed = True session(email).flush(email) user = M.User.upsert('rick') email = user.claim_address('*****@*****.**') email.confirmed = True session(email).flush(email) sender = RepoPushWebhookSender() cids = list(self.repo.all_commit_ids())[:2] payload = sender.get_payload(commit_ids=cids, ref='refs/heads/zz') expected_payload = { 'size': 2, 'ref': 'refs/heads/zz', 'after': '5c47243c8e424136fd5cdd18cd94d34c66d1955c', 'before': 'df30427c488aeab84b2352bdf88a3b19223f9d7a', 'commits': [{ 'id': '5c47243c8e424136fd5cdd18cd94d34c66d1955c', 'url': 'http://localhost/p/test/src-git/ci/5c47243c8e424136fd5cdd18cd94d34c66d1955c/', 'timestamp': datetime.datetime(2013, 3, 28, 18, 54, 16), 'message': 'Not repo root', 'author': {'name': 'Cory Johns', 'email': '*****@*****.**', 'username': '******'}, 'committer': {'name': 'Cory Johns', 'email': '*****@*****.**', 'username': '******'}, 'added': ['bad'], 'removed': [], 'modified': [], 'copied': [], 'renamed': [], }, { 'id': '1e146e67985dcd71c74de79613719bef7bddca4a', 'url': 'http://localhost/p/test/src-git/ci/1e146e67985dcd71c74de79613719bef7bddca4a/', 'timestamp': datetime.datetime(2010, 10, 7, 18, 44, 11), 'message': 'Change README', 'author': {'name': 'Rick Copeland', 'email': '*****@*****.**', 'username': '******'}, 'committer': {'name': 'Rick Copeland', 'email': '*****@*****.**', 'username': '******'}, 'added': [], 'removed': [], 'modified': ['README'], 'copied': [], 'renamed': [], }], 'repository': { 'name': 'Git', 'full_name': '/p/test/src-git/', 'url': 'http://localhost/p/test/src-git/', }, } assert_equals(payload, expected_payload)
def test_webhook_payload(self): user = M.User.upsert('cory') email = user.claim_address('*****@*****.**') email.confirmed = True session(email).flush(email) user = M.User.upsert('rick') email = user.claim_address('*****@*****.**') email.confirmed = True session(email).flush(email) sender = RepoPushWebhookSender() cids = list(self.repo.all_commit_ids())[:2] payload = sender.get_payload(commit_ids=cids, ref='refs/heads/zz') expected_payload = { 'size': 2, 'ref': u'refs/heads/zz', 'after': u'5c47243c8e424136fd5cdd18cd94d34c66d1955c', 'before': u'df30427c488aeab84b2352bdf88a3b19223f9d7a', 'commits': [{ 'id': u'5c47243c8e424136fd5cdd18cd94d34c66d1955c', 'url': u'http://localhost/p/test/src-git/ci/5c47243c8e424136fd5cdd18cd94d34c66d1955c/', 'timestamp': datetime.datetime(2013, 3, 28, 18, 54, 16), 'message': u'Not repo root', 'author': {'name': u'Cory Johns', 'email': u'*****@*****.**', 'username': '******'}, 'committer': {'name': u'Cory Johns', 'email': u'*****@*****.**', 'username': '******'}, 'added': [u'bad'], 'removed': [], 'modified': [], 'copied': [], 'renamed': [], }, { 'id': u'1e146e67985dcd71c74de79613719bef7bddca4a', 'url': u'http://localhost/p/test/src-git/ci/1e146e67985dcd71c74de79613719bef7bddca4a/', 'timestamp': datetime.datetime(2010, 10, 7, 18, 44, 11), 'message': u'Change README', 'author': {'name': u'Rick Copeland', 'email': u'*****@*****.**', 'username': '******'}, 'committer': {'name': u'Rick Copeland', 'email': u'*****@*****.**', 'username': '******'}, 'added': [], 'removed': [], 'modified': [u'README'], 'copied': [], 'renamed': [], }], 'repository': { 'name': u'Git', 'full_name': u'/p/test/src-git/', 'url': u'http://localhost/p/test/src-git/', }, } assert_equals(payload, expected_payload)
def test_send_limit_reached(self, send_webhook, log): sender = RepoPushWebhookSender() sender.get_payload = Mock() self.wh.enforce_limit = Mock(return_value=False) with h.push_config(c, app=self.git): sender.send(dict(arg1=1, arg2=2)) assert_equal(send_webhook.post.call_count, 0) log.warn.assert_called_once_with( 'Webhook fires too often: %s. Skipping', self.wh)
def test_send_with_list(self, send_webhook): sender = RepoPushWebhookSender() sender.get_payload = Mock(side_effect=[1, 2]) self.wh.enforce_limit = Mock(return_value=True) with h.push_config(c, app=self.git): sender.send([dict(arg1=1, arg2=2), dict(arg1=3, arg2=4)]) assert_equal(send_webhook.post.call_count, 2) assert_equal(send_webhook.post.call_args_list, [call(self.wh._id, 1), call(self.wh._id, 2)]) assert_equal(self.wh.enforce_limit.call_count, 1)
def test_webhook_payload(self): sender = RepoPushWebhookSender() cids = list(self.repo.all_commit_ids())[:2] payload = sender.get_payload(commit_ids=cids) expected_payload = { 'size': 2, 'after': 'r6', 'before': 'r4', 'commits': [{ 'id': u'r6', 'url': u'http://localhost/p/test/src/6/', 'timestamp': datetime(2013, 11, 8, 13, 38, 11, 152000), 'message': u'', 'author': {'name': u'coldmind', 'email': u'', 'username': u''}, 'committer': {'name': u'coldmind', 'email': u'', 'username': u''}, 'added': [u'/ЗРЯЧИЙ_ТА_ПОБАЧИТЬ'], 'removed': [], 'modified': [], 'copied': [], 'renamed': [], }, { 'id': u'r5', 'url': u'http://localhost/p/test/src/5/', 'timestamp': datetime(2010, 11, 18, 20, 14, 21, 515000), 'message': u'Copied a => b', 'author': {'name': u'rick446', 'email': u'', 'username': u''}, 'committer': {'name': u'rick446', 'email': u'', 'username': u''}, 'added': [], 'removed': [], 'modified': [], 'copied': [ {'new': u'/b', 'old': u'/a', 'ratio': 1}, ], 'renamed': [], }], 'repository': { 'name': u'SVN', 'full_name': u'/p/test/src/', 'url': u'http://localhost/p/test/src/', }, } assert_equals(payload, expected_payload)
def test_get_payload(self): sender = RepoPushWebhookSender() _ci = lambda x: MagicMock(webhook_info={'id': str(x)}, parent_ids=['0']) with patch.object(self.git.repo, 'commit', new=_ci): with h.push_config(c, app=self.git): result = sender.get_payload(commit_ids=['1', '2', '3'], ref='ref') expected_result = { 'size': 3, 'commits': [{'id': '1'}, {'id': '2'}, {'id': '3'}], 'ref': u'ref', 'after': u'1', 'before': u'0', 'repository': { 'full_name': u'/adobe/adobe-1/src/', 'name': u'Git', 'url': u'http://localhost/adobe/adobe-1/src/', }, } assert_equal(result, expected_result)
def test_enforce_limit(self): def add_webhooks(suffix, n): for i in range(n): webhook = M.Webhook( type='repo-push', app_config_id=self.git.config._id, hook_url='http://httpbin.org/{}/{}'.format(suffix, i), secret='secret') session(webhook).flush(webhook) sender = RepoPushWebhookSender() # default assert_equal(sender.enforce_limit(self.git), True) add_webhooks('one', 3) assert_equal(sender.enforce_limit(self.git), False) # config limit = json.dumps({'git': 5}) with h.push_config(config, **{'webhook.repo_push.max_hooks': limit}): assert_equal(sender.enforce_limit(self.git), True) add_webhooks('two', 3) assert_equal(sender.enforce_limit(self.git), False)
def test_enforce_limit(self): def add_webhooks(suffix, n): for i in range(n): webhook = M.Webhook(type='repo-push', app_config_id=self.git.config._id, hook_url='http://httpbin.org/{}/{}'.format( suffix, i), secret='secret') session(webhook).flush(webhook) sender = RepoPushWebhookSender() # default assert_equal(sender.enforce_limit(self.git), True) add_webhooks('one', 3) assert_equal(sender.enforce_limit(self.git), False) # config limit = json.dumps({'git': 5}) with h.push_config(config, **{'webhook.repo_push.max_hooks': limit}): assert_equal(sender.enforce_limit(self.git), True) add_webhooks('two', 3) assert_equal(sender.enforce_limit(self.git), False)
def refresh_repo(repo, all_commits=False, notify=True, new_clone=False): all_commit_ids = commit_ids = list(repo.all_commit_ids()) if not commit_ids: # the repo is empty, no need to continue return new_commit_ids = unknown_commit_ids(commit_ids) stats_log = h.log_action(log, 'commit') for ci in new_commit_ids: stats_log.info('', meta=dict(module='scm-%s' % repo.repo_id, read='0')) if not all_commits: # Skip commits that are already in the DB commit_ids = new_commit_ids log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path) # Refresh commits seen = set() for i, oid in enumerate(commit_ids): repo.refresh_commit_info(oid, seen, not all_commits) if (i + 1) % 100 == 0: log.info('Refresh commit info %d: %s', (i + 1), oid) refresh_commit_repos(all_commit_ids, repo) # Refresh child references for i, oid in enumerate(commit_ids): ci = CommitDoc.m.find(dict(_id=oid), validate=False).next() refresh_children(ci) if (i + 1) % 100 == 0: log.info('Refresh child info %d for parents of %s', (i + 1), ci._id) if repo._refresh_precompute: # Refresh commit runs commit_run_ids = commit_ids # Check if the CommitRuns for the repo are in a good state by checking for # a CommitRunDoc that contains the last known commit. If there isn't one, # the CommitRuns for this repo are in a bad state - rebuild them # entirely. if commit_run_ids != all_commit_ids: last_commit = last_known_commit_id(all_commit_ids, new_commit_ids) log.info('Last known commit id: %s', last_commit) if not CommitRunDoc.m.find(dict(commit_ids=last_commit)).count(): log.info('CommitRun incomplete, rebuilding with all commits') commit_run_ids = all_commit_ids log.info('Starting CommitRunBuilder for %s', repo.full_fs_path) rb = CommitRunBuilder(commit_run_ids) rb.run() rb.cleanup() log.info('Finished CommitRunBuilder for %s', repo.full_fs_path) # Clear any existing caches for branches/tags if repo.cached_branches: repo.cached_branches = [] session(repo).flush() if repo.cached_tags: repo.cached_tags = [] session(repo).flush() # The first view can be expensive to cache, # so we want to do it here instead of on the first view. repo.get_branches() repo.get_tags() if not all_commits and not new_clone: for commit in commit_ids: new = repo.commit(commit) user = User.by_email_address(new.committed.email) if user is None: user = User.by_username(new.committed.name) if user is not None: g.statsUpdater.newCommit(new, repo.app_config.project, user) actor = user or TransientActor( activity_name=new.committed.name or new.committed.email) g.director.create_activity(actor, 'committed', new, related_nodes=[repo.app_config.project], tags=['commit', repo.tool.lower()]) from allura.webhooks import RepoPushWebhookSender by_branches, by_tags = _group_commits(repo, commit_ids) params = [] for b, commits in by_branches.iteritems(): ref = u'refs/heads/{}'.format(b) if b != '__default__' else None params.append(dict(commit_ids=commits, ref=ref)) for t, commits in by_tags.iteritems(): ref = u'refs/tags/{}'.format(t) params.append(dict(commit_ids=commits, ref=ref)) if params: RepoPushWebhookSender().send(params) log.info('Refresh complete for %s', repo.full_fs_path) g.post_event('repo_refreshed', len(commit_ids), all_commits, new_clone) # Send notifications if notify: send_notifications(repo, reversed(commit_ids))
def refresh_repo(repo, all_commits=False, notify=True, new_clone=False, commits_are_new=None): if commits_are_new is None: commits_are_new = not all_commits and not new_clone all_commit_ids = commit_ids = list(repo.all_commit_ids()) if not commit_ids: # the repo is empty, no need to continue return new_commit_ids = unknown_commit_ids(commit_ids) if not all_commits: # Skip commits that are already in the DB commit_ids = new_commit_ids log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path) # Refresh commits seen = set() for i, oid in enumerate(commit_ids): repo.refresh_commit_info(oid, seen, not all_commits) if (i + 1) % 100 == 0: log.info('Refresh commit info %d: %s', (i + 1), oid) refresh_commit_repos(all_commit_ids, repo) # Refresh child references for i, oid in enumerate(commit_ids): ci = next(CommitDoc.m.find(dict(_id=oid), validate=False)) refresh_children(ci) if (i + 1) % 100 == 0: log.info('Refresh child info %d for parents of %s', (i + 1), ci._id) # Clear any existing caches for branches/tags if repo.cached_branches: repo.cached_branches = [] session(repo).flush() if repo.cached_tags: repo.cached_tags = [] session(repo).flush() # The first view can be expensive to cache, # so we want to do it here instead of on the first view. repo.get_branches() repo.get_tags() if commits_are_new: for commit in commit_ids: new = repo.commit(commit) user = User.by_email_address(new.committed.email) if user is None: user = User.by_username(new.committed.name) if user is not None: g.statsUpdater.newCommit(new, repo.app_config.project, user) actor = user or TransientActor( activity_name=new.committed.name or new.committed.email) g.director.create_activity(actor, 'committed', new, related_nodes=[repo.app_config.project], tags=['commit', repo.tool.lower()]) from allura.webhooks import RepoPushWebhookSender by_branches, by_tags = _group_commits(repo, commit_ids) params = [] for b, commits in six.iteritems(by_branches): ref = 'refs/heads/{}'.format(b) if b != '__default__' else None params.append(dict(commit_ids=commits, ref=ref)) for t, commits in six.iteritems(by_tags): ref = 'refs/tags/{}'.format(t) params.append(dict(commit_ids=commits, ref=ref)) if params: RepoPushWebhookSender().send(params) log.info('Refresh complete for %s', repo.full_fs_path) g.post_event('repo_refreshed', len(commit_ids), all_commits, new_clone) # Send notifications if notify: send_notifications(repo, reversed(commit_ids))
def test_convert_id(self): sender = RepoPushWebhookSender() assert_equal(sender._convert_id(''), '') assert_equal(sender._convert_id('a433fa9'), 'a433fa9') assert_equal(sender._convert_id('a433fa9:13'), 'r13')
def test_after(self): sender = RepoPushWebhookSender() assert_equal(sender._after([]), '') assert_equal(sender._after(['3', '2', '1']), '3')
def test_before(self): sender = RepoPushWebhookSender() with patch.object(self.git.repo, 'commit', autospec=True) as _ci: assert_equal(sender._before(self.git.repo, ['3', '2', '1']), '') _ci.return_value.parent_ids = ['0'] assert_equal(sender._before(self.git.repo, ['3', '2', '1']), '0')