def test_get_payload(self):
sender = RepoPushWebhookSender()
_ci = lambda x: MagicMock(webhook_info={'id': str(x)},
parent_ids=['0'])
with patch.object(self.git.repo, 'commit', new=_ci):
with h.push_config(c, app=self.git):
result = sender.get_payload(commit_ids=['1', '2', '3'],
ref='ref')
expected_result = {
'size': 3,
'commits': [{
'id': '1'
}, {
'id': '2'
}, {
'id': '3'
}],
'ref': u'ref',
'after': u'1',
'before': u'0',
'repository': {
'full_name': u'/adobe/adobe-1/src/',
'name': u'Git',
'url': u'http://localhost/adobe/adobe-1/src/',
},
}
assert_equal(result, expected_result)
def test_send(self, send_webhook):
sender = RepoPushWebhookSender()
sender.get_payload = Mock()
with h.push_config(c, app=self.git):
sender.send(dict(arg1=1, arg2=2))
send_webhook.post.assert_called_once_with(
self.wh._id, sender.get_payload.return_value)
def test_send_no_configured_webhooks(self, send_webhook):
self.wh.delete()
session(self.wh).flush(self.wh)
sender = RepoPushWebhookSender()
with h.push_config(c, app=self.git):
sender.send(dict(arg1=1, arg2=2))
assert_equal(send_webhook.post.call_count, 0)
def test_send_no_configured_webhooks(self, send_webhook):
self.wh.delete()
session(self.wh).flush(self.wh)
sender = RepoPushWebhookSender()
with h.push_config(c, app=self.git):
sender.send(dict(arg1=1, arg2=2))
assert_equal(send_webhook.post.call_count, 0)
def test_send(self, send_webhook):
sender = RepoPushWebhookSender()
sender.get_payload = Mock()
with h.push_config(c, app=self.git):
sender.send(dict(arg1=1, arg2=2))
send_webhook.post.assert_called_once_with(
self.wh._id,
sender.get_payload.return_value)
def test_webhook_payload(self):
user = M.User.upsert('cory')
email = user.claim_address('*****@*****.**')
email.confirmed = True
session(email).flush(email)
user = M.User.upsert('rick')
email = user.claim_address('*****@*****.**')
email.confirmed = True
session(email).flush(email)
sender = RepoPushWebhookSender()
cids = list(self.repo.all_commit_ids())[:2]
payload = sender.get_payload(commit_ids=cids, ref='refs/heads/zz')
expected_payload = {
'size': 2,
'ref': 'refs/heads/zz',
'after': '5c47243c8e424136fd5cdd18cd94d34c66d1955c',
'before': 'df30427c488aeab84b2352bdf88a3b19223f9d7a',
'commits': [{
'id': '5c47243c8e424136fd5cdd18cd94d34c66d1955c',
'url': 'http://localhost/p/test/src-git/ci/5c47243c8e424136fd5cdd18cd94d34c66d1955c/',
'timestamp': datetime.datetime(2013, 3, 28, 18, 54, 16),
'message': 'Not repo root',
'author': {'name': 'Cory Johns',
'email': '*****@*****.**',
'username': '******'},
'committer': {'name': 'Cory Johns',
'email': '*****@*****.**',
'username': '******'},
'added': ['bad'],
'removed': [],
'modified': [],
'copied': [],
'renamed': [],
}, {
'id': '1e146e67985dcd71c74de79613719bef7bddca4a',
'url': 'http://localhost/p/test/src-git/ci/1e146e67985dcd71c74de79613719bef7bddca4a/',
'timestamp': datetime.datetime(2010, 10, 7, 18, 44, 11),
'message': 'Change README',
'author': {'name': 'Rick Copeland',
'email': '*****@*****.**',
'username': '******'},
'committer': {'name': 'Rick Copeland',
'email': '*****@*****.**',
'username': '******'},
'added': [],
'removed': [],
'modified': ['README'],
'copied': [],
'renamed': [],
}],
'repository': {
'name': 'Git',
'full_name': '/p/test/src-git/',
'url': 'http://localhost/p/test/src-git/',
},
}
assert_equals(payload, expected_payload)
def test_webhook_payload(self):
user = M.User.upsert('cory')
email = user.claim_address('*****@*****.**')
email.confirmed = True
session(email).flush(email)
user = M.User.upsert('rick')
email = user.claim_address('*****@*****.**')
email.confirmed = True
session(email).flush(email)
sender = RepoPushWebhookSender()
cids = list(self.repo.all_commit_ids())[:2]
payload = sender.get_payload(commit_ids=cids, ref='refs/heads/zz')
expected_payload = {
'size': 2,
'ref': u'refs/heads/zz',
'after': u'5c47243c8e424136fd5cdd18cd94d34c66d1955c',
'before': u'df30427c488aeab84b2352bdf88a3b19223f9d7a',
'commits': [{
'id': u'5c47243c8e424136fd5cdd18cd94d34c66d1955c',
'url': u'http://localhost/p/test/src-git/ci/5c47243c8e424136fd5cdd18cd94d34c66d1955c/',
'timestamp': datetime.datetime(2013, 3, 28, 18, 54, 16),
'message': u'Not repo root',
'author': {'name': u'Cory Johns',
'email': u'*****@*****.**',
'username': '******'},
'committer': {'name': u'Cory Johns',
'email': u'*****@*****.**',
'username': '******'},
'added': [u'bad'],
'removed': [],
'modified': [],
'copied': [],
'renamed': [],
}, {
'id': u'1e146e67985dcd71c74de79613719bef7bddca4a',
'url': u'http://localhost/p/test/src-git/ci/1e146e67985dcd71c74de79613719bef7bddca4a/',
'timestamp': datetime.datetime(2010, 10, 7, 18, 44, 11),
'message': u'Change README',
'author': {'name': u'Rick Copeland',
'email': u'*****@*****.**',
'username': '******'},
'committer': {'name': u'Rick Copeland',
'email': u'*****@*****.**',
'username': '******'},
'added': [],
'removed': [],
'modified': [u'README'],
'copied': [],
'renamed': [],
}],
'repository': {
'name': u'Git',
'full_name': u'/p/test/src-git/',
'url': u'http://localhost/p/test/src-git/',
},
}
assert_equals(payload, expected_payload)
def test_send_limit_reached(self, send_webhook, log):
sender = RepoPushWebhookSender()
sender.get_payload = Mock()
self.wh.enforce_limit = Mock(return_value=False)
with h.push_config(c, app=self.git):
sender.send(dict(arg1=1, arg2=2))
assert_equal(send_webhook.post.call_count, 0)
log.warn.assert_called_once_with(
'Webhook fires too often: %s. Skipping', self.wh)
def test_send_limit_reached(self, send_webhook, log):
sender = RepoPushWebhookSender()
sender.get_payload = Mock()
self.wh.enforce_limit = Mock(return_value=False)
with h.push_config(c, app=self.git):
sender.send(dict(arg1=1, arg2=2))
assert_equal(send_webhook.post.call_count, 0)
log.warn.assert_called_once_with(
'Webhook fires too often: %s. Skipping', self.wh)
def test_send_with_list(self, send_webhook):
sender = RepoPushWebhookSender()
sender.get_payload = Mock(side_effect=[1, 2])
self.wh.enforce_limit = Mock(return_value=True)
with h.push_config(c, app=self.git):
sender.send([dict(arg1=1, arg2=2), dict(arg1=3, arg2=4)])
assert_equal(send_webhook.post.call_count, 2)
assert_equal(send_webhook.post.call_args_list,
[call(self.wh._id, 1), call(self.wh._id, 2)])
assert_equal(self.wh.enforce_limit.call_count, 1)
def test_webhook_payload(self):
sender = RepoPushWebhookSender()
cids = list(self.repo.all_commit_ids())[:2]
payload = sender.get_payload(commit_ids=cids)
expected_payload = {
'size': 2,
'after': 'r6',
'before': 'r4',
'commits': [{
'id': u'r6',
'url': u'http://localhost/p/test/src/6/',
'timestamp': datetime(2013, 11, 8, 13, 38, 11, 152000),
'message': u'',
'author': {'name': u'coldmind',
'email': u'',
'username': u''},
'committer': {'name': u'coldmind',
'email': u'',
'username': u''},
'added': [u'/ЗРЯЧИЙ_ТА_ПОБАЧИТЬ'],
'removed': [],
'modified': [],
'copied': [],
'renamed': [],
}, {
'id': u'r5',
'url': u'http://localhost/p/test/src/5/',
'timestamp': datetime(2010, 11, 18, 20, 14, 21, 515000),
'message': u'Copied a => b',
'author': {'name': u'rick446',
'email': u'',
'username': u''},
'committer': {'name': u'rick446',
'email': u'',
'username': u''},
'added': [],
'removed': [],
'modified': [],
'copied': [
{'new': u'/b', 'old': u'/a', 'ratio': 1},
],
'renamed': [],
}],
'repository': {
'name': u'SVN',
'full_name': u'/p/test/src/',
'url': u'http://localhost/p/test/src/',
},
}
assert_equals(payload, expected_payload)
def test_webhook_payload(self):
sender = RepoPushWebhookSender()
cids = list(self.repo.all_commit_ids())[:2]
payload = sender.get_payload(commit_ids=cids)
expected_payload = {
'size': 2,
'after': 'r6',
'before': 'r4',
'commits': [{
'id': u'r6',
'url': u'http://localhost/p/test/src/6/',
'timestamp': datetime(2013, 11, 8, 13, 38, 11, 152000),
'message': u'',
'author': {'name': u'coldmind',
'email': u'',
'username': u''},
'committer': {'name': u'coldmind',
'email': u'',
'username': u''},
'added': [u'/ЗРЯЧИЙ_ТА_ПОБАЧИТЬ'],
'removed': [],
'modified': [],
'copied': [],
'renamed': [],
}, {
'id': u'r5',
'url': u'http://localhost/p/test/src/5/',
'timestamp': datetime(2010, 11, 18, 20, 14, 21, 515000),
'message': u'Copied a => b',
'author': {'name': u'rick446',
'email': u'',
'username': u''},
'committer': {'name': u'rick446',
'email': u'',
'username': u''},
'added': [],
'removed': [],
'modified': [],
'copied': [
{'new': u'/b', 'old': u'/a', 'ratio': 1},
],
'renamed': [],
}],
'repository': {
'name': u'SVN',
'full_name': u'/p/test/src/',
'url': u'http://localhost/p/test/src/',
},
}
assert_equals(payload, expected_payload)
def test_get_payload(self):
sender = RepoPushWebhookSender()
_ci = lambda x: MagicMock(webhook_info={'id': str(x)}, parent_ids=['0'])
with patch.object(self.git.repo, 'commit', new=_ci):
with h.push_config(c, app=self.git):
result = sender.get_payload(commit_ids=['1', '2', '3'], ref='ref')
expected_result = {
'size': 3,
'commits': [{'id': '1'}, {'id': '2'}, {'id': '3'}],
'ref': u'ref',
'after': u'1',
'before': u'0',
'repository': {
'full_name': u'/adobe/adobe-1/src/',
'name': u'Git',
'url': u'http://localhost/adobe/adobe-1/src/',
},
}
assert_equal(result, expected_result)
def test_enforce_limit(self):
def add_webhooks(suffix, n):
for i in range(n):
webhook = M.Webhook(
type='repo-push',
app_config_id=self.git.config._id,
hook_url='http://httpbin.org/{}/{}'.format(suffix, i),
secret='secret')
session(webhook).flush(webhook)
sender = RepoPushWebhookSender()
# default
assert_equal(sender.enforce_limit(self.git), True)
add_webhooks('one', 3)
assert_equal(sender.enforce_limit(self.git), False)
# config
limit = json.dumps({'git': 5})
with h.push_config(config, **{'webhook.repo_push.max_hooks': limit}):
assert_equal(sender.enforce_limit(self.git), True)
add_webhooks('two', 3)
assert_equal(sender.enforce_limit(self.git), False)
def test_enforce_limit(self):
def add_webhooks(suffix, n):
for i in range(n):
webhook = M.Webhook(type='repo-push',
app_config_id=self.git.config._id,
hook_url='http://httpbin.org/{}/{}'.format(
suffix, i),
secret='secret')
session(webhook).flush(webhook)
sender = RepoPushWebhookSender()
# default
assert_equal(sender.enforce_limit(self.git), True)
add_webhooks('one', 3)
assert_equal(sender.enforce_limit(self.git), False)
# config
limit = json.dumps({'git': 5})
with h.push_config(config, **{'webhook.repo_push.max_hooks': limit}):
assert_equal(sender.enforce_limit(self.git), True)
add_webhooks('two', 3)
assert_equal(sender.enforce_limit(self.git), False)
def refresh_repo(repo, all_commits=False, notify=True, new_clone=False):
all_commit_ids = commit_ids = list(repo.all_commit_ids())
if not commit_ids:
# the repo is empty, no need to continue
return
new_commit_ids = unknown_commit_ids(commit_ids)
stats_log = h.log_action(log, 'commit')
for ci in new_commit_ids:
stats_log.info('', meta=dict(module='scm-%s' % repo.repo_id, read='0'))
if not all_commits:
# Skip commits that are already in the DB
commit_ids = new_commit_ids
log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path)
# Refresh commits
seen = set()
for i, oid in enumerate(commit_ids):
repo.refresh_commit_info(oid, seen, not all_commits)
if (i + 1) % 100 == 0:
log.info('Refresh commit info %d: %s', (i + 1), oid)
refresh_commit_repos(all_commit_ids, repo)
# Refresh child references
for i, oid in enumerate(commit_ids):
ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
refresh_children(ci)
if (i + 1) % 100 == 0:
log.info('Refresh child info %d for parents of %s', (i + 1),
ci._id)
if repo._refresh_precompute:
# Refresh commit runs
commit_run_ids = commit_ids
# Check if the CommitRuns for the repo are in a good state by checking for
# a CommitRunDoc that contains the last known commit. If there isn't one,
# the CommitRuns for this repo are in a bad state - rebuild them
# entirely.
if commit_run_ids != all_commit_ids:
last_commit = last_known_commit_id(all_commit_ids, new_commit_ids)
log.info('Last known commit id: %s', last_commit)
if not CommitRunDoc.m.find(dict(commit_ids=last_commit)).count():
log.info('CommitRun incomplete, rebuilding with all commits')
commit_run_ids = all_commit_ids
log.info('Starting CommitRunBuilder for %s', repo.full_fs_path)
rb = CommitRunBuilder(commit_run_ids)
rb.run()
rb.cleanup()
log.info('Finished CommitRunBuilder for %s', repo.full_fs_path)
# Clear any existing caches for branches/tags
if repo.cached_branches:
repo.cached_branches = []
session(repo).flush()
if repo.cached_tags:
repo.cached_tags = []
session(repo).flush()
# The first view can be expensive to cache,
# so we want to do it here instead of on the first view.
repo.get_branches()
repo.get_tags()
if not all_commits and not new_clone:
for commit in commit_ids:
new = repo.commit(commit)
user = User.by_email_address(new.committed.email)
if user is None:
user = User.by_username(new.committed.name)
if user is not None:
g.statsUpdater.newCommit(new, repo.app_config.project, user)
actor = user or TransientActor(
activity_name=new.committed.name or new.committed.email)
g.director.create_activity(actor,
'committed',
new,
related_nodes=[repo.app_config.project],
tags=['commit',
repo.tool.lower()])
from allura.webhooks import RepoPushWebhookSender
by_branches, by_tags = _group_commits(repo, commit_ids)
params = []
for b, commits in by_branches.iteritems():
ref = u'refs/heads/{}'.format(b) if b != '__default__' else None
params.append(dict(commit_ids=commits, ref=ref))
for t, commits in by_tags.iteritems():
ref = u'refs/tags/{}'.format(t)
params.append(dict(commit_ids=commits, ref=ref))
if params:
RepoPushWebhookSender().send(params)
log.info('Refresh complete for %s', repo.full_fs_path)
g.post_event('repo_refreshed', len(commit_ids), all_commits, new_clone)
# Send notifications
if notify:
send_notifications(repo, reversed(commit_ids))
def refresh_repo(repo,
all_commits=False,
notify=True,
new_clone=False,
commits_are_new=None):
if commits_are_new is None:
commits_are_new = not all_commits and not new_clone
all_commit_ids = commit_ids = list(repo.all_commit_ids())
if not commit_ids:
# the repo is empty, no need to continue
return
new_commit_ids = unknown_commit_ids(commit_ids)
if not all_commits:
# Skip commits that are already in the DB
commit_ids = new_commit_ids
log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path)
# Refresh commits
seen = set()
for i, oid in enumerate(commit_ids):
repo.refresh_commit_info(oid, seen, not all_commits)
if (i + 1) % 100 == 0:
log.info('Refresh commit info %d: %s', (i + 1), oid)
refresh_commit_repos(all_commit_ids, repo)
# Refresh child references
for i, oid in enumerate(commit_ids):
ci = next(CommitDoc.m.find(dict(_id=oid), validate=False))
refresh_children(ci)
if (i + 1) % 100 == 0:
log.info('Refresh child info %d for parents of %s', (i + 1),
ci._id)
# Clear any existing caches for branches/tags
if repo.cached_branches:
repo.cached_branches = []
session(repo).flush()
if repo.cached_tags:
repo.cached_tags = []
session(repo).flush()
# The first view can be expensive to cache,
# so we want to do it here instead of on the first view.
repo.get_branches()
repo.get_tags()
if commits_are_new:
for commit in commit_ids:
new = repo.commit(commit)
user = User.by_email_address(new.committed.email)
if user is None:
user = User.by_username(new.committed.name)
if user is not None:
g.statsUpdater.newCommit(new, repo.app_config.project, user)
actor = user or TransientActor(
activity_name=new.committed.name or new.committed.email)
g.director.create_activity(actor,
'committed',
new,
related_nodes=[repo.app_config.project],
tags=['commit',
repo.tool.lower()])
from allura.webhooks import RepoPushWebhookSender
by_branches, by_tags = _group_commits(repo, commit_ids)
params = []
for b, commits in six.iteritems(by_branches):
ref = 'refs/heads/{}'.format(b) if b != '__default__' else None
params.append(dict(commit_ids=commits, ref=ref))
for t, commits in six.iteritems(by_tags):
ref = 'refs/tags/{}'.format(t)
params.append(dict(commit_ids=commits, ref=ref))
if params:
RepoPushWebhookSender().send(params)
log.info('Refresh complete for %s', repo.full_fs_path)
g.post_event('repo_refreshed', len(commit_ids), all_commits, new_clone)
# Send notifications
if notify:
send_notifications(repo, reversed(commit_ids))
def test_convert_id(self):
sender = RepoPushWebhookSender()
assert_equal(sender._convert_id(''), '')
assert_equal(sender._convert_id('a433fa9'), 'a433fa9')
assert_equal(sender._convert_id('a433fa9:13'), 'r13')
def test_after(self):
sender = RepoPushWebhookSender()
assert_equal(sender._after([]), '')
assert_equal(sender._after(['3', '2', '1']), '3')
def test_before(self):
sender = RepoPushWebhookSender()
with patch.object(self.git.repo, 'commit', autospec=True) as _ci:
assert_equal(sender._before(self.git.repo, ['3', '2', '1']), '')
_ci.return_value.parent_ids = ['0']
assert_equal(sender._before(self.git.repo, ['3', '2', '1']), '0')
def test_before(self):
sender = RepoPushWebhookSender()
with patch.object(self.git.repo, 'commit', autospec=True) as _ci:
assert_equal(sender._before(self.git.repo, ['3', '2', '1']), '')
_ci.return_value.parent_ids = ['0']
assert_equal(sender._before(self.git.repo, ['3', '2', '1']), '0')
def test_after(self):
sender = RepoPushWebhookSender()
assert_equal(sender._after([]), '')
assert_equal(sender._after(['3', '2', '1']), '3')
def test_convert_id(self):
sender = RepoPushWebhookSender()
assert_equal(sender._convert_id(''), '')
assert_equal(sender._convert_id('a433fa9'), 'a433fa9')
assert_equal(sender._convert_id('a433fa9:13'), 'r13')