def ingest_git_pushes(project, dry_run=False):
"""
This method takes all commits for a repo from Github and determines which ones are considered
part of a push or a merge. Treeherder groups commits by push.
Once we determine which commits are considered the tip revision for a push/merge we then ingest it.
Once we complete the ingestion we compare Treeherder's push API and compare if the pushes are sorted
the same way as in Github.
"""
if not GITHUB_TOKEN:
raise Exception(
"Set GITHUB_TOKEN env variable to avoid rate limiting - Visit https://github.com/settings/tokens."
)
logger.info("--> Converting Github commits to pushes")
_repo = repo_meta(project)
owner, repo = _repo["owner"], _repo["repo"]
github_commits = github.commits_info(owner, repo)
not_push_revision = []
push_revision = []
push_to_date = {}
for _commit in github_commits:
info = github.commit_info(owner, repo, _commit["sha"])
# Revisions that are marked as non-push should be ignored
if _commit["sha"] in not_push_revision:
logger.debug("Not a revision of a push: {}".format(_commit["sha"]))
continue
# Establish which revisions to ignore
for index, parent in enumerate(info["parents"]):
if index != 0:
not_push_revision.append(parent["sha"])
# The 1st parent is the push from `master` from which we forked
oldest_parent_revision = info["parents"][0]["sha"]
push_to_date[oldest_parent_revision] = info["commit"]["committer"][
"date"]
logger.info("Push: {} - Date: {}".format(
oldest_parent_revision, push_to_date[oldest_parent_revision]))
push_revision.append(_commit["sha"])
if not dry_run:
logger.info("--> Ingest Github pushes")
for revision in push_revision:
ingest_git_push(project, revision)
# Test that the *order* of the pushes is correct
logger.info(
"--> Validating that the ingested pushes are in the right order")
client = TreeherderClient(server_url="http://localhost:8000")
th_pushes = client.get_pushes(project, count=len(push_revision))
assert len(push_revision) == len(th_pushes)
for index, revision in enumerate(push_revision):
if revision != th_pushes[index]["revision"]:
logger.warning("{} does not match {}".format(
revision, th_pushes[index]["revision"]))
def test_get_pushes(self):
tdc = TreeherderClient()
url = tdc._get_endpoint_url(tdc.PUSH_ENDPOINT, project='mozilla-inbound')
content = {
"meta": {"count": 3, "repository": "mozilla-inbound", "offset": 0},
"results": self.PUSHES,
}
responses.add(responses.GET, url, json=content, match_querystring=True, status=200)
pushes = tdc.get_pushes("mozilla-inbound")
self.assertEqual(len(pushes), 3)
self.assertEqual(pushes, self.PUSHES)
def test_get_pushes(self):
tdc = TreeherderClient()
url = tdc._get_endpoint_url(tdc.PUSH_ENDPOINT, project='mozilla-inbound')
content = {
"meta": {"count": 3, "repository": "mozilla-inbound",
"offset": 0},
"results": self.PUSHES
}
responses.add(responses.GET, url, json=content, match_querystring=True, status=200)
pushes = tdc.get_pushes("mozilla-inbound")
self.assertEqual(len(pushes), 3)
self.assertEqual(pushes, self.PUSHES)
