def test_ingest_hg_push_good_repo(hg_push, test_repository,
mock_hg_push_commits):
"""Test graceful handling of an unknown HG repo"""
hg_push["payload"]["repo_url"] = "https://hg.mozilla.org/mozilla-central"
assert Push.objects.count() == 0
PushLoader().process(hg_push, "exchange/hgpushes/v1")
assert Push.objects.count() == 1
def test_ingest_github_push_bad_repo(github_push):
"""Test graceful handling of an unknown GH repo"""
github_push[0]["payload"]["details"][
"event.head.repo.url"] = "https://bad.repo.com"
PushLoader().process(github_push[0]["payload"], github_push[0]["exchange"],
"https://firefox-ci-tc.services.mozilla.com")
assert Push.objects.count() == 0
def ingest_push(project, revision, fetch_push_id=None):
_repo = repo_meta(project)
if _repo['url'].startswith('https://github.com'):
pulse = github_push_to_pulse(_repo, revision)
PushLoader().process(pulse["payload"], pulse["exchange"],
_repo["tc_root_url"])
else:
_ingest_hg_push(project, revision)
def store_pulse_pushes(body, exchange, routing_key):
"""
Fetches the pushes pending from pulse exchanges and loads them.
"""
newrelic.agent.add_custom_parameter("exchange", exchange)
newrelic.agent.add_custom_parameter("routing_key", routing_key)
PushLoader().process(body, exchange)
def handle(self, *args, **options):
typeOfIngestion = options["ingestion_type"][0]
root_url = options["root_url"]
if typeOfIngestion == "task":
assert options["taskId"]
loop.run_until_complete(handleTaskId(options["taskId"], root_url))
elif typeOfIngestion == "pr":
assert options["prUrl"]
pr_url = options["prUrl"]
splitUrl = pr_url.split("/")
org = splitUrl[3]
repo = splitUrl[4]
pulse = {
"exchange": "exchange/taskcluster-github/v1/pull-request",
"routingKey": "primary.{}.{}.synchronize".format(org, repo),
"payload": {
"repository": repo,
"organization": org,
"action": "synchronize",
"details": {
"event.pullNumber": splitUrl[6],
"event.base.repo.url": "https://github.com/{}/{}.git".format(org, repo),
"event.head.repo.url": "https://github.com/{}/{}.git".format(org, repo),
},
}
}
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
elif typeOfIngestion == "git-push":
raise Exception("This is not yet implemented")
elif typeOfIngestion == "push":
if not options["enable_eager_celery"]:
logger.info(
"If you want all logs to be parsed use --enable-eager-celery"
)
else:
# Make sure all tasks are run synchronously / immediately
settings.CELERY_TASK_ALWAYS_EAGER = True
# get reference to repo and ingest this particular revision for this project
project = options["project"]
commit = options["commit"]
repo = Repository.objects.get(name=project, active_status="active")
pushlog_url = "%s/json-pushes/?full=1&version=2" % repo.url
process = HgPushlogProcess()
process.run(pushlog_url, project, changeset=commit, last_push_id=None)
if options["ingest_all_tasks"]:
gecko_decision_task = get_decision_task_id(project, commit, repo.tc_root_url)
logger.info("## START ##")
loop.run_until_complete(processTasks(gecko_decision_task, repo.tc_root_url))
logger.info("## END ##")
else:
logger.info(
"You can ingest all tasks for a push with -a/--ingest-all-tasks."
)
def store_pulse_pushes(
body, exchange, routing_key, root_url='https://firefox-ci-tc.services.mozilla.com'
):
"""
Fetches the pushes pending from pulse exchanges and loads them.
"""
newrelic.agent.add_custom_parameter("exchange", exchange)
newrelic.agent.add_custom_parameter("routing_key", routing_key)
PushLoader().process(body, exchange, root_url)
def test_ingest_github_push_merge_commit(github_push, test_repository,
mock_github_push_compare):
"""Test a a merge push which will require hitting the network for the right info"""
test_repository.url = github_push[1]["payload"]["details"][
"event.head.repo.url"].replace(".git", "")
test_repository.branch = github_push[1]["payload"]["details"][
"event.base.repo.branch"]
test_repository.save()
PushLoader().process(github_push[1]["payload"], github_push[1]["exchange"],
"https://firefox-ci-tc.services.mozilla.com")
assert Push.objects.count() == 1
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes,
github_push,
test_repository,
mock_github_push_compare):
"""Test a repository accepting pushes for multiple branches"""
test_repository.url = "https://github.com/mozilla/test_treeherder"
test_repository.branch = "master,foo,bar"
test_repository.save()
github_push["details"]["event.base.repo.branch"] = branch
assert Push.objects.count() == 0
PushLoader().process(github_push, "exchange/taskcluster-github/v1/push")
assert Push.objects.count() == expected_pushes
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes,
github_push,
test_repository,
mock_github_push_compare):
"""Test a repository accepting pushes for multiple branches"""
test_repository.url = github_push[0]["payload"]["details"][
"event.head.repo.url"].replace(".git", "")
test_repository.branch = "master,foo,bar"
test_repository.save()
github_push[0]["payload"]["details"]["event.base.repo.branch"] = branch
assert Push.objects.count() == 0
PushLoader().process(github_push[0]["payload"], github_push[0]["exchange"],
"https://firefox-ci-tc.services.mozilla.com")
assert Push.objects.count() == expected_pushes
def ingest_pr(pr_url, root_url):
_, _, _, org, repo, _, pull_number, _ = pr_url.split("/", 7)
pulse = {
"exchange": "exchange/taskcluster-github/v1/pull-request",
"routingKey": "primary.{}.{}.synchronize".format(org, repo),
"payload": {
"repository": repo,
"organization": org,
"action": "synchronize",
"details": {
"event.pullNumber":
pull_number,
"event.base.repo.url":
"https://github.com/{}/{}.git".format(org, repo),
"event.head.repo.url":
"https://github.com/{}/{}.git".format(org, repo),
},
},
}
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
def ingest_git_push(project, commit):
_repo = repo_meta(project)
pulse = github_push_to_pulse(_repo, commit)
PushLoader().process(pulse["payload"], pulse["exchange"],
_repo["tc_root_url"])
def test_get_transformer_class(exchange, transformer_class):
rsl = PushLoader()
assert rsl.get_transformer_class(exchange) == transformer_class
def test_ingest_github_push_bad_repo(github_push):
"""Test graceful handling of an unknown GH repo"""
github_push["details"]["event.head.repo.url"] = "https://bad.repo.com"
PushLoader().process(github_push, "exchange/taskcluster-github/v1/push")
assert Push.objects.count() == 0
def test_ingest_hg_push_bad_repo(hg_push):
"""Test graceful handling of an unknown HG repo"""
hg_push["payload"]["repo_url"] = "https://bad.repo.com"
PushLoader().process(hg_push, "exchange/hgpushes/v1")
assert Push.objects.count() == 0
def test_unsupported_exchange():
with pytest.raises(PulsePushError):
rsl = PushLoader()
rsl.get_transformer_class("meh")
def test_get_transformer_class(exchange, transformer_class):
rsl = PushLoader()
assert rsl.get_transformer_class(exchange) == transformer_class
def handle(self, *args, **options):
typeOfIngestion = options["ingestion_type"][0]
root_url = options["root_url"]
if typeOfIngestion == "task":
assert options["taskId"]
loop.run_until_complete(handleTaskId(options["taskId"], root_url))
elif typeOfIngestion == "pr":
assert options["prUrl"]
pr_url = options["prUrl"]
splitUrl = pr_url.split("/")
org = splitUrl[3]
repo = splitUrl[4]
pulse = {
"exchange": "exchange/taskcluster-github/v1/pull-request",
"routingKey": "primary.{}.{}.synchronize".format(org, repo),
"payload": {
"repository": repo,
"organization": org,
"action": "synchronize",
"details": {
"event.pullNumber":
splitUrl[6],
"event.base.repo.url":
"https://github.com/{}/{}.git".format(org, repo),
"event.head.repo.url":
"https://github.com/{}/{}.git".format(org, repo),
},
}
}
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
elif typeOfIngestion == "git-push":
raise Exception("This is not yet implemented")
elif typeOfIngestion == "push":
project = options["project"]
commit = options["commit"]
# get reference to repo
repo = Repository.objects.get(name=project, active_status="active")
fetch_push_id = None
# make sure all tasks are run synchronously / immediately
settings.CELERY_TASK_ALWAYS_EAGER = True
# get hg pushlog
pushlog_url = "%s/json-pushes/?full=1&version=2" % repo.url
# ingest this particular revision for this project
process = HgPushlogProcess()
# Use the actual push SHA, in case the changeset specified was a tag
# or branch name (eg tip). HgPushlogProcess returns the full SHA.
process.run(pushlog_url,
project,
changeset=commit,
last_push_id=fetch_push_id)
if options["ingest_all_tasks"]:
# XXX: Need logic to get from project/revision to taskGroupId
logger.info("## START ##")
# loop.run_until_complete(processTasks("ZYnMSfwCS5Cc_Wi_e-ZlSA", repo.tc_root_url))
logger.info("## END ##")
raise Exception(
"This is not yet implemented. You can still use it by changing the code to "
"grab the task group ID for your push.")
else:
logger.info(
"When implemented you will be able to use --ingest-all-tasks to ingest "
"all tasks associated to this push.")
def handle(self, *args, **options):
typeOfIngestion = options["ingestion_type"][0]
root_url = options["root_url"]
if typeOfIngestion == "task":
assert options["taskId"]
loop.run_until_complete(handleTaskId(options["taskId"], root_url))
elif typeOfIngestion == "pr":
assert options["prUrl"]
pr_url = options["prUrl"]
splitUrl = pr_url.split("/")
org = splitUrl[3]
repo = splitUrl[4]
pulse = {
"exchange": "exchange/taskcluster-github/v1/pull-request",
"routingKey": "primary.{}.{}.synchronize".format(org, repo),
"payload": {
"repository": repo,
"organization": org,
"action": "synchronize",
"details": {
"event.pullNumber":
splitUrl[6],
"event.base.repo.url":
"https://github.com/{}/{}.git".format(org, repo),
"event.head.repo.url":
"https://github.com/{}/{}.git".format(org, repo),
},
}
}
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
elif typeOfIngestion.find("git") > -1:
if not os.environ.get("GITHUB_TOKEN"):
logger.warning(
"If you don't set up GITHUB_TOKEN you might hit Github's rate limiting. See docs for info."
)
if typeOfIngestion == "git-push":
ingest_git_push(options["project"], options["commit"])
elif typeOfIngestion == "git-pushes":
ingest_git_pushes(options["project"], options["dryRun"])
elif typeOfIngestion == "push":
if not options["enable_eager_celery"]:
logger.info(
"If you want all logs to be parsed use --enable-eager-celery"
)
else:
# Make sure all tasks are run synchronously / immediately
settings.CELERY_TASK_ALWAYS_EAGER = True
# get reference to repo and ingest this particular revision for this project
project = options["project"]
commit = options["commit"]
if not options['last_n_pushes'] and not commit:
raise CommandError(
'must specify --last_n_pushes or a positional commit argument'
)
elif options['last_n_pushes'] and options['ingest_all_tasks']:
raise CommandError(
'Can\'t specify last_n_pushes and ingest_all_tasks at same time'
)
elif options['last_n_pushes'] and options['commit']:
raise CommandError(
'Can\'t specify last_n_pushes and commit/revision at the same time'
)
# get reference to repo
repo = Repository.objects.get(name=project, active_status="active")
fetch_push_id = None
if options['last_n_pushes']:
last_push_id = last_push_id_from_server(repo)
fetch_push_id = max(1, last_push_id - options['last_n_pushes'])
logger.info(
'last server push id: %d; fetching push %d and newer',
last_push_id, fetch_push_id)
elif options["ingest_all_tasks"]:
gecko_decision_task = get_decision_task_id(
project, commit, repo.tc_root_url)
logger.info("## START ##")
loop.run_until_complete(
processTasks(gecko_decision_task, repo.tc_root_url))
logger.info("## END ##")
else:
logger.info(
"You can ingest all tasks for a push with -a/--ingest-all-tasks."
)
# get hg pushlog
pushlog_url = "%s/json-pushes/?full=1&version=2" % repo.url
# ingest this particular revision for this project
process = HgPushlogProcess()
# Use the actual push SHA, in case the changeset specified was a tag
# or branch name (eg tip). HgPushlogProcess returns the full SHA.
process.run(pushlog_url,
project,
changeset=commit,
last_push_id=fetch_push_id)
def test_unsupported_exchange():
with pytest.raises(PulsePushError):
rsl = PushLoader()
rsl.get_transformer_class("meh")