def test_ingest_hg_push_good_repo(hg_push, test_repository, mock_hg_push_commits): """Test graceful handling of an unknown HG repo""" hg_push["payload"]["repo_url"] = "https://hg.mozilla.org/mozilla-central" assert Push.objects.count() == 0 PushLoader().process(hg_push, "exchange/hgpushes/v1") assert Push.objects.count() == 1
def test_ingest_github_push_bad_repo(github_push): """Test graceful handling of an unknown GH repo""" github_push[0]["payload"]["details"][ "event.head.repo.url"] = "https://bad.repo.com" PushLoader().process(github_push[0]["payload"], github_push[0]["exchange"], "https://firefox-ci-tc.services.mozilla.com") assert Push.objects.count() == 0
def ingest_push(project, revision, fetch_push_id=None): _repo = repo_meta(project) if _repo['url'].startswith('https://github.com'): pulse = github_push_to_pulse(_repo, revision) PushLoader().process(pulse["payload"], pulse["exchange"], _repo["tc_root_url"]) else: _ingest_hg_push(project, revision)
def store_pulse_pushes(body, exchange, routing_key): """ Fetches the pushes pending from pulse exchanges and loads them. """ newrelic.agent.add_custom_parameter("exchange", exchange) newrelic.agent.add_custom_parameter("routing_key", routing_key) PushLoader().process(body, exchange)
def handle(self, *args, **options): typeOfIngestion = options["ingestion_type"][0] root_url = options["root_url"] if typeOfIngestion == "task": assert options["taskId"] loop.run_until_complete(handleTaskId(options["taskId"], root_url)) elif typeOfIngestion == "pr": assert options["prUrl"] pr_url = options["prUrl"] splitUrl = pr_url.split("/") org = splitUrl[3] repo = splitUrl[4] pulse = { "exchange": "exchange/taskcluster-github/v1/pull-request", "routingKey": "primary.{}.{}.synchronize".format(org, repo), "payload": { "repository": repo, "organization": org, "action": "synchronize", "details": { "event.pullNumber": splitUrl[6], "event.base.repo.url": "https://github.com/{}/{}.git".format(org, repo), "event.head.repo.url": "https://github.com/{}/{}.git".format(org, repo), }, } } PushLoader().process(pulse["payload"], pulse["exchange"], root_url) elif typeOfIngestion == "git-push": raise Exception("This is not yet implemented") elif typeOfIngestion == "push": if not options["enable_eager_celery"]: logger.info( "If you want all logs to be parsed use --enable-eager-celery" ) else: # Make sure all tasks are run synchronously / immediately settings.CELERY_TASK_ALWAYS_EAGER = True # get reference to repo and ingest this particular revision for this project project = options["project"] commit = options["commit"] repo = Repository.objects.get(name=project, active_status="active") pushlog_url = "%s/json-pushes/?full=1&version=2" % repo.url process = HgPushlogProcess() process.run(pushlog_url, project, changeset=commit, last_push_id=None) if options["ingest_all_tasks"]: gecko_decision_task = get_decision_task_id(project, commit, repo.tc_root_url) logger.info("## START ##") loop.run_until_complete(processTasks(gecko_decision_task, repo.tc_root_url)) logger.info("## END ##") else: logger.info( "You can ingest all tasks for a push with -a/--ingest-all-tasks." )
def store_pulse_pushes( body, exchange, routing_key, root_url='https://firefox-ci-tc.services.mozilla.com' ): """ Fetches the pushes pending from pulse exchanges and loads them. """ newrelic.agent.add_custom_parameter("exchange", exchange) newrelic.agent.add_custom_parameter("routing_key", routing_key) PushLoader().process(body, exchange, root_url)
def test_ingest_github_push_merge_commit(github_push, test_repository, mock_github_push_compare): """Test a a merge push which will require hitting the network for the right info""" test_repository.url = github_push[1]["payload"]["details"][ "event.head.repo.url"].replace(".git", "") test_repository.branch = github_push[1]["payload"]["details"][ "event.base.repo.branch"] test_repository.save() PushLoader().process(github_push[1]["payload"], github_push[1]["exchange"], "https://firefox-ci-tc.services.mozilla.com") assert Push.objects.count() == 1
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes, github_push, test_repository, mock_github_push_compare): """Test a repository accepting pushes for multiple branches""" test_repository.url = "https://github.com/mozilla/test_treeherder" test_repository.branch = "master,foo,bar" test_repository.save() github_push["details"]["event.base.repo.branch"] = branch assert Push.objects.count() == 0 PushLoader().process(github_push, "exchange/taskcluster-github/v1/push") assert Push.objects.count() == expected_pushes
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes, github_push, test_repository, mock_github_push_compare): """Test a repository accepting pushes for multiple branches""" test_repository.url = github_push[0]["payload"]["details"][ "event.head.repo.url"].replace(".git", "") test_repository.branch = "master,foo,bar" test_repository.save() github_push[0]["payload"]["details"]["event.base.repo.branch"] = branch assert Push.objects.count() == 0 PushLoader().process(github_push[0]["payload"], github_push[0]["exchange"], "https://firefox-ci-tc.services.mozilla.com") assert Push.objects.count() == expected_pushes
def ingest_pr(pr_url, root_url): _, _, _, org, repo, _, pull_number, _ = pr_url.split("/", 7) pulse = { "exchange": "exchange/taskcluster-github/v1/pull-request", "routingKey": "primary.{}.{}.synchronize".format(org, repo), "payload": { "repository": repo, "organization": org, "action": "synchronize", "details": { "event.pullNumber": pull_number, "event.base.repo.url": "https://github.com/{}/{}.git".format(org, repo), "event.head.repo.url": "https://github.com/{}/{}.git".format(org, repo), }, }, } PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
def ingest_git_push(project, commit): _repo = repo_meta(project) pulse = github_push_to_pulse(_repo, commit) PushLoader().process(pulse["payload"], pulse["exchange"], _repo["tc_root_url"])
def test_get_transformer_class(exchange, transformer_class): rsl = PushLoader() assert rsl.get_transformer_class(exchange) == transformer_class
def test_ingest_github_push_bad_repo(github_push): """Test graceful handling of an unknown GH repo""" github_push["details"]["event.head.repo.url"] = "https://bad.repo.com" PushLoader().process(github_push, "exchange/taskcluster-github/v1/push") assert Push.objects.count() == 0
def test_ingest_hg_push_bad_repo(hg_push): """Test graceful handling of an unknown HG repo""" hg_push["payload"]["repo_url"] = "https://bad.repo.com" PushLoader().process(hg_push, "exchange/hgpushes/v1") assert Push.objects.count() == 0
def test_unsupported_exchange(): with pytest.raises(PulsePushError): rsl = PushLoader() rsl.get_transformer_class("meh")
def handle(self, *args, **options): typeOfIngestion = options["ingestion_type"][0] root_url = options["root_url"] if typeOfIngestion == "task": assert options["taskId"] loop.run_until_complete(handleTaskId(options["taskId"], root_url)) elif typeOfIngestion == "pr": assert options["prUrl"] pr_url = options["prUrl"] splitUrl = pr_url.split("/") org = splitUrl[3] repo = splitUrl[4] pulse = { "exchange": "exchange/taskcluster-github/v1/pull-request", "routingKey": "primary.{}.{}.synchronize".format(org, repo), "payload": { "repository": repo, "organization": org, "action": "synchronize", "details": { "event.pullNumber": splitUrl[6], "event.base.repo.url": "https://github.com/{}/{}.git".format(org, repo), "event.head.repo.url": "https://github.com/{}/{}.git".format(org, repo), }, } } PushLoader().process(pulse["payload"], pulse["exchange"], root_url) elif typeOfIngestion == "git-push": raise Exception("This is not yet implemented") elif typeOfIngestion == "push": project = options["project"] commit = options["commit"] # get reference to repo repo = Repository.objects.get(name=project, active_status="active") fetch_push_id = None # make sure all tasks are run synchronously / immediately settings.CELERY_TASK_ALWAYS_EAGER = True # get hg pushlog pushlog_url = "%s/json-pushes/?full=1&version=2" % repo.url # ingest this particular revision for this project process = HgPushlogProcess() # Use the actual push SHA, in case the changeset specified was a tag # or branch name (eg tip). HgPushlogProcess returns the full SHA. process.run(pushlog_url, project, changeset=commit, last_push_id=fetch_push_id) if options["ingest_all_tasks"]: # XXX: Need logic to get from project/revision to taskGroupId logger.info("## START ##") # loop.run_until_complete(processTasks("ZYnMSfwCS5Cc_Wi_e-ZlSA", repo.tc_root_url)) logger.info("## END ##") raise Exception( "This is not yet implemented. You can still use it by changing the code to " "grab the task group ID for your push.") else: logger.info( "When implemented you will be able to use --ingest-all-tasks to ingest " "all tasks associated to this push.")
def handle(self, *args, **options): typeOfIngestion = options["ingestion_type"][0] root_url = options["root_url"] if typeOfIngestion == "task": assert options["taskId"] loop.run_until_complete(handleTaskId(options["taskId"], root_url)) elif typeOfIngestion == "pr": assert options["prUrl"] pr_url = options["prUrl"] splitUrl = pr_url.split("/") org = splitUrl[3] repo = splitUrl[4] pulse = { "exchange": "exchange/taskcluster-github/v1/pull-request", "routingKey": "primary.{}.{}.synchronize".format(org, repo), "payload": { "repository": repo, "organization": org, "action": "synchronize", "details": { "event.pullNumber": splitUrl[6], "event.base.repo.url": "https://github.com/{}/{}.git".format(org, repo), "event.head.repo.url": "https://github.com/{}/{}.git".format(org, repo), }, } } PushLoader().process(pulse["payload"], pulse["exchange"], root_url) elif typeOfIngestion.find("git") > -1: if not os.environ.get("GITHUB_TOKEN"): logger.warning( "If you don't set up GITHUB_TOKEN you might hit Github's rate limiting. See docs for info." ) if typeOfIngestion == "git-push": ingest_git_push(options["project"], options["commit"]) elif typeOfIngestion == "git-pushes": ingest_git_pushes(options["project"], options["dryRun"]) elif typeOfIngestion == "push": if not options["enable_eager_celery"]: logger.info( "If you want all logs to be parsed use --enable-eager-celery" ) else: # Make sure all tasks are run synchronously / immediately settings.CELERY_TASK_ALWAYS_EAGER = True # get reference to repo and ingest this particular revision for this project project = options["project"] commit = options["commit"] if not options['last_n_pushes'] and not commit: raise CommandError( 'must specify --last_n_pushes or a positional commit argument' ) elif options['last_n_pushes'] and options['ingest_all_tasks']: raise CommandError( 'Can\'t specify last_n_pushes and ingest_all_tasks at same time' ) elif options['last_n_pushes'] and options['commit']: raise CommandError( 'Can\'t specify last_n_pushes and commit/revision at the same time' ) # get reference to repo repo = Repository.objects.get(name=project, active_status="active") fetch_push_id = None if options['last_n_pushes']: last_push_id = last_push_id_from_server(repo) fetch_push_id = max(1, last_push_id - options['last_n_pushes']) logger.info( 'last server push id: %d; fetching push %d and newer', last_push_id, fetch_push_id) elif options["ingest_all_tasks"]: gecko_decision_task = get_decision_task_id( project, commit, repo.tc_root_url) logger.info("## START ##") loop.run_until_complete( processTasks(gecko_decision_task, repo.tc_root_url)) logger.info("## END ##") else: logger.info( "You can ingest all tasks for a push with -a/--ingest-all-tasks." ) # get hg pushlog pushlog_url = "%s/json-pushes/?full=1&version=2" % repo.url # ingest this particular revision for this project process = HgPushlogProcess() # Use the actual push SHA, in case the changeset specified was a tag # or branch name (eg tip). HgPushlogProcess returns the full SHA. process.run(pushlog_url, project, changeset=commit, last_push_id=fetch_push_id)