def test_valid(self, fetch_allthethings_data): """is_downstream should return True for test jobs and False for build jobs.""" fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS self.assertEquals( is_downstream('Platform1 repo opt test mochitest-1'), True) self.assertEquals( is_downstream('Platform1 repo build'), False)
def trigger(builder, revision, files=None, dry_run=False, extra_properties=None): """Helper to trigger a job. Returns a request. """ _add_builder_to_scheduling_manager(revision=revision, buildername=builder) repo_name = query_repo_name_from_buildername(builder) if is_downstream(builder) and not files: raise MozciError( 'We have requested to trigger a test job, however, we have not provided ' 'which files to run against.') return trigger_arbitrary_job(repo_name=repo_name, builder=builder, revision=revision, auth=get_credentials(), files=files, dry_run=dry_run, extra_properties=extra_properties)
def get_upstream_buildernames(repo_name): """Return every upstream buildername in a repo.""" buildernames = filter_buildernames([repo_name], ["hg bundle", "pgo"], list_builders()) upstream_jobs = [] for buildername in buildernames: if not is_downstream(buildername): upstream_jobs.append(buildername) return upstream_jobs
def generate_builders_relations_dictionary(): """Create a dictionary that maps every upstream job to its downstream jobs.""" builders = list_builders() relations = collections.defaultdict(list) for buildername in builders: if is_downstream(buildername): relations[determine_upstream_builder(buildername)].append(buildername) return relations
def _unique_build_request(buildername, revision): """ We want to prevent requesting a build job too many times in the same session. """ global SCHEDULING_MANAGER sch_mgr = SCHEDULING_MANAGER if is_downstream(buildername): return True else: if revision in sch_mgr and buildername in sch_mgr[revision]: LOG.debug("We have already scheduled the build '%s' for " "revision %s during this session. We don't allow " "multiple requests." % (buildername, revision)) return False return True
def separate_downstream(jobs): """Split list of jobs in downstream and upstream.""" upstream_jobs = [] downstream_jobs = [] for job in jobs: buildername = job[0] # If the job is not on allthethings.json, we will find an exception try: downstream_status = is_downstream(buildername) except: continue if downstream_status: downstream_jobs.append(job) else: upstream_jobs.append(job) return sorted(upstream_jobs), sorted(downstream_jobs)
def trigger(builder, revision, files=None, dry_run=False, extra_properties=None): """Helper to trigger a job. Returns a request. """ _add_builder_to_scheduling_manager(revision=revision, buildername=builder) repo_name = query_repo_name_from_buildername(builder) if is_downstream(builder) and not files: raise MozciError('We have requested to trigger a test job, however, we have not provided ' 'which files to run against.') return trigger_arbitrary_job(repo_name=repo_name, builder=builder, revision=revision, auth=get_credentials(), files=files, dry_run=dry_run, extra_properties=extra_properties)
def test_valid(self, fetch_allthethings_data): """is_downstream should return True for test jobs and False for build jobs.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert is_downstream('Ubuntu VM 12.04 x64 mozilla-beta debug test mochitest-1') is True assert is_downstream('Linux x86-64 mozilla-beta build') is False
def _generate_tc_tasks_from_builders(builders, repo_name, revision): """ Return TC tasks based on a list of builders. Input: a list of builders and a revision Output: list of TC tasks base on builders we receive :param builders: List of builder names :type builders: list :param repo_name: push revision :type repo_name: str :param revision: push revision :type revision: str :return: TC tasks :rtype: dict """ tasks = [] build_builders = {} # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for builder in builders: if is_upstream(builder): properties = {'upload_to_task_id': slugId()} # Bug 1274483 - Android multi-locale nightly builds need to upload to two different # tasks, thus, it fails when we tell it to upload to the same task twice. builder_details = get_buildername_metadata(builder) if builder_details['platform_name'].startswith('android') and \ builder_details['nightly'] is True and \ 'l10n' not in builder: properties = {} task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties=properties, ) tasks.append(task) # We want to keep track of how many build builders we have build_builders[builder] = task for builder in builders: if is_downstream(builder): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective, package_url, tests_url = \ determine_trigger_objective(revision, builder) # The build job is already completed, we can trigger the test job if objective == builder: if objective in build_builders: LOG.warning("We're creating a new build even though there's " "already an existing completed build we could have " "used. We hope you wanted to do this.") task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=build_builders[objective]['taskId'], properties={'upload_to_task_id': slugId()}, ) tasks.append(task) else: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, properties={ 'packageUrl': package_url, 'testUrl': tests_url }, ) tasks.append(task) # The build job is running, there is nothing we can do elif objective is None: LOG.warning("We can add %s builder since the build associated " "is running. This is because it is a Buildbot job.") pass # We need to trigger the build job and the test job else: if objective not in build_builders: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={'upload_to_task_id': slugId()}, ) tasks.append(task) taskId = task['taskId'] else: taskId = build_builders[objective]['taskId'] # Add test job task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=taskId, ) tasks.append(task) return tasks
def buildbot_graph_builder(builders, revision, complete=True): """ Return graph of builders based on a list of builders. # XXX: It would be better if had a BuildbotGraph class instead of messing with dictionaries. https://github.com/mozilla/mozilla_ci_tools/issues/353 Input: a list of builders and a revision Output: a set which includes a graph with the builders we received, the necessary upstream jobs and list of builders which can been used to schedule jobs through trigger_job() NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling. Graph of N levels: { 'Builder a1': { 'Builder a2': { ... 'Builder aN': None }, }, 'Builder b1': None } :param builders: List of builder names :type builders: list :param revision: push revision :type revision: str :return: A graph of buildernames (single level of graphs) :rtype: dict :param complete: indicate that build has been completed or not :type: booleans """ graph = {} ready_to_trigger = [] # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for b in builders: if not valid_builder(buildername=b, quiet=True): continue if is_downstream(b): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective = determine_trigger_objective(revision, b)[0] # The build job is already completed, we can trigger the test job if objective == b: # XXX: Fix me - Adding test jobs to the graph without a build associated # to it does not work. This will be fixed once we switch to scheduling # Buildbot jobs through BBB if complete: graph[b] = None else: ready_to_trigger.append(b) # The build job is running, there is nothing we can do elif objective is None: pass # We need to trigger the build job and the test job else: if objective not in graph: graph[objective] = {} graph[objective][b] = None else: if b not in graph: graph[b] = {} # We might have left a build job poiting to an empty dict for builder in graph: if graph[builder] == {}: graph[builder] = None return graph, ready_to_trigger
def test_valid(self, fetch_allthethings_data): """is_downstream should return True for test jobs and False for build jobs.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert is_downstream( 'Windows 7 VM 32-bit try opt test cppunit') is True assert is_downstream('WINNT 5.2 mozilla-esr45 build') is False
def _generate_tc_tasks_from_builders(builders, repo_name, revision): """ Return TC tasks based on a list of builders. Input: a list of builders and a revision Output: list of TC tasks base on builders we receive :param builders: List of builder names :type builders: list :param repo_name: push revision :type repo_name: str :param revision: push revision :type revision: str :return: TC tasks :rtype: dict """ tasks = [] build_builders = {} # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for builder in builders: if is_upstream(builder): task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={'upload_to_task_id': slugId()}, ) tasks.append(task) # We want to keep track of how many build builders we have build_builders[builder] = task for builder in builders: if is_downstream(builder): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective, package_url, tests_url = \ determine_trigger_objective(revision, builder) # The build job is already completed, we can trigger the test job if objective == builder: if objective in build_builders: LOG.warning( "We're creating a new build even though there's " "already an existing completed build we could have " "used. We hope you wanted to do this.") task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=build_builders[objective]['taskId'], properties={'upload_to_task_id': slugId()}, ) tasks.append(task) else: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, properties={ 'packageUrl': package_url, 'testUrl': tests_url }, ) tasks.append(task) # The build job is running, there is nothing we can do elif objective is None: LOG.warning( "We can add %s builder since the build associated " "is running. This is because it is a Buildbot job.") pass # We need to trigger the build job and the test job else: if objective not in build_builders: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={'upload_to_task_id': slugId()}, ) tasks.append(task) taskId = task['taskId'] else: taskId = build_builders[objective]['taskId'] # Add test job task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=taskId, ) tasks.append(task) return tasks
def buildbot_graph_builder(builders, revision, complete=True): """ Return graph of builders based on a list of builders. # XXX: It would be better if had a BuildbotGraph class instead of messing with dictionaries. https://github.com/mozilla/mozilla_ci_tools/issues/353 Input: a list of builders and a revision Output: a graph with the builders we received and the necessary upstream jobs Graph of N levels: { 'Builder a1': { 'Builder a2': { ... 'Builder aN': None }, }, 'Builder b1': None } :param builders: List of builder names :type builders: list :param revision: push revision :type revision: str :return: A graph of buildernames (single level of graphs) :rtype: dict """ graph = {} ready_to_trigger = [] # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for b in builders: if is_downstream(b): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective = determine_trigger_objective(revision, b)[0] # The build job is already completed, we can trigger the test job if objective == b: # If complete is True, we add the job to our graph if complete: graph[b] = None else: ready_to_trigger.append(b) # The build job is running, there is nothing we can do elif objective is None: pass # We need to trigger the build job and the test job else: if objective not in graph: graph[objective] = {} graph[objective][b] = None else: if b not in graph: graph[b] = {} # We might have left a build job poiting to an empty dict for builder in graph: if graph[builder] == {}: graph[builder] = None return graph, ready_to_trigger
def test_valid(self, fetch_allthethings_data): """is_downstream should return True for test jobs and False for build jobs.""" fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS self.assertEquals(is_downstream('Platform1 repo opt test mochitest-1'), True) self.assertEquals(is_downstream('Platform1 repo build'), False)
def buildbot_graph_builder(builders, revision): """ Return graph of builders based on a list of builders. # XXX: It would be better if had a BuildbotGraph class instead of messing with dictionaries. https://github.com/mozilla/mozilla_ci_tools/issues/353 Input: a list of builders and a revision Output: a graph with the builders we received and the necessary upstream jobs Graph of N levels: { 'Builder a1': { 'Builder a2': { ... 'Builder aN': None }, }, 'Builder b1': None } :param builders: List of builder names :type builders: list :param revision: push revision :type revision: str :return: A graph of buildernames (single level of graphs) :rtype: dict """ graph = {} # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for b in builders: if is_downstream(b): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective = determine_trigger_objective(revision, b)[0] # The build job is already completed, we can trigger the test job if objective == b: graph[b] = None # The build job is running, there is nothing we can do elif objective is None: pass # We need to trigger the build job and the test job else: if objective not in graph: graph[objective] = {} graph[objective][b] = None else: if b not in graph: graph[b] = {} # We might have left a build job poiting to an empty dict for builder in graph: if graph[builder] == {}: graph[builder] = None return graph
def test_valid(self, fetch_allthethings_data): """is_downstream should return True for test jobs and False for build jobs.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert is_downstream('Windows 7 VM 32-bit try opt test cppunit') is True assert is_downstream('WINNT 5.2 mozilla-esr45 build') is False
def _generate_tc_tasks_from_builders(builders, repo_name, revision): """ Return TC tasks based on a list of builders. Input: a list of builders and a revision Output: list of TC tasks base on builders we receive :param builders: List of builder names :type builders: list :param repo_name: push revision :type repo_name: str :param revision: push revision :type revision: str :return: TC tasks :rtype: dict """ tasks = [] build_builders = {} # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for builder in builders: if is_upstream(builder): task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={"upload_to_task_id": slugId()}, ) tasks.append(task) # We want to keep track of how many build builders we have build_builders[builder] = task for builder in builders: if is_downstream(builder): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective, package_url, tests_url = determine_trigger_objective(revision, builder) # The build job is already completed, we can trigger the test job if objective == builder: if objective in build_builders: LOG.warning( "We're creating a new build even though there's " "already an existing completed build we could have " "used. We hope you wanted to do this." ) task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=build_builders[objective]["taskId"], properties={"upload_to_task_id": slugId()}, ) tasks.append(task) else: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, properties={"packageUrl": package_url, "testUrl": tests_url}, ) tasks.append(task) # The build job is running, there is nothing we can do elif objective is None: LOG.warning( "We can add %s builder since the build associated " "is running. This is because it is a Buildbot job." ) pass # We need to trigger the build job and the test job else: if objective not in build_builders: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={"upload_to_task_id": slugId()}, ) tasks.append(task) taskId = task["taskId"] else: taskId = build_builders[objective]["taskId"] # Add test job task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=taskId, ) tasks.append(task) return tasks