def test_nightly_build(test_job, expected): """Test whether nightly builds are caught by get_buildername_metadata""" import mozci.platforms mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS) obtained = get_buildername_metadata(test_job)['nightly'] assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained, expected)
def test_suite_name(test_job, expected): """Test if the function finds the right suite_name.""" import mozci.platforms mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS) obtained = get_buildername_metadata(test_job)['suite_name'] assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained, expected)
def test_suite_name(self, fetch_allthethings_data): """Test if the function finds the right suite_name.""" fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS test_cases = [("Platform1 repo talos tp5o", "tp5o"), ("Platform1 repo opt test mochitest-1", "mochitest-1")] for t in test_cases: self.assertEquals( get_buildername_metadata(t[0])['suite_name'], t[1])
def test_suite_name(self, fetch_allthethings_data): """Test if the function finds the right suite_name.""" fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS test_cases = [ ("Platform1 repo talos tp5o", "tp5o"), ("Platform1 repo opt test mochitest-1", "mochitest-1") ] for t in test_cases: self.assertEquals(get_buildername_metadata(t[0])['suite_name'], t[1])
def ignored(data): '''It determines if the request will be processed or not.''' try: info = get_buildername_metadata(data['payload']['buildername']) if info['build_type'] == "pgo" and \ info['repo_name'] in ('mozilla-inbound', 'fx-team', 'autoland') and \ info['platform_name'] != 'win64': return False else: return True except MissingBuilderError, e: LOG.warning(str(e)) return True
def ignored(data): """It determines if the request will be processed or not.""" try: info = get_buildername_metadata(data["payload"]["buildername"]) if ( info["build_type"] == "pgo" and info["repo_name"] in ("mozilla-inbound", "fx-team", "autoland") and info["platform_name"] != "win64" ): return False else: return True except MissingBuilderError, e: LOG.warning(str(e)) return True
def _generate_tasks(repo_name, revision, builders_graph, metadata=None, task_graph_id=None, parent_task_id=None, required_task_ids=[], **kwargs): """ Generate a TC json object with tasks based on a graph of graphs of buildernames :param repo_name: The name of a repository e.g. mozilla-inbound :type repo_name: str :param revision: Changeset ID of a revision. :type revision: str :param builders_graph: It is a graph made up of a dictionary where each key is a Buildbot buildername. The value for each key is either None or another graph of dependent builders. :type builders_graph: dict :param metadata: Metadata information to set for the tasks. :type metadata: json :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param parent_task_id: Task from which to find artifacts. It is not a dependency. :type parent_task_id: int :returns: A dictionary of TC tasks :rtype: dict """ if not type(required_task_ids) == list: raise MozciError("required_task_ids must be a list") tasks = [] if type(builders_graph) != dict: raise MozciError("The buildbot graph should be a dictionary") # Let's iterate through the root builders in this graph for builder, dependent_graph in builders_graph.iteritems(): # Due to bug 1221091 this will be used to know to which task # the artifacts will be uploaded to upload_to_task_id = slugId() properties = {'upload_to_task_id': upload_to_task_id} builder_details = get_buildername_metadata(builder) # Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks, # thus, it fails when we tell it to upload to the same task twice. if builder_details['platform_name'].startswith('android') and \ builder_details['nightly'] is True and \ 'l10n' not in builder: properties = {} task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, metadata=metadata, task_graph_id=task_graph_id, parent_task_id=parent_task_id, properties=properties, requires=required_task_ids, **kwargs ) task_id = task['taskId'] tasks.append(task) if dependent_graph: # If there are builders this builder triggers let's add them as well tasks = tasks + _generate_tasks( repo_name=repo_name, revision=revision, builders_graph=dependent_graph, metadata=metadata, task_graph_id=task_graph_id, # The parent task id is used to find artifacts; only one can be given parent_task_id=upload_to_task_id, # The required tasks are the one holding this task from running required_task_ids=[task_id], **kwargs ) return tasks
def _generate_tc_tasks_from_builders(builders, repo_name, revision): """ Return TC tasks based on a list of builders. Input: a list of builders and a revision Output: list of TC tasks base on builders we receive :param builders: List of builder names :type builders: list :param repo_name: push revision :type repo_name: str :param revision: push revision :type revision: str :return: TC tasks :rtype: dict """ tasks = [] build_builders = {} # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for builder in builders: if is_upstream(builder): properties = {'upload_to_task_id': slugId()} # Bug 1274483 - Android multi-locale nightly builds need to upload to two different # tasks, thus, it fails when we tell it to upload to the same task twice. builder_details = get_buildername_metadata(builder) if builder_details['platform_name'].startswith('android') and \ builder_details['nightly'] is True and \ 'l10n' not in builder: properties = {} task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties=properties, ) tasks.append(task) # We want to keep track of how many build builders we have build_builders[builder] = task for builder in builders: if is_downstream(builder): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective, package_url, tests_url = \ determine_trigger_objective(revision, builder) # The build job is already completed, we can trigger the test job if objective == builder: if objective in build_builders: LOG.warning("We're creating a new build even though there's " "already an existing completed build we could have " "used. We hope you wanted to do this.") task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=build_builders[objective]['taskId'], properties={'upload_to_task_id': slugId()}, ) tasks.append(task) else: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, properties={ 'packageUrl': package_url, 'testUrl': tests_url }, ) tasks.append(task) # The build job is running, there is nothing we can do elif objective is None: LOG.warning("We can add %s builder since the build associated " "is running. This is because it is a Buildbot job.") pass # We need to trigger the build job and the test job else: if objective not in build_builders: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={'upload_to_task_id': slugId()}, ) tasks.append(task) taskId = task['taskId'] else: taskId = build_builders[objective]['taskId'] # Add test job task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=taskId, ) tasks.append(task) return tasks
def _create_task(buildername, repo_name, revision, metadata=None, task_graph_id=None, parent_task_id=None, requires=None, properties={}, *args, **kwargs): """Return takcluster task to trigger a buildbot builder. This function creates a generic task with the minimum amount of information required for the buildbot-bridge to consider it valid. You can establish a list dependencies to other tasks through the requires field. :param buildername: The name of a buildbot builder. :type buildername: str :param repo_name: The name of a repository e.g. mozilla-inbound :type repo_name: str :param revision: Changeset ID of a revision. :type revision: str :param metadata: Metadata for the task. If not specified, generate it. :type metadata: json :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param parent_task_id: Task from which to find artifacts. It is not a dependency. :type parent_task_id: str :param requires: List of taskIds of other tasks which this task depends on. :type requires: list :returns: TaskCluster graph :rtype: dict """ if not valid_builder(buildername): raise MozciError("The builder '%s' is not a valid one." % buildername) builder_info = get_buildername_metadata(buildername) if builder_info['repo_name'] != repo_name: raise MozciError( "The builder '%s' should be for repo: %s." % (buildername, repo_name) ) repo_url = query_repo_url(repo_name) push_info = query_push_by_revision(repo_url=repo_url, revision=revision) full_revision = str(push_info.changesets[0].node) # Needed because of bug 1195751 all_properties = { 'product': builder_info['product'], 'who': push_info.user, } all_properties.update(properties) all_properties.update(get_builder_extra_properties(buildername)) metadata = metadata if metadata is not None else \ generate_metadata(repo_name=repo_name, revision=revision, name=buildername) # The task's name is used in the task-graph-inspector to list all tasks # and using the buildername makes it easy for a person to recognize each job. metadata['name'] = buildername # XXX: We should validate that the parent task is a valid parent platform # e.g. do not schedule Windows tests against Linux builds task = create_task( repo_name=repo_name, revision=revision, taskGroupId=task_graph_id, workerType='buildbot-bridge', provisionerId='buildbot-bridge', payload={ 'buildername': buildername, 'sourcestamp': { 'branch': repo_name, 'revision': full_revision }, 'properties': all_properties, }, metadata=metadata, ) if requires: task['requires'] = requires # Setting a parent_task_id as a property allows Mozharness to # determine the artifacts we need for this job to run properly if parent_task_id: task['task']['payload']['properties']['parent_task_id'] = parent_task_id return task
def _create_task(buildername, repo_name, revision, task_graph_id=None, parent_task_id=None, requires=None, properties={}): """Return takcluster task to trigger a buildbot builder. This function creates a generic task with the minimum amount of information required for the buildbot-bridge to consider it valid. You can establish a list dependencies to other tasks through the requires field. :param buildername: The name of a buildbot builder. :type buildername: str :param repo_name: The name of a repository e.g. mozilla-inbound :type repo_name: str :param revision: Changeset ID of a revision. :type revision: str :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param parent_task_id: Task from which to find artifacts. It is not a dependency. :type parent_task_id: str :param requires: List of taskIds of other tasks which this task depends on. :type requires: list :returns: TaskCluster graph :rtype: dict """ if not valid_builder(buildername): raise MozciError("The builder '%s' is not a valid one." % buildername) builder_info = get_buildername_metadata(buildername) if builder_info['repo_name'] != repo_name: raise MozciError("The builder '%s' should be for repo: %s." % (buildername, repo_name)) repo_url = query_repo_url(repo_name) push_info = query_revision_info(repo_url, revision) # Needed because of bug 1195751 all_properties = { 'product': builder_info['product'], 'who': push_info['user'], } all_properties.update(properties) # XXX: We should validate that the parent task is a valid parent platform # e.g. do not schedule Windows tests against Linux builds task = create_task(repo_name=repo_name, revision=revision, taskGroupId=task_graph_id, workerType='buildbot-bridge', provisionerId='buildbot-bridge', payload={ 'buildername': buildername, 'sourcestamp': { 'branch': repo_name, 'revision': revision }, 'properties': all_properties, }, metadata_name=buildername) if requires: task['requires'] = requires # Setting a parent_task_id as a property allows Mozharness to # determine the artifacts we need for this job to run properly if parent_task_id: task['task']['payload']['properties'][ 'parent_task_id'] = parent_task_id return task
def _create_task(buildername, repo_name, revision, task_graph_id=None, parent_task_id=None, requires=None, properties={}): """Return takcluster task to trigger a buildbot builder. This function creates a generic task with the minimum amount of information required for the buildbot-bridge to consider it valid. You can establish a list dependencies to other tasks through the requires field. :param buildername: The name of a buildbot builder. :type buildername: str :param repo_name: The name of a repository e.g. mozilla-inbound :type repo_name: str :param revision: Changeset ID of a revision. :type revision: str :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param parent_task_id: Task from which to find artifacts. It is not a dependency. :type parent_task_id: str :param requires: List of taskIds of other tasks which this task depends on. :type requires: list :returns: TaskCluster graph :rtype: dict """ if not valid_builder(buildername): raise MozciError("The builder '%s' is not a valid one." % buildername) builder_info = get_buildername_metadata(buildername) if builder_info['repo_name'] != repo_name: raise MozciError( "The builder '%s' should be for repo: %s." % (buildername, repo_name) ) repo_url = query_repo_url(repo_name) push_info = query_revision_info(repo_url, revision) # Needed because of bug 1195751 all_properties = { 'product': builder_info['product'], 'who': push_info['user'], } all_properties.update(properties) # XXX: We should validate that the parent task is a valid parent platform # e.g. do not schedule Windows tests against Linux builds task = create_task( repo_name=repo_name, revision=revision, taskGroupId=task_graph_id, workerType='buildbot-bridge', provisionerId='buildbot-bridge', payload={ 'buildername': buildername, 'sourcestamp': { 'branch': repo_name, 'revision': revision }, 'properties': all_properties, }, metadata_name=buildername ) if requires: task['requires'] = requires # Setting a parent_task_id as a property allows Mozharness to # determine the artifacts we need for this job to run properly if parent_task_id: task['task']['payload']['properties']['parent_task_id'] = parent_task_id return task
def _generate_tasks(repo_name, revision, builders_graph, metadata=None, task_graph_id=None, parent_task_id=None, required_task_ids=[], **kwargs): """ Generate a TC json object with tasks based on a graph of graphs of buildernames :param repo_name: The name of a repository e.g. mozilla-inbound :type repo_name: str :param revision: Changeset ID of a revision. :type revision: str :param builders_graph: It is a graph made up of a dictionary where each key is a Buildbot buildername. The value for each key is either None or another graph of dependent builders. :type builders_graph: dict :param metadata: Metadata information to set for the tasks. :type metadata: json :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param parent_task_id: Task from which to find artifacts. It is not a dependency. :type parent_task_id: int :returns: A dictionary of TC tasks :rtype: dict """ if not type(required_task_ids) == list: raise MozciError("required_task_ids must be a list") tasks = [] if type(builders_graph) != dict: raise MozciError("The buildbot graph should be a dictionary") # Let's iterate through the root builders in this graph for builder, dependent_graph in builders_graph.iteritems(): # Due to bug 1221091 this will be used to know to which task # the artifacts will be uploaded to upload_to_task_id = slugId() properties = {'upload_to_task_id': upload_to_task_id} builder_details = get_buildername_metadata(builder) # Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks, # thus, it fails when we tell it to upload to the same task twice. if builder_details['platform_name'].startswith('android') and \ builder_details['nightly'] is True and \ 'l10n' not in builder: properties = {} task = _create_task(buildername=builder, repo_name=repo_name, revision=revision, metadata=metadata, task_graph_id=task_graph_id, parent_task_id=parent_task_id, properties=properties, requires=required_task_ids, **kwargs) task_id = task['taskId'] tasks.append(task) if dependent_graph: # If there are builders this builder triggers let's add them as well tasks = tasks + _generate_tasks( repo_name=repo_name, revision=revision, builders_graph=dependent_graph, metadata=metadata, task_graph_id=task_graph_id, # The parent task id is used to find artifacts; only one can be given parent_task_id=upload_to_task_id, # The required tasks are the one holding this task from running required_task_ids=[task_id], **kwargs) return tasks
def _generate_tc_tasks_from_builders(builders, repo_name, revision): """ Return TC tasks based on a list of builders. Input: a list of builders and a revision Output: list of TC tasks base on builders we receive :param builders: List of builder names :type builders: list :param repo_name: push revision :type repo_name: str :param revision: push revision :type revision: str :return: TC tasks :rtype: dict """ tasks = [] build_builders = {} # We need to determine what upstream jobs need to be triggered besides the # builders already on our list for builder in builders: if is_upstream(builder): properties = {'upload_to_task_id': slugId()} # Bug 1274483 - Android multi-locale nightly builds need to upload to two different # tasks, thus, it fails when we tell it to upload to the same task twice. builder_details = get_buildername_metadata(builder) if builder_details['platform_name'].startswith('android') and \ builder_details['nightly'] is True and \ 'l10n' not in builder: properties = {} task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties=properties, ) tasks.append(task) # We want to keep track of how many build builders we have build_builders[builder] = task for builder in builders: if is_downstream(builder): # For test jobs, determine_trigger_objective()[0] can be 3 things: # - the build job, if no build job exists # - the test job, if the build job is already completed # - None, if the build job is running objective, package_url, tests_url = \ determine_trigger_objective(revision, builder) # The build job is already completed, we can trigger the test job if objective == builder: if objective in build_builders: LOG.warning( "We're creating a new build even though there's " "already an existing completed build we could have " "used. We hope you wanted to do this.") task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=build_builders[objective]['taskId'], properties={'upload_to_task_id': slugId()}, ) tasks.append(task) else: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, properties={ 'packageUrl': package_url, 'testUrl': tests_url }, ) tasks.append(task) # The build job is running, there is nothing we can do elif objective is None: LOG.warning( "We can add %s builder since the build associated " "is running. This is because it is a Buildbot job.") pass # We need to trigger the build job and the test job else: if objective not in build_builders: task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, properties={'upload_to_task_id': slugId()}, ) tasks.append(task) taskId = task['taskId'] else: taskId = build_builders[objective]['taskId'] # Add test job task = _create_task( buildername=builder, repo_name=repo_name, revision=revision, # task_graph_id=task_graph_id, parent_task_id=taskId, ) tasks.append(task) return tasks
def _create_task(buildername, repo_name, revision, metadata=None, task_graph_id=None, parent_task_id=None, requires=None, properties={}, *args, **kwargs): """Return takcluster task to trigger a buildbot builder. This function creates a generic task with the minimum amount of information required for the buildbot-bridge to consider it valid. You can establish a list dependencies to other tasks through the requires field. :param buildername: The name of a buildbot builder. :type buildername: str :param repo_name: The name of a repository e.g. mozilla-inbound :type repo_name: str :param revision: Changeset ID of a revision. :type revision: str :param metadata: Metadata for the task. If not specified, generate it. :type metadata: json :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param parent_task_id: Task from which to find artifacts. It is not a dependency. :type parent_task_id: str :param requires: List of taskIds of other tasks which this task depends on. :type requires: list :returns: TaskCluster graph :rtype: dict """ if not valid_builder(buildername): raise MozciError("The builder '%s' is not a valid one." % buildername) builder_info = get_buildername_metadata(buildername) if builder_info['repo_name'] != repo_name: raise MozciError("The builder '%s' should be for repo: %s." % (buildername, repo_name)) repo_url = query_repo_url(repo_name) push_info = query_push_by_revision(repo_url=repo_url, revision=revision) full_revision = str(push_info.changesets[0].node) # Needed because of bug 1195751 all_properties = { 'product': builder_info['product'], 'who': push_info.user, } all_properties.update(properties) all_properties.update(get_builder_extra_properties(buildername)) metadata = metadata if metadata is not None else \ generate_metadata(repo_name=repo_name, revision=revision, name=buildername) # The task's name is used in the task-graph-inspector to list all tasks # and using the buildername makes it easy for a person to recognize each job. metadata['name'] = buildername # XXX: We should validate that the parent task is a valid parent platform # e.g. do not schedule Windows tests against Linux builds task = create_task( repo_name=repo_name, revision=revision, taskGroupId=task_graph_id, workerType='buildbot-bridge', provisionerId='buildbot-bridge', payload={ 'buildername': buildername, 'sourcestamp': { 'branch': repo_name, 'revision': full_revision }, 'properties': all_properties, }, metadata=metadata, ) if requires: task['requires'] = requires # Setting a parent_task_id as a property allows Mozharness to # determine the artifacts we need for this job to run properly if parent_task_id: task['task']['payload']['properties'][ 'parent_task_id'] = parent_task_id return task