예제 #1
0
def test_nightly_build(test_job, expected):
    """Test whether nightly builds are caught by get_buildername_metadata"""
    import mozci.platforms
    mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
    obtained = get_buildername_metadata(test_job)['nightly']
    assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained,
                                                                    expected)
예제 #2
0
def test_suite_name(test_job, expected):
    """Test if the function finds the right suite_name."""
    import mozci.platforms
    mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
    obtained = get_buildername_metadata(test_job)['suite_name']
    assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained,
                                                                    expected)
예제 #3
0
 def test_suite_name(self, fetch_allthethings_data):
     """Test if the function finds the right suite_name."""
     fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS
     test_cases = [("Platform1 repo talos tp5o", "tp5o"),
                   ("Platform1 repo opt test mochitest-1", "mochitest-1")]
     for t in test_cases:
         self.assertEquals(
             get_buildername_metadata(t[0])['suite_name'], t[1])
 def test_suite_name(self, fetch_allthethings_data):
     """Test if the function finds the right suite_name."""
     fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS
     test_cases = [
         ("Platform1 repo talos tp5o", "tp5o"),
         ("Platform1 repo opt test mochitest-1", "mochitest-1")
     ]
     for t in test_cases:
         self.assertEquals(get_buildername_metadata(t[0])['suite_name'], t[1])
예제 #5
0
def ignored(data):
    '''It determines if the request will be processed or not.'''
    try:
        info = get_buildername_metadata(data['payload']['buildername'])
        if info['build_type'] == "pgo" and \
           info['repo_name'] in ('mozilla-inbound', 'fx-team', 'autoland') and \
           info['platform_name'] != 'win64':
            return False
        else:
            return True
    except MissingBuilderError, e:
        LOG.warning(str(e))

        return True
예제 #6
0
def ignored(data):
    """It determines if the request will be processed or not."""
    try:
        info = get_buildername_metadata(data["payload"]["buildername"])
        if (
            info["build_type"] == "pgo"
            and info["repo_name"] in ("mozilla-inbound", "fx-team", "autoland")
            and info["platform_name"] != "win64"
        ):
            return False
        else:
            return True
    except MissingBuilderError, e:
        LOG.warning(str(e))

        return True
예제 #7
0
def _generate_tasks(repo_name, revision, builders_graph, metadata=None, task_graph_id=None,
                    parent_task_id=None, required_task_ids=[], **kwargs):
    """ Generate a TC json object with tasks based on a graph of graphs of buildernames

    :param repo_name: The name of a repository e.g. mozilla-inbound
    :type repo_name: str
    :param revision: Changeset ID of a revision.
    :type revision: str
    :param builders_graph:
        It is a graph made up of a dictionary where each
        key is a Buildbot buildername. The value for each key is either None
        or another graph of dependent builders.
    :type builders_graph: dict
    :param metadata: Metadata information to set for the tasks.
    :type metadata: json
    :param task_graph_id: TC graph id to which this task belongs to
    :type task_graph_id: str
    :param parent_task_id: Task from which to find artifacts. It is not a dependency.
    :type parent_task_id: int
    :returns: A dictionary of TC tasks
    :rtype: dict

    """
    if not type(required_task_ids) == list:
        raise MozciError("required_task_ids must be a list")

    tasks = []

    if type(builders_graph) != dict:
        raise MozciError("The buildbot graph should be a dictionary")

    # Let's iterate through the root builders in this graph
    for builder, dependent_graph in builders_graph.iteritems():
        # Due to bug 1221091 this will be used to know to which task
        # the artifacts will be uploaded to
        upload_to_task_id = slugId()
        properties = {'upload_to_task_id': upload_to_task_id}
        builder_details = get_buildername_metadata(builder)

        # Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks,
        # thus, it fails when we tell it to upload to the same task twice.
        if builder_details['platform_name'].startswith('android') and \
           builder_details['nightly'] is True and \
           'l10n' not in builder:
            properties = {}

        task = _create_task(
            buildername=builder,
            repo_name=repo_name,
            revision=revision,
            metadata=metadata,
            task_graph_id=task_graph_id,
            parent_task_id=parent_task_id,
            properties=properties,
            requires=required_task_ids,
            **kwargs
        )
        task_id = task['taskId']
        tasks.append(task)

        if dependent_graph:
            # If there are builders this builder triggers let's add them as well
            tasks = tasks + _generate_tasks(
                repo_name=repo_name,
                revision=revision,
                builders_graph=dependent_graph,
                metadata=metadata,
                task_graph_id=task_graph_id,
                # The parent task id is used to find artifacts; only one can be given
                parent_task_id=upload_to_task_id,
                # The required tasks are the one holding this task from running
                required_task_ids=[task_id],
                **kwargs
            )

    return tasks
예제 #8
0
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
    """ Return TC tasks based on a list of builders.

    Input: a list of builders and a revision
    Output: list of TC tasks base on builders we receive

    :param builders: List of builder names
    :type builders: list
    :param repo_name: push revision
    :type repo_name: str
    :param revision: push revision
    :type revision: str
    :return: TC tasks
    :rtype: dict

    """
    tasks = []
    build_builders = {}

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for builder in builders:
        if is_upstream(builder):
            properties = {'upload_to_task_id': slugId()}

            # Bug 1274483 - Android multi-locale nightly builds need to upload to two different
            # tasks, thus, it fails when we tell it to upload to the same task twice.
            builder_details = get_buildername_metadata(builder)
            if builder_details['platform_name'].startswith('android') and \
               builder_details['nightly'] is True and \
               'l10n' not in builder:
                properties = {}

            task = _create_task(
                buildername=builder,
                repo_name=repo_name,
                revision=revision,
                # task_graph_id=task_graph_id,
                properties=properties,
            )
            tasks.append(task)

            # We want to keep track of how many build builders we have
            build_builders[builder] = task

    for builder in builders:
        if is_downstream(builder):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective, package_url, tests_url = \
                determine_trigger_objective(revision, builder)

            # The build job is already completed, we can trigger the test job
            if objective == builder:
                if objective in build_builders:
                    LOG.warning("We're creating a new build even though there's "
                                "already an existing completed build we could have "
                                "used. We hope you wanted to do this.")
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        parent_task_id=build_builders[objective]['taskId'],
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                else:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        properties={
                            'packageUrl': package_url,
                            'testUrl': tests_url
                        },
                    )
                    tasks.append(task)

            # The build job is running, there is nothing we can do
            elif objective is None:
                LOG.warning("We can add %s builder since the build associated "
                            "is running. This is because it is a Buildbot job.")
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in build_builders:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                    taskId = task['taskId']
                else:
                    taskId = build_builders[objective]['taskId']

                # Add test job
                task = _create_task(
                    buildername=builder,
                    repo_name=repo_name,
                    revision=revision,
                    # task_graph_id=task_graph_id,
                    parent_task_id=taskId,
                )
                tasks.append(task)

    return tasks
예제 #9
0
def _create_task(buildername, repo_name, revision, metadata=None, task_graph_id=None,
                 parent_task_id=None, requires=None, properties={}, *args, **kwargs):
    """Return takcluster task to trigger a buildbot builder.

    This function creates a generic task with the minimum amount of
    information required for the buildbot-bridge to consider it valid.
    You can establish a list dependencies to other tasks through the requires
    field.

    :param buildername: The name of a buildbot builder.
    :type buildername: str
    :param repo_name: The name of a repository e.g. mozilla-inbound
    :type repo_name: str
    :param revision: Changeset ID of a revision.
    :type revision: str
    :param metadata: Metadata for the task. If not specified, generate it.
    :type metadata: json
    :param task_graph_id: TC graph id to which this task belongs to
    :type task_graph_id: str
    :param parent_task_id: Task from which to find artifacts. It is not a dependency.
    :type parent_task_id: str
    :param requires: List of taskIds of other tasks which this task depends on.
    :type requires: list
    :returns: TaskCluster graph
    :rtype: dict

    """
    if not valid_builder(buildername):
        raise MozciError("The builder '%s' is not a valid one." % buildername)

    builder_info = get_buildername_metadata(buildername)
    if builder_info['repo_name'] != repo_name:
        raise MozciError(
            "The builder '%s' should be for repo: %s." % (buildername, repo_name)
        )

    repo_url = query_repo_url(repo_name)
    push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
    full_revision = str(push_info.changesets[0].node)

    # Needed because of bug 1195751
    all_properties = {
        'product': builder_info['product'],
        'who': push_info.user,
    }
    all_properties.update(properties)

    all_properties.update(get_builder_extra_properties(buildername))

    metadata = metadata if metadata is not None else \
        generate_metadata(repo_name=repo_name,
                          revision=revision,
                          name=buildername)

    # The task's name is used in the task-graph-inspector to list all tasks
    # and using the buildername makes it easy for a person to recognize each job.
    metadata['name'] = buildername

    # XXX: We should validate that the parent task is a valid parent platform
    #      e.g. do not schedule Windows tests against Linux builds
    task = create_task(
        repo_name=repo_name,
        revision=revision,
        taskGroupId=task_graph_id,
        workerType='buildbot-bridge',
        provisionerId='buildbot-bridge',
        payload={
            'buildername': buildername,
            'sourcestamp': {
                'branch': repo_name,
                'revision': full_revision
            },
            'properties': all_properties,
        },
        metadata=metadata,
    )

    if requires:
        task['requires'] = requires

    # Setting a parent_task_id as a property allows Mozharness to
    # determine the artifacts we need for this job to run properly
    if parent_task_id:
        task['task']['payload']['properties']['parent_task_id'] = parent_task_id

    return task
예제 #10
0
def _create_task(buildername,
                 repo_name,
                 revision,
                 task_graph_id=None,
                 parent_task_id=None,
                 requires=None,
                 properties={}):
    """Return takcluster task to trigger a buildbot builder.

    This function creates a generic task with the minimum amount of
    information required for the buildbot-bridge to consider it valid.
    You can establish a list dependencies to other tasks through the requires
    field.

    :param buildername: The name of a buildbot builder.
    :type buildername: str
    :param repo_name: The name of a repository e.g. mozilla-inbound
    :type repo_name: str
    :param revision: Changeset ID of a revision.
    :type revision: str
    :param task_graph_id: TC graph id to which this task belongs to
    :type task_graph_id: str
    :param parent_task_id: Task from which to find artifacts. It is not a dependency.
    :type parent_task_id: str
    :param requires: List of taskIds of other tasks which this task depends on.
    :type requires: list
    :returns: TaskCluster graph
    :rtype: dict

    """
    if not valid_builder(buildername):
        raise MozciError("The builder '%s' is not a valid one." % buildername)

    builder_info = get_buildername_metadata(buildername)
    if builder_info['repo_name'] != repo_name:
        raise MozciError("The builder '%s' should be for repo: %s." %
                         (buildername, repo_name))

    repo_url = query_repo_url(repo_name)
    push_info = query_revision_info(repo_url, revision)

    # Needed because of bug 1195751
    all_properties = {
        'product': builder_info['product'],
        'who': push_info['user'],
    }
    all_properties.update(properties)

    # XXX: We should validate that the parent task is a valid parent platform
    #      e.g. do not schedule Windows tests against Linux builds
    task = create_task(repo_name=repo_name,
                       revision=revision,
                       taskGroupId=task_graph_id,
                       workerType='buildbot-bridge',
                       provisionerId='buildbot-bridge',
                       payload={
                           'buildername': buildername,
                           'sourcestamp': {
                               'branch': repo_name,
                               'revision': revision
                           },
                           'properties': all_properties,
                       },
                       metadata_name=buildername)

    if requires:
        task['requires'] = requires

    # Setting a parent_task_id as a property allows Mozharness to
    # determine the artifacts we need for this job to run properly
    if parent_task_id:
        task['task']['payload']['properties'][
            'parent_task_id'] = parent_task_id

    return task
def _create_task(buildername, repo_name, revision, task_graph_id=None,
                 parent_task_id=None, requires=None, properties={}):
    """Return takcluster task to trigger a buildbot builder.

    This function creates a generic task with the minimum amount of
    information required for the buildbot-bridge to consider it valid.
    You can establish a list dependencies to other tasks through the requires
    field.

    :param buildername: The name of a buildbot builder.
    :type buildername: str
    :param repo_name: The name of a repository e.g. mozilla-inbound
    :type repo_name: str
    :param revision: Changeset ID of a revision.
    :type revision: str
    :param task_graph_id: TC graph id to which this task belongs to
    :type task_graph_id: str
    :param parent_task_id: Task from which to find artifacts. It is not a dependency.
    :type parent_task_id: str
    :param requires: List of taskIds of other tasks which this task depends on.
    :type requires: list
    :returns: TaskCluster graph
    :rtype: dict

    """
    if not valid_builder(buildername):
        raise MozciError("The builder '%s' is not a valid one." % buildername)

    builder_info = get_buildername_metadata(buildername)
    if builder_info['repo_name'] != repo_name:
        raise MozciError(
            "The builder '%s' should be for repo: %s." % (buildername, repo_name)
        )

    repo_url = query_repo_url(repo_name)
    push_info = query_revision_info(repo_url, revision)

    # Needed because of bug 1195751
    all_properties = {
        'product': builder_info['product'],
        'who': push_info['user'],
    }
    all_properties.update(properties)

    # XXX: We should validate that the parent task is a valid parent platform
    #      e.g. do not schedule Windows tests against Linux builds
    task = create_task(
        repo_name=repo_name,
        revision=revision,
        taskGroupId=task_graph_id,
        workerType='buildbot-bridge',
        provisionerId='buildbot-bridge',
        payload={
            'buildername': buildername,
            'sourcestamp': {
                'branch': repo_name,
                'revision': revision
            },
            'properties': all_properties,
        },
        metadata_name=buildername
    )

    if requires:
        task['requires'] = requires

    # Setting a parent_task_id as a property allows Mozharness to
    # determine the artifacts we need for this job to run properly
    if parent_task_id:
        task['task']['payload']['properties']['parent_task_id'] = parent_task_id

    return task
예제 #12
0
def test_nightly_build(test_job, expected):
    """Test whether nightly builds are caught by get_buildername_metadata"""
    import mozci.platforms
    mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
    obtained = get_buildername_metadata(test_job)['nightly']
    assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained, expected)
예제 #13
0
def test_suite_name(test_job, expected):
    """Test if the function finds the right suite_name."""
    import mozci.platforms
    mozci.platforms.fetch_allthethings_data = Mock(return_value=ALLTHETHINGS)
    obtained = get_buildername_metadata(test_job)['suite_name']
    assert obtained == expected, 'obtained: "%s", expected "%s"' % (obtained, expected)
예제 #14
0
def _generate_tasks(repo_name,
                    revision,
                    builders_graph,
                    metadata=None,
                    task_graph_id=None,
                    parent_task_id=None,
                    required_task_ids=[],
                    **kwargs):
    """ Generate a TC json object with tasks based on a graph of graphs of buildernames

    :param repo_name: The name of a repository e.g. mozilla-inbound
    :type repo_name: str
    :param revision: Changeset ID of a revision.
    :type revision: str
    :param builders_graph:
        It is a graph made up of a dictionary where each
        key is a Buildbot buildername. The value for each key is either None
        or another graph of dependent builders.
    :type builders_graph: dict
    :param metadata: Metadata information to set for the tasks.
    :type metadata: json
    :param task_graph_id: TC graph id to which this task belongs to
    :type task_graph_id: str
    :param parent_task_id: Task from which to find artifacts. It is not a dependency.
    :type parent_task_id: int
    :returns: A dictionary of TC tasks
    :rtype: dict

    """
    if not type(required_task_ids) == list:
        raise MozciError("required_task_ids must be a list")

    tasks = []

    if type(builders_graph) != dict:
        raise MozciError("The buildbot graph should be a dictionary")

    # Let's iterate through the root builders in this graph
    for builder, dependent_graph in builders_graph.iteritems():
        # Due to bug 1221091 this will be used to know to which task
        # the artifacts will be uploaded to
        upload_to_task_id = slugId()
        properties = {'upload_to_task_id': upload_to_task_id}
        builder_details = get_buildername_metadata(builder)

        # Bug 1274483 - Android multi-locale nightly builds need to upload to two different tasks,
        # thus, it fails when we tell it to upload to the same task twice.
        if builder_details['platform_name'].startswith('android') and \
           builder_details['nightly'] is True and \
           'l10n' not in builder:
            properties = {}

        task = _create_task(buildername=builder,
                            repo_name=repo_name,
                            revision=revision,
                            metadata=metadata,
                            task_graph_id=task_graph_id,
                            parent_task_id=parent_task_id,
                            properties=properties,
                            requires=required_task_ids,
                            **kwargs)
        task_id = task['taskId']
        tasks.append(task)

        if dependent_graph:
            # If there are builders this builder triggers let's add them as well
            tasks = tasks + _generate_tasks(
                repo_name=repo_name,
                revision=revision,
                builders_graph=dependent_graph,
                metadata=metadata,
                task_graph_id=task_graph_id,
                # The parent task id is used to find artifacts; only one can be given
                parent_task_id=upload_to_task_id,
                # The required tasks are the one holding this task from running
                required_task_ids=[task_id],
                **kwargs)

    return tasks
예제 #15
0
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
    """ Return TC tasks based on a list of builders.

    Input: a list of builders and a revision
    Output: list of TC tasks base on builders we receive

    :param builders: List of builder names
    :type builders: list
    :param repo_name: push revision
    :type repo_name: str
    :param revision: push revision
    :type revision: str
    :return: TC tasks
    :rtype: dict

    """
    tasks = []
    build_builders = {}

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for builder in builders:
        if is_upstream(builder):
            properties = {'upload_to_task_id': slugId()}

            # Bug 1274483 - Android multi-locale nightly builds need to upload to two different
            # tasks, thus, it fails when we tell it to upload to the same task twice.
            builder_details = get_buildername_metadata(builder)
            if builder_details['platform_name'].startswith('android') and \
               builder_details['nightly'] is True and \
               'l10n' not in builder:
                properties = {}

            task = _create_task(
                buildername=builder,
                repo_name=repo_name,
                revision=revision,
                # task_graph_id=task_graph_id,
                properties=properties,
            )
            tasks.append(task)

            # We want to keep track of how many build builders we have
            build_builders[builder] = task

    for builder in builders:
        if is_downstream(builder):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective, package_url, tests_url = \
                determine_trigger_objective(revision, builder)

            # The build job is already completed, we can trigger the test job
            if objective == builder:
                if objective in build_builders:
                    LOG.warning(
                        "We're creating a new build even though there's "
                        "already an existing completed build we could have "
                        "used. We hope you wanted to do this.")
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        parent_task_id=build_builders[objective]['taskId'],
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                else:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        properties={
                            'packageUrl': package_url,
                            'testUrl': tests_url
                        },
                    )
                    tasks.append(task)

            # The build job is running, there is nothing we can do
            elif objective is None:
                LOG.warning(
                    "We can add %s builder since the build associated "
                    "is running. This is because it is a Buildbot job.")
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in build_builders:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                    taskId = task['taskId']
                else:
                    taskId = build_builders[objective]['taskId']

                # Add test job
                task = _create_task(
                    buildername=builder,
                    repo_name=repo_name,
                    revision=revision,
                    # task_graph_id=task_graph_id,
                    parent_task_id=taskId,
                )
                tasks.append(task)

    return tasks
예제 #16
0
def _create_task(buildername,
                 repo_name,
                 revision,
                 metadata=None,
                 task_graph_id=None,
                 parent_task_id=None,
                 requires=None,
                 properties={},
                 *args,
                 **kwargs):
    """Return takcluster task to trigger a buildbot builder.

    This function creates a generic task with the minimum amount of
    information required for the buildbot-bridge to consider it valid.
    You can establish a list dependencies to other tasks through the requires
    field.

    :param buildername: The name of a buildbot builder.
    :type buildername: str
    :param repo_name: The name of a repository e.g. mozilla-inbound
    :type repo_name: str
    :param revision: Changeset ID of a revision.
    :type revision: str
    :param metadata: Metadata for the task. If not specified, generate it.
    :type metadata: json
    :param task_graph_id: TC graph id to which this task belongs to
    :type task_graph_id: str
    :param parent_task_id: Task from which to find artifacts. It is not a dependency.
    :type parent_task_id: str
    :param requires: List of taskIds of other tasks which this task depends on.
    :type requires: list
    :returns: TaskCluster graph
    :rtype: dict

    """
    if not valid_builder(buildername):
        raise MozciError("The builder '%s' is not a valid one." % buildername)

    builder_info = get_buildername_metadata(buildername)
    if builder_info['repo_name'] != repo_name:
        raise MozciError("The builder '%s' should be for repo: %s." %
                         (buildername, repo_name))

    repo_url = query_repo_url(repo_name)
    push_info = query_push_by_revision(repo_url=repo_url, revision=revision)
    full_revision = str(push_info.changesets[0].node)

    # Needed because of bug 1195751
    all_properties = {
        'product': builder_info['product'],
        'who': push_info.user,
    }
    all_properties.update(properties)

    all_properties.update(get_builder_extra_properties(buildername))

    metadata = metadata if metadata is not None else \
        generate_metadata(repo_name=repo_name,
                          revision=revision,
                          name=buildername)

    # The task's name is used in the task-graph-inspector to list all tasks
    # and using the buildername makes it easy for a person to recognize each job.
    metadata['name'] = buildername

    # XXX: We should validate that the parent task is a valid parent platform
    #      e.g. do not schedule Windows tests against Linux builds
    task = create_task(
        repo_name=repo_name,
        revision=revision,
        taskGroupId=task_graph_id,
        workerType='buildbot-bridge',
        provisionerId='buildbot-bridge',
        payload={
            'buildername': buildername,
            'sourcestamp': {
                'branch': repo_name,
                'revision': full_revision
            },
            'properties': all_properties,
        },
        metadata=metadata,
    )

    if requires:
        task['requires'] = requires

    # Setting a parent_task_id as a property allows Mozharness to
    # determine the artifacts we need for this job to run properly
    if parent_task_id:
        task['task']['payload']['properties'][
            'parent_task_id'] = parent_task_id

    return task