Ejemplo n.º 1
0
def buildbot_graph_builder(builders, revision, complete=True):
    """ Return graph of builders based on a list of builders.

    # XXX: It would be better if had a BuildbotGraph class instead of messing
           with dictionaries.
           https://github.com/mozilla/mozilla_ci_tools/issues/353

    Input: a list of builders and a revision
    Output: a set which includes a graph with the builders we received, the necessary upstream
            jobs and list of builders which can been used to schedule jobs through trigger_job()

    NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via
          the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling.

    Graph of N levels:
        {
           'Builder a1': {
               'Builder a2': {
                   ...
                       'Builder aN': None
               },
           },
           'Builder b1': None
        }

    :param builders: List of builder names
    :type builders: list
    :param revision: push revision
    :type revision: str
    :return: A graph of buildernames (single level of graphs)
    :rtype: dict
    :param complete: indicate that build has been completed or not
    :type: booleans

    """
    graph = {}
    ready_to_trigger = []

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for b in builders:
        if not valid_builder(buildername=b, quiet=True):
            continue

        if is_downstream(b):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective = determine_trigger_objective(revision, b)[0]

            # The build job is already completed, we can trigger the test job
            if objective == b:
                # XXX: Fix me - Adding test jobs to the graph without a build associated
                # to it does not work. This will be fixed once we switch to scheduling
                # Buildbot jobs through BBB
                if complete:
                    graph[b] = None
                else:
                    ready_to_trigger.append(b)

            # The build job is running, there is nothing we can do
            elif objective is None:
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in graph:
                    graph[objective] = {}
                graph[objective][b] = None
        else:
            if b not in graph:
                graph[b] = {}

    # We might have left a build job poiting to an empty dict
    for builder in graph:
        if graph[builder] == {}:
            graph[builder] = None

    return graph, ready_to_trigger
Ejemplo n.º 2
0
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
    """ Return TC tasks based on a list of builders.

    Input: a list of builders and a revision
    Output: list of TC tasks base on builders we receive

    :param builders: List of builder names
    :type builders: list
    :param repo_name: push revision
    :type repo_name: str
    :param revision: push revision
    :type revision: str
    :return: TC tasks
    :rtype: dict

    """
    tasks = []
    build_builders = {}

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for builder in builders:
        if is_upstream(builder):
            properties = {'upload_to_task_id': slugId()}

            # Bug 1274483 - Android multi-locale nightly builds need to upload to two different
            # tasks, thus, it fails when we tell it to upload to the same task twice.
            builder_details = get_buildername_metadata(builder)
            if builder_details['platform_name'].startswith('android') and \
               builder_details['nightly'] is True and \
               'l10n' not in builder:
                properties = {}

            task = _create_task(
                buildername=builder,
                repo_name=repo_name,
                revision=revision,
                # task_graph_id=task_graph_id,
                properties=properties,
            )
            tasks.append(task)

            # We want to keep track of how many build builders we have
            build_builders[builder] = task

    for builder in builders:
        if is_downstream(builder):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective, package_url, tests_url = \
                determine_trigger_objective(revision, builder)

            # The build job is already completed, we can trigger the test job
            if objective == builder:
                if objective in build_builders:
                    LOG.warning("We're creating a new build even though there's "
                                "already an existing completed build we could have "
                                "used. We hope you wanted to do this.")
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        parent_task_id=build_builders[objective]['taskId'],
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                else:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        properties={
                            'packageUrl': package_url,
                            'testUrl': tests_url
                        },
                    )
                    tasks.append(task)

            # The build job is running, there is nothing we can do
            elif objective is None:
                LOG.warning("We can add %s builder since the build associated "
                            "is running. This is because it is a Buildbot job.")
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in build_builders:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                    taskId = task['taskId']
                else:
                    taskId = build_builders[objective]['taskId']

                # Add test job
                task = _create_task(
                    buildername=builder,
                    repo_name=repo_name,
                    revision=revision,
                    # task_graph_id=task_graph_id,
                    parent_task_id=taskId,
                )
                tasks.append(task)

    return tasks
Ejemplo n.º 3
0
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
    """ Return TC tasks based on a list of builders.

    Input: a list of builders and a revision
    Output: list of TC tasks base on builders we receive

    :param builders: List of builder names
    :type builders: list
    :param repo_name: push revision
    :type repo_name: str
    :param revision: push revision
    :type revision: str
    :return: TC tasks
    :rtype: dict

    """
    tasks = []
    build_builders = {}

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for builder in builders:
        if is_upstream(builder):
            task = _create_task(
                buildername=builder,
                repo_name=repo_name,
                revision=revision,
                # task_graph_id=task_graph_id,
                properties={'upload_to_task_id': slugId()},
            )
            tasks.append(task)

            # We want to keep track of how many build builders we have
            build_builders[builder] = task

    for builder in builders:
        if is_downstream(builder):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective, package_url, tests_url = \
                determine_trigger_objective(revision, builder)

            # The build job is already completed, we can trigger the test job
            if objective == builder:
                if objective in build_builders:
                    LOG.warning(
                        "We're creating a new build even though there's "
                        "already an existing completed build we could have "
                        "used. We hope you wanted to do this.")
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        parent_task_id=build_builders[objective]['taskId'],
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                else:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        properties={
                            'packageUrl': package_url,
                            'testUrl': tests_url
                        },
                    )
                    tasks.append(task)

            # The build job is running, there is nothing we can do
            elif objective is None:
                LOG.warning(
                    "We can add %s builder since the build associated "
                    "is running. This is because it is a Buildbot job.")
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in build_builders:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        properties={'upload_to_task_id': slugId()},
                    )
                    tasks.append(task)
                    taskId = task['taskId']
                else:
                    taskId = build_builders[objective]['taskId']

                # Add test job
                task = _create_task(
                    buildername=builder,
                    repo_name=repo_name,
                    revision=revision,
                    # task_graph_id=task_graph_id,
                    parent_task_id=taskId,
                )
                tasks.append(task)

    return tasks
Ejemplo n.º 4
0
def buildbot_graph_builder(builders, revision, complete=True):
    """ Return graph of builders based on a list of builders.

    # XXX: It would be better if had a BuildbotGraph class instead of messing
           with dictionaries.
           https://github.com/mozilla/mozilla_ci_tools/issues/353

    Input: a list of builders and a revision
    Output: a graph with the builders we received and the necessary upstream jobs

    Graph of N levels:
        {
           'Builder a1': {
               'Builder a2': {
                   ...
                       'Builder aN': None
               },
           },
           'Builder b1': None
        }

    :param builders: List of builder names
    :type builders: list
    :param revision: push revision
    :type revision: str
    :return: A graph of buildernames (single level of graphs)
    :rtype: dict

    """
    graph = {}
    ready_to_trigger = []

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for b in builders:
        if is_downstream(b):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective = determine_trigger_objective(revision, b)[0]

            # The build job is already completed, we can trigger the test job
            if objective == b:
                # If complete is True, we add the job to our graph
                if complete:
                    graph[b] = None
                else:
                    ready_to_trigger.append(b)

            # The build job is running, there is nothing we can do
            elif objective is None:
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in graph:
                    graph[objective] = {}
                graph[objective][b] = None
        else:
            if b not in graph:
                graph[b] = {}

    # We might have left a build job poiting to an empty dict
    for builder in graph:
        if graph[builder] == {}:
            graph[builder] = None

    return graph, ready_to_trigger
def buildbot_graph_builder(builders, revision):
    """ Return graph of builders based on a list of builders.

    # XXX: It would be better if had a BuildbotGraph class instead of messing
           with dictionaries.
           https://github.com/mozilla/mozilla_ci_tools/issues/353

    Input: a list of builders and a revision
    Output: a graph with the builders we received and the necessary upstream jobs

    Graph of N levels:
        {
           'Builder a1': {
               'Builder a2': {
                   ...
                       'Builder aN': None
               },
           },
           'Builder b1': None
        }

    :param builders: List of builder names
    :type builders: list
    :param revision: push revision
    :type revision: str
    :return: A graph of buildernames (single level of graphs)
    :rtype: dict

    """
    graph = {}

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for b in builders:
        if is_downstream(b):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective = determine_trigger_objective(revision, b)[0]

            # The build job is already completed, we can trigger the test job
            if objective == b:
                graph[b] = None

            # The build job is running, there is nothing we can do
            elif objective is None:
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in graph:
                    graph[objective] = {}
                graph[objective][b] = None
        else:
            if b not in graph:
                graph[b] = {}

    # We might have left a build job poiting to an empty dict
    for builder in graph:
        if graph[builder] == {}:
            graph[builder] = None

    return graph
Ejemplo n.º 6
0
def _generate_tc_tasks_from_builders(builders, repo_name, revision):
    """ Return TC tasks based on a list of builders.

    Input: a list of builders and a revision
    Output: list of TC tasks base on builders we receive

    :param builders: List of builder names
    :type builders: list
    :param repo_name: push revision
    :type repo_name: str
    :param revision: push revision
    :type revision: str
    :return: TC tasks
    :rtype: dict

    """
    tasks = []
    build_builders = {}

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for builder in builders:
        if is_upstream(builder):
            task = _create_task(
                buildername=builder,
                repo_name=repo_name,
                revision=revision,
                # task_graph_id=task_graph_id,
                properties={"upload_to_task_id": slugId()},
            )
            tasks.append(task)

            # We want to keep track of how many build builders we have
            build_builders[builder] = task

    for builder in builders:
        if is_downstream(builder):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective, package_url, tests_url = determine_trigger_objective(revision, builder)

            # The build job is already completed, we can trigger the test job
            if objective == builder:
                if objective in build_builders:
                    LOG.warning(
                        "We're creating a new build even though there's "
                        "already an existing completed build we could have "
                        "used. We hope you wanted to do this."
                    )
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        parent_task_id=build_builders[objective]["taskId"],
                        properties={"upload_to_task_id": slugId()},
                    )
                    tasks.append(task)
                else:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        properties={"packageUrl": package_url, "testUrl": tests_url},
                    )
                    tasks.append(task)

            # The build job is running, there is nothing we can do
            elif objective is None:
                LOG.warning(
                    "We can add %s builder since the build associated "
                    "is running. This is because it is a Buildbot job."
                )
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in build_builders:
                    task = _create_task(
                        buildername=builder,
                        repo_name=repo_name,
                        revision=revision,
                        # task_graph_id=task_graph_id,
                        properties={"upload_to_task_id": slugId()},
                    )
                    tasks.append(task)
                    taskId = task["taskId"]
                else:
                    taskId = build_builders[objective]["taskId"]

                # Add test job
                task = _create_task(
                    buildername=builder,
                    repo_name=repo_name,
                    revision=revision,
                    # task_graph_id=task_graph_id,
                    parent_task_id=taskId,
                )
                tasks.append(task)

    return tasks
Ejemplo n.º 7
0
def buildbot_graph_builder(builders, revision, complete=True):
    """ Return graph of builders based on a list of builders.

    # XXX: It would be better if had a BuildbotGraph class instead of messing
           with dictionaries.
           https://github.com/mozilla/mozilla_ci_tools/issues/353

    Input: a list of builders and a revision
    Output: a set which includes a graph with the builders we received, the necessary upstream
            jobs and list of builders which can been used to schedule jobs through trigger_job()

    NOTE: We will make it only return builder graph once Buildbot jobs are scheduled via
          the Buildbot Bridge (BBB) instead of the normal Buildbot scheduling.

    Graph of N levels:
        {
           'Builder a1': {
               'Builder a2': {
                   ...
                       'Builder aN': None
               },
           },
           'Builder b1': None
        }

    :param builders: List of builder names
    :type builders: list
    :param revision: push revision
    :type revision: str
    :return: A graph of buildernames (single level of graphs)
    :rtype: dict
    :param complete: indicate that build has been completed or not
    :type: booleans

    """
    graph = {}
    ready_to_trigger = []

    # We need to determine what upstream jobs need to be triggered besides the
    # builders already on our list
    for b in builders:
        if not valid_builder(buildername=b, quiet=True):
            continue

        if is_downstream(b):
            # For test jobs, determine_trigger_objective()[0] can be 3 things:
            # - the build job, if no build job exists
            # - the test job, if the build job is already completed
            # - None, if the build job is running
            objective = determine_trigger_objective(revision, b)[0]

            # The build job is already completed, we can trigger the test job
            if objective == b:
                # XXX: Fix me - Adding test jobs to the graph without a build associated
                # to it does not work. This will be fixed once we switch to scheduling
                # Buildbot jobs through BBB
                if complete:
                    graph[b] = None
                else:
                    ready_to_trigger.append(b)

            # The build job is running, there is nothing we can do
            elif objective is None:
                pass

            # We need to trigger the build job and the test job
            else:
                if objective not in graph:
                    graph[objective] = {}
                graph[objective][b] = None
        else:
            if b not in graph:
                graph[b] = {}

    # We might have left a build job poiting to an empty dict
    for builder in graph:
        if graph[builder] == {}:
            graph[builder] = None

    return graph, ready_to_trigger