Ejemplo n.º 1
0
    def test_in_memory_cache(self,  get):
        """Calling the function with in-memory cache should return without calling request.get."""
        pushlog.VALID_CACHE = {("try", "146071751b1e"): True}
        self.assertEquals(
            pushlog.valid_revision("try", "146071751b1e"), True)

        assert get.call_count == 0
Ejemplo n.º 2
0
    def test_valid_without_any_cache(self, get):
        """Calling the function without in-memory cache."""
        # Making sure the original cache is empty
        pushlog.VALID_CACHE = {}
        self.assertEquals(pushlog.valid_revision("try", "4e030c8cf8c3"), True)

        # The in-memory cache should be filed now
        self.assertEquals(pushlog.VALID_CACHE, {("try", "4e030c8cf8c3"): True})
Ejemplo n.º 3
0
    def test_valid_without_any_cache(self, get):
        """Calling the function without in-memory cache."""
        # Making sure the original cache is empty
        pushlog.VALID_CACHE = {}
        self.assertEquals(
            pushlog.valid_revision("try", "4e030c8cf8c3"), True)

        # The in-memory cache should be filed now
        self.assertEquals(
            pushlog.VALID_CACHE, {("try", "4e030c8cf8c3"): True})
Ejemplo n.º 4
0
def query_jobs_schedule(repo_name, revision):
    """ Query Buildapi for jobs """
    repo_url = query_repo_url(repo_name)
    if not pushlog.valid_revision(repo_url, revision):
        raise BuildapiException

    url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
    LOG.debug("About to fetch %s" % url)
    req = requests.get(url, auth=get_credentials())

    # If the revision doesn't exist on buildapi, that means there are
    # no builapi jobs for this revision
    if req.status_code not in [200]:
        return []

    return req.json()
Ejemplo n.º 5
0
def query_jobs_schedule(repo_name, revision):
    """
    Query Buildapi for jobs.

    Raises a BuildapiError if the revision doesn't exist in repo_name.
    """
    repo_url = query_repo_url(repo_name)
    if VALIDATE and not pushlog.valid_revision(repo_url, revision):
        raise BuildapiError

    url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
    LOG.debug("About to fetch %s" % url)
    req = requests.get(url, auth=get_credentials())

    # If the revision doesn't exist on buildapi, that means there are
    # no builapi jobs for this revision
    if req.status_code not in [200]:
        return []

    return req.json()
Ejemplo n.º 6
0
def trigger_range(buildername,
                  revisions,
                  times=1,
                  dry_run=False,
                  files=None,
                  extra_properties=None,
                  trigger_build_if_missing=True):
    """Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
    repo_name = query_repo_name_from_buildername(buildername)
    repo_url = repositories.query_repo_url(repo_name)

    if revisions != []:
        LOG.info("We want to have %s job(s) of %s on revisions %s" %
                 (times, buildername, str(revisions)))

    for rev in revisions:
        LOG.info("")
        LOG.info("=== %s ===" % rev)
        if VALIDATE and not pushlog.valid_revision(repo_url, rev):
            LOG.info(
                "We can't trigger anything on pushes without a valid revision."
            )
            continue

        LOG.info("We want to have %s job(s) of %s on revision %s" %
                 (times, buildername, rev))

        # 1) How many potentially completed jobs can we get for this buildername?
        matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev,
                                                       buildername)
        successful_jobs, pending_jobs, running_jobs, _, failed_jobs = \
            _status_summary(matching_jobs)

        potential_jobs = pending_jobs + running_jobs + successful_jobs + failed_jobs
        # TODO: change this debug message when we have a less hardcoded _status_summary
        LOG.debug("We found %d pending/running jobs, %d successful jobs and "
                  "%d failed jobs" %
                  (pending_jobs + running_jobs, successful_jobs, failed_jobs))

        if potential_jobs >= times:
            LOG.info(
                "We have %d job(s) for '%s' which is enough for the %d job(s) we want."
                % (potential_jobs, buildername, times))

        else:
            # 2) If we have less potential jobs than 'times' instances then
            #    we need to fill it in.
            LOG.info("We have found %d potential job(s) matching '%s' on %s. "
                     "We need to trigger more." %
                     (potential_jobs, buildername, rev))

            # If a job matching what we want already exists, we can
            # use the retrigger API in self-serve to retrigger that
            # instead of creating a new arbitrary job
            if len(matching_jobs) > 0 and files is None:
                request_id = QUERY_SOURCE.get_buildapi_request_id(
                    repo_name, matching_jobs[0])
                make_retrigger_request(repo_name=repo_name,
                                       request_id=request_id,
                                       auth=get_credentials(),
                                       count=(times - potential_jobs),
                                       dry_run=dry_run)

            # If no matching job exists, we have to trigger a new arbitrary job
            else:
                list_of_requests = trigger_job(
                    revision=rev,
                    buildername=buildername,
                    times=(times - potential_jobs),
                    dry_run=dry_run,
                    files=files,
                    extra_properties=extra_properties,
                    trigger_build_if_missing=trigger_build_if_missing)

                if list_of_requests and any(req.status_code != 202
                                            for req in list_of_requests):
                    LOG.warning("Not all requests succeeded.")
Ejemplo n.º 7
0
def trigger_job(revision,
                buildername,
                times=1,
                files=None,
                dry_run=False,
                extra_properties=None,
                trigger_build_if_missing=True):
    """Trigger a job through self-serve.

    We return a list of all requests made.
    """
    repo_name = query_repo_name_from_buildername(buildername)
    builder_to_trigger = None
    list_of_requests = []
    repo_url = repositories.query_repo_url(repo_name)

    if VALIDATE and not pushlog.valid_revision(repo_url, revision):
        return list_of_requests

    LOG.info(
        "===> We want to trigger '%s' on revision '%s' a total of %d time(s)."
        % (buildername, revision, times))
    LOG.info("")  # Extra line to help visual of logs

    if VALIDATE and not valid_builder(buildername):
        LOG.error("The builder %s requested is invalid" % buildername)
        # XXX How should we exit cleanly?
        exit(-1)

    if files:
        builder_to_trigger = buildername
        _all_urls_reachable(files)
    else:
        builder_to_trigger, package_url, test_url = determine_trigger_objective(
            revision=revision,
            buildername=buildername,
            trigger_build_if_missing=trigger_build_if_missing)

        if builder_to_trigger != buildername and times != 1:
            # The user wants to trigger a downstream job,
            # however, we need a build job instead.
            # We should trigger the downstream job multiple times, however,
            # we only trigger the upstream jobs once.
            LOG.debug("Since we need to trigger a build job we don't need to "
                      "trigger it %s times but only once." % times)
            if trigger_build_if_missing:
                LOG.info("In order to trigger %s %i times, "
                         "please run the script again after %s ends." %
                         (buildername, times, builder_to_trigger))
            else:
                LOG.info(
                    "We won't trigger '%s' because there is no working build."
                    % buildername)
                LOG.info("")
            times = 1

    if builder_to_trigger:
        if dry_run:
            LOG.info("Dry-run: We were going to request '%s' %s times." %
                     (builder_to_trigger, times))
            # Running with dry_run being True will only output information
            trigger(builder=builder_to_trigger,
                    revision=revision,
                    files=[package_url, test_url],
                    dry_run=dry_run,
                    extra_properties=extra_properties)
        else:
            for _ in range(times):
                req = trigger(builder=builder_to_trigger,
                              revision=revision,
                              files=[package_url, test_url],
                              dry_run=dry_run,
                              extra_properties=extra_properties)
                if req is not None:
                    list_of_requests.append(req)
    else:
        LOG.debug("Nothing needs to be triggered")

    # Cleanup old buildjson files.
    clean_directory()

    return list_of_requests
Ejemplo n.º 8
0
def trigger_range(
    buildername, revisions, times=1, dry_run=False, files=None, extra_properties=None, trigger_build_if_missing=True
):
    """Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
    repo_name = query_repo_name_from_buildername(buildername)
    repo_url = repositories.query_repo_url(repo_name)

    if revisions != []:
        LOG.info("We want to have %s job(s) of %s on revisions %s" % (times, buildername, str(revisions)))

    for rev in revisions:
        LOG.info("")
        LOG.info("=== %s ===" % rev)
        if VALIDATE and not pushlog.valid_revision(repo_url, rev):
            LOG.info("We can't trigger anything on pushes without a valid revision.")
            continue

        LOG.info("We want to have %s job(s) of %s on revision %s" % (times, buildername, rev))

        # 1) How many potentially completed jobs can we get for this buildername?
        matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev, buildername)
        successful_jobs, pending_jobs, running_jobs, _, failed_jobs = _status_summary(matching_jobs)

        potential_jobs = pending_jobs + running_jobs + successful_jobs + failed_jobs
        # TODO: change this debug message when we have a less hardcoded _status_summary
        LOG.debug(
            "We found %d pending/running jobs, %d successful jobs and "
            "%d failed jobs" % (pending_jobs + running_jobs, successful_jobs, failed_jobs)
        )

        if potential_jobs >= times:
            LOG.info(
                "We have %d job(s) for '%s' which is enough for the %d job(s) we want."
                % (potential_jobs, buildername, times)
            )

        else:
            # 2) If we have less potential jobs than 'times' instances then
            #    we need to fill it in.
            LOG.info(
                "We have found %d potential job(s) matching '%s' on %s. "
                "We need to trigger more." % (potential_jobs, buildername, rev)
            )

            # If a job matching what we want already exists, we can
            # use the retrigger API in self-serve to retrigger that
            # instead of creating a new arbitrary job
            if len(matching_jobs) > 0 and files is None:
                request_id = QUERY_SOURCE.get_buildapi_request_id(repo_name, matching_jobs[0])
                make_retrigger_request(repo_name, request_id, count=(times - potential_jobs), dry_run=dry_run)

            # If no matching job exists, we have to trigger a new arbitrary job
            else:
                list_of_requests = trigger_job(
                    revision=rev,
                    buildername=buildername,
                    times=(times - potential_jobs),
                    dry_run=dry_run,
                    files=files,
                    extra_properties=extra_properties,
                    trigger_build_if_missing=trigger_build_if_missing,
                )

                if list_of_requests and any(req.status_code != 202 for req in list_of_requests):
                    LOG.warning("Not all requests succeeded.")
Ejemplo n.º 9
0
def trigger_job(
    revision, buildername, times=1, files=None, dry_run=False, extra_properties=None, trigger_build_if_missing=True
):
    """Trigger a job through self-serve.

    We return a list of all requests made.
    """
    repo_name = query_repo_name_from_buildername(buildername)
    builder_to_trigger = None
    list_of_requests = []
    repo_url = repositories.query_repo_url(repo_name)

    if VALIDATE and not pushlog.valid_revision(repo_url, revision):
        return list_of_requests

    LOG.info("===> We want to trigger '%s' on revision '%s' a total of %d time(s)." % (buildername, revision, times))
    LOG.info("")  # Extra line to help visual of logs

    if VALIDATE and not valid_builder(buildername):
        LOG.error("The builder %s requested is invalid" % buildername)
        # XXX How should we exit cleanly?
        exit(-1)

    if files:
        builder_to_trigger = buildername
        _all_urls_reachable(files)
    else:
        builder_to_trigger, files = determine_trigger_objective(
            revision=revision, buildername=buildername, trigger_build_if_missing=trigger_build_if_missing
        )

        if builder_to_trigger != buildername and times != 1:
            # The user wants to trigger a downstream job,
            # however, we need a build job instead.
            # We should trigger the downstream job multiple times, however,
            # we only trigger the upstream jobs once.
            LOG.debug(
                "Since we need to trigger a build job we don't need to " "trigger it %s times but only once." % times
            )
            if trigger_build_if_missing:
                LOG.info(
                    "In order to trigger %s %i times, "
                    "please run the script again after %s ends." % (buildername, times, builder_to_trigger)
                )
            else:
                LOG.info("We won't trigger '%s' because there is no working build." % buildername)
                LOG.info("")
            times = 1

    if builder_to_trigger:
        if dry_run:
            LOG.info("Dry-run: We were going to request '%s' %s times." % (builder_to_trigger, times))
            # Running with dry_run being True will only output information
            trigger(builder_to_trigger, revision, files, dry_run, extra_properties)
        else:
            for _ in range(times):
                req = trigger(builder_to_trigger, revision, files, dry_run, extra_properties)
                if req is not None:
                    list_of_requests.append(req)
    else:
        LOG.debug("Nothing needs to be triggered")

    # Cleanup old buildjson files.
    clean_directory()

    return list_of_requests
Ejemplo n.º 10
0
 def test_invalid(self, get):
     """Calling the function with a bad revision."""
     self.assertEquals(
         pushlog.valid_revision("try", "123456123456"), False)