def test_in_memory_cache(self, get):
        """Calling the function with in-memory cache should return without calling request.get."""
        pushlog_client.VALID_CACHE = {("try", "146071751b1e"): True}
        self.assertEquals(pushlog_client.valid_revision("try", "146071751b1e"),
                          True)

        assert get.call_count == 0
    def test_in_memory_cache(self,  get):
        """Calling the function with in-memory cache should return without calling request.get."""
        pushlog_client.VALID_CACHE = {("try", "146071751b1e"): True}
        self.assertEquals(
            pushlog_client.valid_revision("try", "146071751b1e"), True)

        assert get.call_count == 0
    def test_valid_without_any_cache(self, get):
        """Calling the function without in-memory cache."""
        # Making sure the original cache is empty
        pushlog_client.VALID_CACHE = {}
        self.assertEquals(
            pushlog_client.valid_revision("try", "4e030c8cf8c3"), True)

        # The in-memory cache should be filed now
        self.assertEquals(
            pushlog_client.VALID_CACHE, {("try", "4e030c8cf8c3"): True})
    def test_valid_without_any_cache(self, get):
        """Calling the function without in-memory cache."""
        # Making sure the original cache is empty
        pushlog_client.VALID_CACHE = {}
        self.assertEquals(pushlog_client.valid_revision("try", "4e030c8cf8c3"),
                          True)

        # The in-memory cache should be filed now
        self.assertEquals(pushlog_client.VALID_CACHE,
                          {("try", "4e030c8cf8c3"): True})
def trigger_range(buildername, revisions, times=1, dry_run=False,
                  files=None, extra_properties=None, trigger_build_if_missing=True):
    """Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
    repo_name = query_repo_name_from_buildername(buildername)
    repo_url = repositories.query_repo_url(repo_name)

    if revisions != []:
        LOG.info("We want to have %s job(s) of %s on the following revisions: "
                 % (times, buildername))
        for r in revisions:
            LOG.info(" - %s" % r)

    for rev in revisions:
        LOG.info("")
        LOG.info("=== %s ===" % rev)
        if VALIDATE and not valid_revision(repo_url, rev):
            LOG.info("We can't trigger anything on pushes without a valid revision.")
            continue

        LOG.info("We want to have %s job(s) of %s" % (times, buildername))

        # 1) How many potentially completed jobs can we get for this buildername?
        matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev, buildername)
        status_summary = StatusSummary(matching_jobs)

        # TODO: change this debug message when we have a less hardcoded _status_summary
        LOG.debug("We found %d pending/running jobs, %d successful jobs and "
                  "%d failed jobs" % (status_summary.pending_jobs + status_summary.running_jobs,
                                      status_summary.successful_jobs, status_summary.failed_jobs))

        if status_summary.potential_jobs >= times:
            LOG.info("We have %d job(s) for '%s' which is enough for the %d job(s) we want." %
                     (status_summary.potential_jobs, buildername, times))

        else:
            # 2) If we have less potential jobs than 'times' instances then
            #    we need to fill it in.
            LOG.info("We have found %d potential job(s) matching '%s' on %s. "
                     "We need to trigger more." % (status_summary.potential_jobs, buildername, rev))

            # If a job matching what we want already exists, we can
            # use the retrigger API in self-serve to retrigger that
            # instead of creating a new arbitrary job
            if len(matching_jobs) > 0 and files is None:
                request_id = QUERY_SOURCE.get_buildapi_request_id(repo_name, matching_jobs[0])
                make_retrigger_request(
                    repo_name=repo_name,
                    request_id=request_id,
                    auth=get_credentials(),
                    count=(times - status_summary.potential_jobs),
                    dry_run=dry_run)

            # If no matching job exists, we have to trigger a new arbitrary job
            else:
                list_of_requests = trigger_job(
                    revision=rev,
                    buildername=buildername,
                    times=(times - status_summary.potential_jobs),
                    dry_run=dry_run,
                    files=files,
                    extra_properties=extra_properties,
                    trigger_build_if_missing=trigger_build_if_missing)

                if list_of_requests and any(req.status_code != 202 for req in list_of_requests):
                    LOG.warning("Not all requests succeeded.")
def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
                extra_properties=None, trigger_build_if_missing=True):
    """Trigger a job through self-serve.

    We return a list of all requests made.
    """
    repo_name = query_repo_name_from_buildername(buildername)
    builder_to_trigger = None
    list_of_requests = []
    repo_url = repositories.query_repo_url(repo_name)

    if VALIDATE and not valid_revision(repo_url, revision):
        return list_of_requests

    LOG.info("==> We want to trigger '%s' a total of %d time(s)." % (buildername, times))
    LOG.info("")  # Extra line to help visual of logs

    if VALIDATE and not valid_builder(buildername):
        LOG.error("The builder %s requested is invalid" % buildername)
        # XXX How should we exit cleanly?
        exit(-1)

    if files:
        builder_to_trigger = buildername
        _all_urls_reachable(files)
    else:
        builder_to_trigger, package_url, test_url = determine_trigger_objective(
            revision=revision,
            buildername=buildername,
            trigger_build_if_missing=trigger_build_if_missing,
            will_use_buildapi=True
        )

        if builder_to_trigger != buildername and times != 1:
            # The user wants to trigger a downstream job,
            # however, we need a build job instead.
            # We should trigger the downstream job multiple times, however,
            # we only trigger the upstream jobs once.
            LOG.debug("Since we need to trigger a build job we don't need to "
                      "trigger it %s times but only once." % times)
            if trigger_build_if_missing:
                LOG.info("In order to trigger %s %i times, "
                         "please run the script again after %s ends."
                         % (buildername, times, builder_to_trigger))
            else:
                LOG.info("We won't trigger '%s' because there is no working build."
                         % buildername)
                LOG.info("")
            times = 1

    if builder_to_trigger:
        if dry_run:
            LOG.info("Dry-run: We were going to request '%s' %s times." %
                     (builder_to_trigger, times))
            # Running with dry_run being True will only output information
            trigger(
                builder=builder_to_trigger,
                revision=revision,
                files=[package_url, test_url],
                dry_run=dry_run,
                extra_properties=extra_properties
            )
        else:
            for _ in range(times):
                req = trigger(
                    builder=builder_to_trigger,
                    revision=revision,
                    files=[package_url, test_url],
                    dry_run=dry_run,
                    extra_properties=extra_properties
                )
                if req is not None:
                    list_of_requests.append(req)
    else:
        LOG.debug("Nothing needs to be triggered")

    # Cleanup old buildjson files.
    clean_directory()

    return list_of_requests
 def test_invalid(self, get):
     """Calling the function with a bad revision."""
     self.assertEquals(
         pushlog_client.valid_revision("try", "123456123456"), False)
 def test_invalid(self, get):
     """Calling the function with a bad revision."""
     self.assertEquals(pushlog_client.valid_revision("try", "123456123456"),
                       False)
Пример #9
0
def trigger_range(buildername,
                  revisions,
                  times=1,
                  dry_run=False,
                  files=None,
                  extra_properties=None,
                  trigger_build_if_missing=True):
    """Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
    repo_name = query_repo_name_from_buildername(buildername)
    repo_url = repositories.query_repo_url(repo_name)

    if revisions != []:
        LOG.info(
            "We want to have %s job(s) of %s on the following revisions: " %
            (times, buildername))
        for r in revisions:
            LOG.info(" - %s" % r)

    for rev in revisions:
        LOG.info("")
        LOG.info("=== %s ===" % rev)
        if VALIDATE and not valid_revision(repo_url, rev):
            LOG.info(
                "We can't trigger anything on pushes without a valid revision."
            )
            continue

        # 1) How many potentially completed jobs can we get for this buildername?
        matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev,
                                                       buildername)
        status_summary = StatusSummary(matching_jobs)

        # TODO: change this debug message when we have a less hardcoded _status_summary
        LOG.debug("We found %d pending/running jobs, %d successful jobs and "
                  "%d failed jobs" %
                  (status_summary.pending_jobs + status_summary.running_jobs,
                   status_summary.successful_jobs, status_summary.failed_jobs))

        if status_summary.potential_jobs >= times:
            LOG.info(
                "We have %d job(s) for '%s' which is enough for the %d job(s) we want."
                % (status_summary.potential_jobs, buildername, times))

        else:
            # 2) If we have less potential jobs than 'times' instances then
            #    we need to fill it in.
            LOG.info("We have found %d potential job(s) matching '%s' on %s. "
                     "We need to trigger more." %
                     (status_summary.potential_jobs, buildername, rev))

            schedule_new_job = True
            # If a job matching what we want already exists, we can
            # use the retrigger API in self-serve to retrigger that
            # instead of creating a new arbitrary job
            if len(matching_jobs) > 0 and files is None:
                try:
                    request_id = QUERY_SOURCE.get_buildapi_request_id(
                        repo_name, matching_jobs[0])
                    make_retrigger_request(
                        repo_name=repo_name,
                        request_id=request_id,
                        auth=get_credentials(),
                        count=(times - status_summary.potential_jobs),
                        dry_run=dry_run)
                    schedule_new_job = False
                except (IndexError, ConnectionError, ReadTimeout,
                        ValueError) as e:
                    # Logging until we can determine why we get these errors
                    # We should have one of these:
                    # {'requests': [{'request_id': int]}
                    # {'request_id': int}
                    LOG.info(matching_jobs[0])
                    LOG.info(str(e))
                    LOG.warning("We failed to retrigger the job, however, "
                                "we will try to schedule a new one.")

            # If no matching job exists, we have to trigger a new arbitrary job
            if schedule_new_job:
                list_of_requests = trigger_job(
                    revision=rev,
                    buildername=buildername,
                    times=(times - status_summary.potential_jobs),
                    dry_run=dry_run,
                    files=files,
                    extra_properties=extra_properties,
                    trigger_build_if_missing=trigger_build_if_missing)

                if list_of_requests and any(req.status_code != 202
                                            for req in list_of_requests):
                    LOG.warning("Not all requests succeeded.")
Пример #10
0
def trigger_job(revision,
                buildername,
                times=1,
                files=None,
                dry_run=False,
                extra_properties={},
                trigger_build_if_missing=True):
    """Trigger a job through self-serve.

    We return a list of all requests made.
    """
    if not extra_properties:
        extra_properties = {}
    extra_properties.update(get_builder_extra_properties(buildername))

    repo_name = query_repo_name_from_buildername(buildername)
    builder_to_trigger = None
    list_of_requests = []
    repo_url = repositories.query_repo_url(repo_name)
    if len(revision) != 40:
        LOG.warning(
            'We should not be using revisions less than 40 chars ({}).'.format(
                revision))
        push_info = query_push_by_revision(repo_url, revision)
        revision = push_info.changesets[0].node
        assert len(revision) == 40, 'This should have been a 40 char revision.'

    if VALIDATE and not valid_revision(repo_url, revision):
        return list_of_requests

    LOG.info("==> We want to trigger '%s' a total of %d time(s)." %
             (buildername, times))
    LOG.info("")  # Extra line to help visual of logs

    if VALIDATE and not valid_builder(buildername):
        LOG.error("The builder %s requested is invalid" % buildername)
        # XXX How should we exit cleanly?
        exit(-1)

    if files:
        builder_to_trigger = buildername
        _all_urls_reachable(files)
    else:
        builder_to_trigger, package_url, test_url = determine_trigger_objective(
            revision=revision,
            buildername=buildername,
            trigger_build_if_missing=trigger_build_if_missing,
            will_use_buildapi=True)

        if builder_to_trigger != buildername and times != 1:
            # The user wants to trigger a downstream job,
            # however, we need a build job instead.
            # We should trigger the downstream job multiple times, however,
            # we only trigger the upstream jobs once.
            LOG.debug("Since we need to trigger a build job we don't need to "
                      "trigger it %s times but only once." % times)
            if trigger_build_if_missing:
                LOG.info("In order to trigger %s %i times, "
                         "please run the script again after %s ends." %
                         (buildername, times, builder_to_trigger))
            else:
                LOG.info(
                    "We won't trigger '%s' because there is no working build."
                    % buildername)
                LOG.info("")
            times = 1

    if builder_to_trigger:
        if dry_run:
            LOG.info("Dry-run: We were going to request '%s' %s times." %
                     (builder_to_trigger, times))
            # Running with dry_run being True will only output information
            trigger(builder=builder_to_trigger,
                    revision=revision,
                    files=[package_url, test_url],
                    dry_run=dry_run,
                    extra_properties=extra_properties)
        else:
            for _ in range(times):
                req = trigger(builder=builder_to_trigger,
                              revision=revision,
                              files=[package_url, test_url],
                              dry_run=dry_run,
                              extra_properties=extra_properties)
                if req is not None:
                    list_of_requests.append(req)
    else:
        LOG.debug("Nothing needs to be triggered")

    # Cleanup old buildjson files.
    clean_directory()

    return list_of_requests