Exemple #1
0
    def __init__(self, config):
        """
        like all the the other inits around this code, the init will gather
        relevant information for this class and put them into local shortcuts
        :param config: The global promoter config or the reduced dlrnclient
        config
        """
        self.config = config
        # TODO(gcerami): fix credentials gathering
        dlrnapi_client.configuration.password = self.config.dlrnauth_password
        dlrnapi_client.configuration.username = self.config.dlrnauth_username
        api_client = dlrnapi_client.ApiClient(host=self.config.api_url)
        self.api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
        self.last_promotions = {}

        # Variable to detect changes on the hash while we are running a
        # promotion
        self.named_hashes_map = {}

        # This way of preparing parameters and configuration is copied
        # directly from dlrnapi CLI and ansible module
        self.hashes_params = dlrnapi_client.PromotionQuery()
        self.jobs_params = dlrnapi_client.Params2()
        self.jobs_params_aggregate = dlrnapi_client.Params3()
        self.report_params = dlrnapi_client.Params3()
        self.promote_params = dlrnapi_client.Promotion()
        self.log.debug("Promoter DLRN client: API URL: {}, user: {}"
                       "".format(api_client.host,
                                 self.config.dlrnauth_username))
Exemple #2
0
def get_dlrn_instance(config=None):

    if config is None:
        return None

    api_client = dlrnapi_client.ApiClient(host=config.get('main', 'api_url'))
    return dlrnapi_client.DefaultApi(api_client=api_client)
Exemple #3
0
def check_dlrn_promoted_hash(stage_info):
    ''' Check that the commit, distro hash has been promoted to
        promotion_target as recorded in DLRN. '''

    dlrn_host = stage_info['dlrn_host']
    promotion_target = stage_info['promotion_target']
    commit_hash = stage_info['promotions']['promotion_candidate'][
        'commit_hash']
    distro_hash = stage_info['promotions']['promotion_candidate'][
        'distro_hash']

    logger = logging.getLogger('TestPromoter')
    api_client = dlrnapi_client.ApiClient(host=dlrn_host)
    dlrn = dlrnapi_client.DefaultApi(api_client=api_client)
    params = dlrnapi_client.PromotionQuery()
    params.commit_hash = commit_hash
    params.distro_hash = distro_hash
    try:
        api_response = dlrn.api_promotions_get(params)
        logger.debug(api_response)
    except dlrnapi_client.rest.ApiException:
        logger.error('Exception when calling api_promotions_get: %s',
                     dlrnapi_client.rest.ApiException)
        raise

    error_message = ("Expected commit hash: {}"
                     " has not been promoted to {}."
                     "".format(commit_hash, promotion_target))
    conditions = [(promotion.promote_name == promotion_target)
                  for promotion in api_response]
    assert any(conditions), error_message
Exemple #4
0
def main():
    module = AnsibleModule(argument_spec=dict(
        action=dict(required=True,
                    choices=[
                        'repo-get', 'repo-use', 'repo-status', 'report-result',
                        'repo-promote', 'commit-import', 'promotion-get',
                        'build-metrics'
                    ]),
        host=dict(required=True),
        user=dict(),
        password=dict(no_log=True),
        reporting_job_id=dict(),
        max_age=dict(default='0', type='int'),
        success=dict(type='bool'),
        job_id=dict(),
        sequential_mode=dict(type='bool', default=False),
        previous_job_id=dict(),
        commit_hash=dict(),
        distro_hash=dict(),
        info_url=dict(),
        timestamp=dict(),
        notes=dict(),
        promote_name=dict(),
        repo_url=dict(),
        start_date=dict(),
        end_date=dict(),
        package_name=dict(),
    ))

    action = module.params['action']
    username = module.params['user']
    password = module.params['password']
    if action in auth_required_actions and (username is None
                                            or password is None):
        module.fail_json(msg="Action %s requires authentication" % action)

    api_client = dlrnapi_client.ApiClient(host=module.params['host'])
    dlrnapi_client.configuration.username = username
    dlrnapi_client.configuration.password = password
    api_instance = dlrnapi_client.DefaultApi(api_client=api_client)

    options = DLRNAPIWrapper(module.params)
    options.check_options(action, module)

    try:
        output = command_funcs[action](api_instance, options)
        if type(output) == list:
            output_f = [x.to_dict() for x in output]
        else:
            output_f = output.to_dict()
        module.exit_json(changed=True, result=output_f)
    except ApiException as e:
        module.fail_json(msg="Exception when calling "
                         "%s: %s\n" % (command_funcs[action], e))
def promoter(config):
    logger = logging.getLogger('promoter')

    release = config.get('main', 'release')
    get_lock('promoter')

    logger.info('STARTED promotion process for release: %s', release)

    try:
        git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
                                           cwd=os.path.abspath(sys.path[0]))
        logger.info(
            'Current git hash of repo containing the promoter '
            'script: %s', git_hash.strip())
    except OSError:
        logger.debug('Failed to get the current git repo hash, check if '
                     'git is installed.')
    except subprocess.CalledProcessError:
        logger.debug('Failed to get the current git repo hash, probably not '
                     'running inside a git repository.')

    # setup the API connection
    dry_run = config.getboolean('main', 'dry_run')
    api_client = dlrnapi_client.ApiClient(host=config.get('main', 'api_url'))
    dlrnapi_client.configuration.username = config.get('main', 'username')
    if os.getenv('DLRNAPI_PASSWORD', None) is None:
        logger.warning('DLRNAPI_PASSWORD env variable is missing or empty, '
                       'promotion attempt will fail!')
    dlrnapi_client.configuration.password = os.getenv('DLRNAPI_PASSWORD', None)
    api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
    latest_hashes_count = config.getint('main', 'latest_hashes_count')
    config.remove_section('main')
    logger.info('Using API URL: %s', api_client.host)

    # load the promote_from data
    promote_from = {k: v for k, v in config.items('promote_from')}
    logger.debug('Attempting to promote these DLRN links: %s', promote_from)
    config.remove_section('promote_from')

    # load the promotion requirements
    job_reqs = {}
    sections = config.sections()
    for section in sections:
        job_reqs[section] = [k for k, v in config.items(section)]
    logger.debug('Promotion requirements loaded: %s', job_reqs)

    promote_all_links(api_instance, promote_from, job_reqs, dry_run, release,
                      latest_hashes_count)
    logger.info("FINISHED promotion process")
def check_dlrn_promoted_hash(stage_info=None, **kwargs):
    """
    Check that the the supposed hash has been promoted to
    promotion_target as recorded in DLRN.
    :param stage_info: a dictionary containing parameter of the staging env
    :param kwargs: additional parameter for non-staged executions
    :return: None
    """
    if stage_info is not None:
        # We are checking a stage
        api_url = stage_info['dlrn']['server']['api_url']
        promotion_target = stage_info['dlrn']['promotion_target']
        candidate_commit = \
            stage_info['dlrn']['promotions']['promotion_candidate']
        candidate_hash = DlrnHash(source=candidate_commit)

        api_client = dlrnapi_client.ApiClient(host=api_url)
        dlrn_client = dlrnapi_client.DefaultApi(api_client=api_client)
        params = dlrnapi_client.PromotionQuery()
        params.limit = 1
        params.promote_name = promotion_target
    else:
        # We are checking production server
        # TODO(gcerami) implement this branch ?
        pass

    try:
        api_response = dlrn_client.api_promotions_get(params)
        log.debug(api_response)
    except dlrnapi_client.rest.ApiException:
        log.error('Exception when calling api_promotions_get: %s',
                  dlrnapi_client.rest.ApiException)
        raise

    error_msg = "No promotions for hash {}".format(candidate_hash)
    assert api_response != [], error_msg
    promotion_hash = DlrnHash(source=api_response[0])
    error_message = ("Expected full hash: {}"
                     " has not been promoted to {}."
                     "".format(promotion_hash.full_hash, promotion_target))
    conditions = [(promotion.promote_name == promotion_target)
                  for promotion in api_response]
    assert any(conditions), error_message
Exemple #7
0
def get_promotions(release, promote_name):

    host = get_endpoint(release)

    api_client = dlrnapi_client.ApiClient(host=host)
    api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
    params = dlrnapi_client.PromotionQuery()

    if promote_name:
        params.promote_name = promote_name

    try:
        api_response = api_instance.api_promotions_get(params)

    except ApiException as e:
        print("Exception when calling DefaultApi->api_promotions_get: %s\n" %
              e)

    return api_response
Exemple #8
0
def main():
    parser = argparse.ArgumentParser(prog='dlrnapi')

    parser.add_argument('--url', required=True, help='URL to use')
    parser.add_argument('--username',
                        '-u',
                        help='username for authentication, defaults to '
                        '"DLRNAPI_USERNAME" environment variable if set',
                        default=os.getenv('DLRNAPI_USERNAME', None))
    parser.add_argument('--password',
                        '-p',
                        help='password for authentication, defaults to '
                        '"DLRNAPI_PASSWORD" environment variable is set',
                        default=os.getenv('DLRNAPI_PASSWORD', None))

    subparsers = parser.add_subparsers(dest='command',
                                       title='subcommands',
                                       description='available subcommands')
    # Subcommand get-repo
    parser_last = subparsers.add_parser('repo-get',
                                        help='Get last tested repo')
    parser_last.add_argument('--max-age', type=int, default=0, help='max_age')
    parser_last.add_argument('--success',
                             type=str,
                             default=None,
                             help='Find repos with a successful/unsuccessful '
                             'vote, if true or false are specified')
    parser_last.add_argument('--job-id',
                             type=str,
                             default=None,
                             help='Name of the CI that sent the vote. If not '
                             'set, no filter will be set on CI')
    parser_last.add_argument('--sequential-mode',
                             dest='sequential',
                             action='store_true',
                             help='Use the sequential mode algorithm. In this '
                             'case, return the last tested repo within '
                             'that timeframe for the CI job described by '
                             '--previous-job-id')
    parser_last.set_defaults(sequential=False)

    parser_last.add_argument('--previous-job-id',
                             type=str,
                             default=None,
                             help='If --sequential-mode is set, look for jobs'
                             ' tested by this CI')

    # Subcommand use-repo
    parser_use_last = subparsers.add_parser('repo-use',
                                            help='Get the last tested repo '
                                            'since a specific time '
                                            '(optionally for a CI job), '
                                            'and add an "in progress" '
                                            'entry in the CI job table '
                                            'for this.')
    parser_use_last.add_argument('--max-age',
                                 type=int,
                                 default=0,
                                 help='max_age')
    parser_use_last.add_argument('--reporting-job-id',
                                 type=str,
                                 required=True,
                                 help=' Name of the CI that will add the "in '
                                 'progress" entry in the CI job table.')
    parser_use_last.add_argument('--success',
                                 type=str,
                                 default=None,
                                 help='Find repos with a successful/'
                                 'unsuccessful vote, if true or false '
                                 'are specified')
    parser_use_last.add_argument('--job-id',
                                 type=str,
                                 default=None,
                                 help='Name of the CI that sent the vote. If '
                                 'not set, no filter will be set on CI')
    parser_use_last.add_argument('--sequential-mode',
                                 dest='sequential',
                                 action='store_true',
                                 help='Use the sequential mode algorithm. In '
                                 'this case, return the last tested repo '
                                 'within that timeframe for the CI job '
                                 'described by --previous-job-id')
    parser_use_last.set_defaults(sequential=False)
    parser_use_last.add_argument('--previous-job-id',
                                 type=str,
                                 default=None,
                                 help='If --sequential-mode is true, look for '
                                 'jobs tested by this CI')

    # Subcommand repo-status
    parser_st = subparsers.add_parser('repo-status',
                                      help='Get all the CI reports for a '
                                      'specific repository.')
    parser_st.add_argument('--commit-hash',
                           type=str,
                           required=True,
                           help='commit_hash of the repo to fetch '
                           'information for.')
    parser_st.add_argument('--distro-hash',
                           type=str,
                           required=True,
                           help='distro_hash of the repo to fetch '
                           'information for.')
    parser_st.add_argument('--success',
                           type=str,
                           default=None,
                           help='If set to a value (true/false), only return '
                           'the CI reports with the specified vote. If '
                           'not set, return all CI reports.')

    # Subcommand report-result
    parser_rep = subparsers.add_parser('report-result',
                                       help='Report the result of a CI job')
    parser_rep.add_argument('--job-id',
                            type=str,
                            required=True,
                            help='Name of the CI sending the vote')
    parser_rep.add_argument('--commit-hash',
                            type=str,
                            required=True,
                            help='commit_hash of tested repo')
    parser_rep.add_argument('--distro-hash',
                            type=str,
                            required=True,
                            help='distro_hash of tested repo')
    parser_rep.add_argument('--info-url',
                            type=str,
                            required=True,
                            help='URL where to find additional information '
                            'from the CI execution')
    parser_rep.add_argument('--timestamp',
                            type=str,
                            required=True,
                            help='Timestamp (in seconds since the epoch)')
    parser_rep.add_argument('--success',
                            type=str,
                            required=True,
                            help='Was the CI execution successful? Set to '
                            'true or false.')
    parser_rep.add_argument('--notes', type=str, help='Additional notes')

    # Subcommand promote
    parser_prom = subparsers.add_parser('repo-promote',
                                        help='Promote a repository')
    parser_prom.add_argument('--commit-hash',
                             type=str,
                             required=True,
                             help='commit_hash of the repo to be promoted')
    parser_prom.add_argument('--distro-hash',
                             type=str,
                             required=True,
                             help='distro_hash of the repo to be promoted')
    parser_prom.add_argument('--promote-name',
                             type=str,
                             required=True,
                             help='Name to be used for the promotion')

    # Subcommand promotion-get
    parser_promget = subparsers.add_parser('promotion-get',
                                           help='Get information about '
                                           'promotions')
    parser_promget.add_argument('--commit-hash',
                                type=str,
                                required=False,
                                help='commit_hash of the repo to search '
                                'promotions for. Requires --distro-hash '
                                'if specified.')
    parser_promget.add_argument('--distro-hash',
                                type=str,
                                required=False,
                                help='distro_hash of the repo to search '
                                'promotions for. Requires --commit-hash '
                                'if specified.')
    parser_promget.add_argument('--promote-name',
                                type=str,
                                required=False,
                                help='Filter results for this promotion name.')
    parser_promget.add_argument('--offset',
                                type=int,
                                required=False,
                                help='Show results after this offset. Each '
                                'query will only return 100 entries by '
                                'default.')
    parser_promget.add_argument('--limit',
                                type=int,
                                required=False,
                                help='Limit the results to the first limit '
                                'items')

    # Subcommand commit-import
    parser_imp = subparsers.add_parser('commit-import',
                                       help='Import a commit built by another'
                                       ' instance')
    parser_imp.add_argument('--repo-url',
                            type=str,
                            required=True,
                            help='Base repository URL for the remote repo '
                            'to import')

    # Subcommand build-metrics
    parser_metrics = subparsers.add_parser(
        'build-metrics', help='Fetch build metrics in a time period')
    parser_metrics.add_argument(
        '--start-date',
        type=str,
        required=True,
        help='Start date for the query, in YYYY-MM-DD format')
    parser_metrics.add_argument(
        '--end-date',
        type=str,
        required=True,
        help='End date for the query, in YYYY-MM-DD format')
    parser_metrics.add_argument(
        '--package-name',
        type=str,
        required=False,
        help='If specified, only fetch metrics for this package name')

    options, args = parser.parse_known_args(sys.argv[1:])

    # create an instance of the API class
    api_client = dlrnapi_client.ApiClient(host=options.url)
    dlrnapi_client.configuration.username = options.username
    dlrnapi_client.configuration.password = options.password
    api_instance = dlrnapi_client.DefaultApi(api_client=api_client)

    try:
        api_response = command_funcs[options.command](api_instance, options)
        print(
            json.dumps(api_response,
                       cls=ResponseEncoder,
                       indent=2,
                       sort_keys=True))
    except ApiException as e:
        # Handle 404 exceptions gracefully
        if e.status == 404:
            print(
                "ERROR: Got error 404, probably endpoint %s is not available" %
                options.url)
            return 1
        else:
            raise
    except Exception as e:
        raise e
def promoter(config):
    logger = logging.getLogger('promoter')

    distro = (config.get('main', 'distro_name').lower(),
              config.get('main', 'distro_version'))

    release = config.get('main', 'release')
    api_url = config.get('main', 'api_url')

    # No boolean values here, as these are passed directly to ansible-playbook
    # command line
    manifest_push = config.get('main', 'manifest_push', fallback="false")
    target_registries_push = config.get('main', 'target_registries_push',
                                        fallback="true")

    logger.info('STARTED promotion process for release: %s', release)

    try:
        git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
                                           cwd=os.path.abspath(sys.path[0]))
        logger.info('Current git hash of repo containing the promoter '
                    'script: %s', git_hash.strip())
    except OSError:
        logger.debug('Failed to get the current git repo hash, check if '
                     'git is installed.')
    except subprocess.CalledProcessError:
        logger.debug('Failed to get the current git repo hash, probably not '
                     'running inside a git repository.')

    # setup the API connection
    dry_run = config.getboolean('main', 'dry_run')
    api_client = dlrnapi_client.ApiClient(host=config.get('main', 'api_url'))
    dlrnapi_client.configuration.username = config.get('main', 'username')
    if os.getenv('DLRNAPI_PASSWORD', None) is None:
        logger.warning('DLRNAPI_PASSWORD env variable is missing or empty, '
                       'promotion attempt will fail!')
    dlrnapi_client.configuration.password = os.getenv('DLRNAPI_PASSWORD', None)
    api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
    latest_hashes_count = config.getint('main', 'latest_hashes_count')
    config.remove_section('main')
    logger.info('Using API URL: %s', api_client.host)

    # load the promote_from data
    promote_from = {k: v for k, v in config.items('promote_from')}
    logger.debug('Attempting to promote these DLRN links: %s',
                 promote_from)
    config.remove_section('promote_from')

    # load the promotion requirements
    job_reqs = {}
    sections = config.sections()
    for section in sections:
        job_reqs[section] = [k for k, v in config.items(section)]
    logger.debug('Promotion requirements loaded: %s', job_reqs)

    global start_named_hashes
    start_named_hashes = fetch_current_named_hashes(
                                           release, promote_from, api_instance)
    logger.info('Named hashes at start of promotion process: %s',
                start_named_hashes)

    promote_all_links(
        api_instance,
        promote_from,
        job_reqs,
        dry_run,
        distro,
        release,
        latest_hashes_count,
        api_url,
        manifest_push,
        target_registries_push)
    logger.info("FINISHED promotion process")
Exemple #10
0
    for status in api_response:
        if wait_job is None and status.job_id == wait_job_name:
            wait_job = status
        elif launch_job is None and status.job_id == launch_job_name:
            launch_job = status
    logger.info("Selected wait job build: {}".format(wait_job))
    logger.info("Selected launch job build: {}".format(launch_job))
    # Trigger if job is not already trigger and wait job is fine
    return launch_job is None and wait_job is not None and wait_job.success


if __name__ == '__main__':

    parser = argparse.ArgumentParser(
        description="Check condition at DLRN to trigger job")
    parser.add_argument(
        '--dlrn-url',
        default="https://trunk.rdoproject.org/api-centos-master-uc")
    parser.add_argument('--promotion-name', default="tripleo-ci-testing")
    parser.add_argument(
        '--wait-job',
        default="periodic-tripleo-centos-7-master-containers-build")
    parser.add_argument('--launch-job', default="test-connection-to-hardware")
    args = parser.parse_args()

    client = dlrnapi_client.ApiClient(host=args.dlrn_url)
    dlrn = dlrnapi_client.DefaultApi(api_client=client)
    if not check_trigger_condition(dlrn, args.promotion_name, args.wait_job,
                                   args.launch_job):
        sys.exit(1)
def get_dlrn_client(config):
    api_client = dlrnapi_client.ApiClient(host=config['main']['api_url'])
    return dlrnapi_client.DefaultApi(api_client=api_client)