Exemple #1
0
def find_own_running_build():
    '''
    Determines the current build job running on concourse by relying on the "meta" contract (
    see steps/meta), which prints a JSON document containing a UUID. By iterating through all
    current build jobs (considering running jobs only), and comparing the UUID read via file
    system and the UUID from build log output, it is possible to tell whether or not a given
    build job is the one from which this function was invoked.
    '''
    if not _running_on_ci():
        raise RuntimeError(
            'Can only find own running build if running on CI infrastructure.')

    meta_dir = os.path.join(os.path.abspath(check_env('CC_ROOT_DIR')),
                            concourse.model.traits.meta.META_INFO_DIR_NAME)
    meta_info_file = os.path.join(
        meta_dir,
        concourse.steps.meta.jobmetadata_filename,
    )

    with open(meta_info_file, 'r') as f:
        metadata_json = json.load(f)

    build_job_uuid = metadata_json['uuid']

    pipeline_metadata = get_pipeline_metadata()
    config_set = ctx().cfg_factory().cfg_set(
        pipeline_metadata.current_config_set_name)
    concourse_cfg = config_set.concourse()
    client = concourse.client.from_cfg(concourse_cfg,
                                       pipeline_metadata.team_name)

    # only consider limited amount of jobs to avoid large number of requests in case we do not
    # find ourself (assumption: there are only few running jobs in parallel at a given time)
    consider_builds = 20
    builds = client.job_builds(pipeline_metadata.pipeline_name,
                               pipeline_metadata.job_name)
    builds = [
        build for build in builds
        if build.status() is concourse.client.model.BuildStatus.RUNNING
    ][:consider_builds]

    # avoid parsing too much output. usually, there will be only one line (our JSON output)
    # sometimes (new image version is retrieved), there will be a few lines more.
    for build in builds:
        build_events = build.events()
        build_plan = build.plan()
        meta_task_id = build_plan.task_id(
            concourse.model.traits.meta.META_STEP_NAME)

        last_line = None
        try:
            # "our" output is always the last line (ignore download logs from image retrieval)
            for last_line in build_events.iter_buildlog(meta_task_id):
                pass
        except StopIteration:
            pass

        uuid_json = json.loads(last_line)
        if uuid_json['uuid'] == build_job_uuid:
            return build
    else:
        raise RuntimeError('Could not determine own Concourse job.')
Exemple #2
0
def _retrieve_model_element(cfg_type: str, cfg_name: str):
    cfg_factory = ctx().cfg_factory()
    return cfg_factory._cfg_element(cfg_type_name=cfg_type, cfg_name=cfg_name)
Exemple #3
0
def deploy_or_upgrade_landscape(
    config_set_name: CliHint(typehint=str, help=CONFIG_SET_HELP),
    components: CliHint(
        type=LandscapeComponent,
        typehint=[LandscapeComponent],
        choices=[component for component in LandscapeComponent],
        help=
        "list of components to deploy. By default, ALL components will be deployed."
    ) = None,
    webhook_dispatcher_chart_dir: CliHint(
        typehint=str,
        help="directory of webhook dispatcher chart",
    ) = None,
    concourse_deployment_name: CliHint(
        typehint=str,
        help="namespace and deployment name for Concourse") = 'concourse',
    timeout_seconds: CliHint(
        typehint=int, help="how long to wait for concourse startup") = 180,
    webhook_dispatcher_deployment_name: str = 'webhook-dispatcher',
    dry_run: bool = True,
):
    '''Deploys the given components of the Concourse landscape.
    '''
    # handle default (all known components)
    if not components:
        components = [component for component in LandscapeComponent]
    # Validate
    if LandscapeComponent.WHD in components:
        if not webhook_dispatcher_chart_dir:
            raise ValueError(
                f"--webhook-dispatcher-chart-dir must be given if component "
                f"'{LandscapeComponent.WHD.value}' is to be deployed.")
        else:
            webhook_dispatcher_chart_dir = existing_dir(
                webhook_dispatcher_chart_dir)

    _display_info(
        dry_run=dry_run,
        operation="DEPLOYED",
        deployment_name=concourse_deployment_name,
        components=components,
    )

    if dry_run:
        return

    cfg_factory = ctx().cfg_factory()
    config_set = cfg_factory.cfg_set(config_set_name)
    concourse_cfg = config_set.concourse()

    # Set the global kubernetes cluster context to the cluster specified in the ConcourseConfig
    kubernetes_config_name = concourse_cfg.kubernetes_cluster_config()
    kubernetes_cfg = cfg_factory.kubernetes(kubernetes_config_name)
    kube_ctx.set_kubecfg(kubernetes_cfg.kubeconfig())
    ensure_cluster_version(kubernetes_cfg)

    if LandscapeComponent.SECRETS_SERVER in components:
        info('Deploying Secrets Server')
        deploy_secrets_server(config_set_name=config_set_name, )

    if LandscapeComponent.CONCOURSE in components:
        info('Deploying Concourse')
        deploy_or_upgrade_concourse(
            config_set_name=config_set_name,
            deployment_name=concourse_deployment_name,
            timeout_seconds=timeout_seconds,
        )

    if LandscapeComponent.WHD in components:
        info('Deploying Webhook Dispatcher')
        deploy_or_upgrade_webhook_dispatcher(
            config_set_name=config_set_name,
            chart_dir=webhook_dispatcher_chart_dir,
            deployment_name=webhook_dispatcher_deployment_name,
        )

    if LandscapeComponent.MONITORING in components:
        info('Deploying Monitoring stack')
        deploy_or_upgrade_monitoring(config_set_name=config_set_name, )

    if LandscapeComponent.CLAMAV in components:
        info('Deploying ClamAV')
        deploy_or_upgrade_clamav(config_set_name=config_set_name, )
Exemple #4
0
def render_pipelines(
    cfg_name: str,
    out_dir: str,
    template_path: str=_template_path(),
    org: str=None, # if set, filter for org
    repo: str=None, # if set, filter for repo
    host: str=None, # if set, filter for gh-host
):
    if not os.path.isdir(out_dir):
        os.makedirs(out_dir)

    cfg_factory = ctx().cfg_factory()
    config_set = cfg_factory.cfg_set(cfg_name=cfg_name)

    concourse_cfg = config_set.concourse()
    job_mapping_set = cfg_factory.job_mapping(concourse_cfg.job_mapping_cfg_name())

    template_include_dir = template_path

    if repo:
        repository_filter = lambda repo_obj: repo_obj.name == repo
    else:
        repository_filter = None

    def org_names(job_mapping):
        for org in job_mapping.github_organisations():
            yield org.org_name()

    def remove_github_org_configs(job_mapping, org: str, host: str):
        def want_gh_org(org_cfg: model.concourse.GithubOrganisationConfig):
            if org and org_cfg.org_name() != org:
                return False

            gh_cfg: model.github.GithubConfig = cfg_factory.github(org_cfg.github_cfg_name())
            if host and gh_cfg.hostname() != host:
                return False

            return True

        gh_orgs = {
            ghorg.name(): ghorg.raw
            for ghorg in job_mapping.github_organisations()
            if want_gh_org(ghorg)
        }

        job_mapping.raw['github_orgs'] = gh_orgs

    job_mappings = []
    for job_mapping in job_mapping_set.job_mappings().values():
        job_mapping: ccm.JobMapping

        if org and not org in org_names(job_mapping):
            continue

        if org or host:
            remove_github_org_configs(job_mapping, org, host)

        job_mappings.append(job_mapping)

    def_enumerators = [
        GithubOrganisationDefinitionEnumerator(
            job_mapping=job_mapping,
            cfg_set=config_set,
            repository_filter=repository_filter,
        ) for job_mapping in job_mappings
    ]

    preprocessor = DefinitionDescriptorPreprocessor()

    template_retriever = TemplateRetriever(template_path=[template_path])
    renderer = Renderer(
        template_retriever=template_retriever,
        template_include_dir=template_include_dir,
        cfg_set=config_set,
        render_origin=RenderOrigin.LOCAL,
    )

    deployer = FilesystemDeployer(base_dir=out_dir)

    replicator = PipelineReplicator(
        definition_enumerators=def_enumerators,
        descriptor_preprocessor=preprocessor,
        definition_renderer=renderer,
        definition_deployer=deployer,
    )

    replicator.replicate()
Exemple #5
0
def _current_concourse_config():
    if not _running_on_ci():
        raise RuntimeError(
            'Can only determine own concourse config if running on CI')

    return ctx().cfg_set().concourse()
Exemple #6
0
def deploy_or_upgrade_landscape(
    config_set_name: CliHint(typehint=str, help=CONFIG_SET_HELP),
    components: CliHint(
        type=LandscapeComponent,
        typehint=[LandscapeComponent],
        choices=[component for component in LandscapeComponent],
        help="list of components to deploy. By default, ALL components will be deployed."
    )=None,
    webhook_dispatcher_chart_dir: CliHint(
        typehint=str,
        help="directory of webhook dispatcher chart",
    )=None,
    gardenlinux_cache_chart_dir: CliHint(
        typehint=str,
        help="directory of gardenlinux-cache chart",
    )=None,
    concourse_deployment_name: CliHint(
        typehint=str, help="namespace and deployment name for Concourse"
    )='concourse',
    whitesource_backend_chart_dir: CliHint(
        typehint=str,
        help="directory of Whitesource Backend chart",
    )=None,
    whitesource_backend_deployment_name: CliHint(
        typehint=str, help="namespace and deployment name for Whitesource"
    )='whitesource-backend',
    whitesource_cfg_name: CliHint(
        typehint=str,
        help='Whitesource Config',
    )='gardener',
    timeout_seconds: CliHint(typehint=int, help="how long to wait for concourse startup")=180,
    webhook_dispatcher_deployment_name: str='webhook-dispatcher',
    gardenlinux_cache_deployment_name: str='gardenlinux-cache',
    dry_run: bool=True,
):
    '''Deploys the given components of the Concourse landscape.
    '''
    # handle default (all known components)
    if not components:
        components = [component for component in LandscapeComponent]
    # Validate
    if LandscapeComponent.WHD in components:
        if not webhook_dispatcher_chart_dir:
            raise ValueError(
                f"--webhook-dispatcher-chart-dir must be given if component "
                f"'{LandscapeComponent.WHD.value}' is to be deployed."
            )
        else:
            webhook_dispatcher_chart_dir = existing_dir(webhook_dispatcher_chart_dir)
    if LandscapeComponent.GARDENLINUX_CACHE in components:
        if not gardenlinux_cache_chart_dir:
            raise ValueError(
                f"--gardenlinux-cache-chart-dir must be given if component "
                f"'{LandscapeComponent.GARDENLINUX_CACHE.value}' is to be deployed."
            )
        else:
            gardenlinux_cache_chart_dir = existing_dir(gardenlinux_cache_chart_dir)

    _display_info(
        dry_run=dry_run,
        operation="DEPLOYED",
        deployment_name=concourse_deployment_name,
        components=components,
    )

    if dry_run:
        return

    cfg_factory = ctx().cfg_factory()
    config_set = cfg_factory.cfg_set(config_set_name)
    concourse_cfg = config_set.concourse()

    # Set the global kubernetes cluster context to the cluster specified in the ConcourseConfig
    kubernetes_config_name = concourse_cfg.kubernetes_cluster_config()
    kubernetes_cfg = cfg_factory.kubernetes(kubernetes_config_name)
    kube_ctx.set_kubecfg(kubernetes_cfg.kubeconfig())

    if LandscapeComponent.SECRETS_SERVER in components:
        info('Deploying Secrets Server')
        deploy_secrets_server(
            config_set_name=config_set_name,
        )

    if LandscapeComponent.CONCOURSE in components:
        info('Deploying Concourse')
        deploy_or_upgrade_concourse(
            config_set_name=config_set_name,
            deployment_name=concourse_deployment_name,
            timeout_seconds=timeout_seconds,
        )

    if LandscapeComponent.WHD in components:
        info('Deploying Webhook Dispatcher')
        deploy_or_upgrade_webhook_dispatcher(
            config_set_name=config_set_name,
            chart_dir=webhook_dispatcher_chart_dir,
            deployment_name=webhook_dispatcher_deployment_name,
        )

    if LandscapeComponent.CLAMAV in components:
        info ('Deploying ClamAV')
        deploy_or_upgrade_clamav(
            config_set_name=config_set_name,
        )

    if LandscapeComponent.GARDENLINUX_CACHE in components:
        info ('Deploying Gardenlinux Cache')
        deploy_or_upgrade_gardenlinux_cache(
            config_set_name=config_set_name,
            chart_dir=gardenlinux_cache_chart_dir,
            deployment_name=gardenlinux_cache_deployment_name,
        )

    if LandscapeComponent.WHITESOURCE_BACKEND in components:
        info ('Deploying Whitesource Backend')
        extra_args = {}
        if whitesource_backend_deployment_name:
            extra_args['deployment_name'] = whitesource_backend_deployment_name
        if whitesource_cfg_name:
            extra_args['whitesource_cfg_name'] = whitesource_cfg_name
        if whitesource_backend_chart_dir:
            extra_args['chart_dir'] = whitesource_backend_chart_dir

        deploy_or_upgrade_whitesource_api_extension(
            config_set_name=config_set_name,
            **extra_args,
        )
Exemple #7
0
def determine_mail_recipients(
        github_cfg_name,
        src_dirs=(),
        components: typing.Sequence[cm.Component] = (),
        component_names=(),
        codeowners_files=(),
        branch_name='master',
        ctx_repo_url=None,
):
    '''
    returns a generator yielding all email addresses for the given (git) repository work tree
    Email addresses are looked up:
    - from head commit: author and committer
    - from *CODEOWNERS files [0]

    Email addresses are not de-duplicated (this should be done by consumers)

    [0] https://help.github.com/articles/about-codeowners/
    '''
    if not any((components, component_names, src_dirs, codeowners_files)):
        return  # nothing to do

    if components and component_names:
        raise ValueError('only one of components, component_names must be set')

    if component_names and not ctx_repo_url:
        raise ValueError(
            'If component_names is given, ctx_repo_url must also be given')

    cfg_factory = ctx().cfg_factory()

    github_cfg = cfg_factory.github(github_cfg_name)
    github_api = ccc.github.github_api(github_cfg)

    yield from determine_head_commit_recipients(src_dirs)

    yield from determine_local_repository_codeowners_recipients(
        github_api=github_api,
        src_dirs=src_dirs,
    )

    yield from determine_codeowner_file_recipients(
        github_api=github_api,
        codeowners_files=codeowners_files,
    )

    if component_names:
        entries_and_apis = [
            _codeowners_parser_from_component_name(
                component_name=component_name,
                ctx_repo_url=ctx_repo_url,
                branch_name=branch_name) for component_name in component_names
        ]
    elif components:
        entries_and_apis = [
            _codeowners_parser_from_component(component=component,
                                              branch_name=branch_name)
            for component in components
        ]
    else:
        raise ValueError('One of components and component_names must be given')

    for api, codeowner_entries in entries_and_apis:
        yield from github.codeowners.resolve_email_addresses(
            codeowners_entries=codeowner_entries,
            github_api=api,
        )