def run(
    dry_run,
    thread_pool_size=10,
    internal=None,
    use_jump_host=True,
    take_over=True,
    defer=None,
):
    namespaces = [
        namespace_info for namespace_info in queries.get_namespaces()
        if namespace_info.get("limitRanges")
    ]

    namespaces = construct_resources(namespaces)

    if not namespaces:
        logging.debug("No LimitRanges definition found in app-interface!")
        sys.exit(0)

    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=["LimitRange"],
        internal=internal,
        use_jump_host=use_jump_host,
    )
    defer(oc_map.cleanup)

    add_desired_state(namespaces, ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri, thread_pool_size, take_over=take_over)

    if ri.has_error_registered():
        sys.exit(1)
def run(dry_run=False,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        vault_output_path='',
        defer=None):
    namespaces = [
        namespace_info for namespace_info in queries.get_namespaces()
        if namespace_info.get('openshiftServiceAccountTokens')
    ]
    for namespace_info in namespaces:
        if not namespace_info.get('openshiftServiceAccountTokens'):
            continue
        for sat in namespace_info['openshiftServiceAccountTokens']:
            namespaces.append(sat['namespace'])

    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['Secret'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())
    fetch_desired_state(namespaces, ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri)
    if not dry_run and vault_output_path:
        write_outputs_to_vault(vault_output_path, ri)

    if ri.has_error_registered():
        sys.exit(1)
def run(dry_run,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        defer=None):
    namespaces = [
        namespace_info for namespace_info in queries.get_namespaces()
        if namespace_info.get("managedRoles")
        and is_in_shard(f"{namespace_info['cluster']['name']}/" +
                        f"{namespace_info['name']}")
    ]
    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=["RoleBinding.authorization.openshift.io"],
        internal=internal,
        use_jump_host=use_jump_host,
    )
    defer(oc_map.cleanup)
    fetch_desired_state(ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri, thread_pool_size)

    if ri.has_error_registered():
        sys.exit(1)
def run(
    dry_run,
    thread_pool_size=10,
    internal=None,
    use_jump_host=True,
    vault_output_path="",
    defer=None,
):
    namespaces = canonicalize_namespaces(queries.get_serviceaccount_tokens())
    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=["Secret"],
        internal=internal,
        use_jump_host=use_jump_host,
    )
    defer(oc_map.cleanup)
    fetch_desired_state(namespaces, ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri, thread_pool_size)
    if not dry_run and vault_output_path:
        write_outputs_to_vault(vault_output_path, ri)

    if ri.has_error_registered():
        sys.exit(1)
Exemplo n.º 5
0
def run(dry_run=False,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        take_over=True,
        defer=None):
    namespaces = [
        namespace_info for namespace_info in queries.get_namespaces()
        if namespace_info.get('limitRanges')
    ]

    namespaces = construct_resources(namespaces)

    if not namespaces:
        logging.debug("No LimitRanges definition found in app-interface!")
        sys.exit(0)

    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['LimitRange'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())

    add_desired_state(namespaces, ri, oc_map)
    ob.realize_data(dry_run,
                    oc_map,
                    ri,
                    enable_deletion=True,
                    take_over=take_over)
Exemplo n.º 6
0
def run(dry_run=False,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        defer=None):

    try:
        gqlapi = gql.get_api()
        namespaces = [
            namespace_info
            for namespace_info in gqlapi.query(NAMESPACES_QUERY)['namespaces']
            if namespace_info.get('networkPoliciesAllow')
        ]
        ri, oc_map = ob.fetch_current_state(
            namespaces=namespaces,
            thread_pool_size=thread_pool_size,
            integration=QONTRACT_INTEGRATION,
            integration_version=QONTRACT_INTEGRATION_VERSION,
            override_managed_types=['NetworkPolicy'],
            internal=internal,
            use_jump_host=use_jump_host)
        defer(lambda: oc_map.cleanup())
        fetch_desired_state(namespaces, ri, oc_map)
        ob.realize_data(dry_run, oc_map, ri)

    except Exception as e:
        msg = 'There was problem running openshift network policies reconcile.'
        msg += ' Exception: {}'
        msg = msg.format(str(e))
        logging.error(msg)
        sys.exit(1)
def run(
    dry_run: bool,
    thread_pool_size: int = 10,
    internal: Optional[bool] = None,
    use_jump_host=True,
    defer=None,
):
    gabi_instances = queries.get_gabi_instances()
    if not gabi_instances:
        logging.debug("No gabi instances found in app-interface")
        sys.exit(ExitCodes.SUCCESS)

    gabi_namespaces = [i["namespace"] for g in gabi_instances for i in g["instances"]]

    ri, oc_map = ob.fetch_current_state(
        namespaces=gabi_namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=["ConfigMap"],
        internal=internal,
        use_jump_host=use_jump_host,
    )
    defer(oc_map.cleanup)
    fetch_desired_state(gabi_instances, ri)
    ob.realize_data(dry_run, oc_map, ri, thread_pool_size)

    if ri.has_error_registered():
        sys.exit(1)
Exemplo n.º 8
0
def run(dry_run,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        take_over=True,
        defer=None):

    namespaces = [
        namespace_info for namespace_info in queries.get_namespaces()
        if namespace_info.get('quota')
    ]

    if not namespaces:
        logging.debug("No ResourceQuota definition found in app-interface!")
        sys.exit(0)

    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['ResourceQuota'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())
    fetch_desired_state(namespaces, ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri)

    if ri.has_error_registered():
        sys.exit(1)
Exemplo n.º 9
0
def run(dry_run,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        defer=None):

    gqlapi = gql.get_api()

    namespaces = []
    for namespace_info in gqlapi.query(NAMESPACES_QUERY)['namespaces']:
        if not namespace_info.get('networkPoliciesAllow'):
            continue

        shard_key = (f"{namespace_info['cluster']['name']}/"
                     f"{namespace_info['name']}")

        if not is_in_shard(shard_key):
            continue

        namespaces.append(namespace_info)

    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['NetworkPolicy'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())
    fetch_desired_state(namespaces, ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri)

    if ri.has_error_registered():
        sys.exit(1)
def run(dry_run=False, thread_pool_size=10, internal=None,
        use_jump_host=True, defer=None):
    performance_parameters = queries.get_performance_parameters()
    observability_namespaces = [
        pp['namespace']['cluster']['observabilityNamespace']
        for pp in performance_parameters
        if pp['namespace']['cluster']['observabilityNamespace'] is not None]

    if not observability_namespaces:
        logging.error('No observability namespaces found')
        sys.exit(1)

    ri, oc_map = ob.fetch_current_state(
        namespaces=observability_namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['PrometheusRule'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())
    fetch_desired_state(performance_parameters, ri)
    ob.realize_data(dry_run, oc_map, ri)

    if ri.has_error_registered():
        sys.exit(1)
Exemplo n.º 11
0
def run(dry_run: bool,
        thread_pool_size: int,
        internal: bool,
        use_jump_host: bool,
        defer=None) -> None:
    # prepare
    desired_endpoints = get_endpoints()
    namespaces = {
        p.blackboxExporter.namespace.get("name"): p.blackboxExporter.namespace
        for p in desired_endpoints if p.blackboxExporter
    }

    if namespaces:
        ri, oc_map = ob.fetch_current_state(
            namespaces.values(),
            thread_pool_size=thread_pool_size,
            internal=internal,
            use_jump_host=use_jump_host,
            integration=QONTRACT_INTEGRATION,
            integration_version=QONTRACT_INTEGRATION_VERSION,
            override_managed_types=["Probe"])
        defer(oc_map.cleanup)

        # reconcile
        for provider, endpoints in desired_endpoints.items():
            fill_desired_state(provider, endpoints, ri)
        ob.realize_data(dry_run,
                        oc_map,
                        ri,
                        thread_pool_size,
                        recycle_pods=False)

        if ri.has_error_registered():
            sys.exit(1)
def run(dry_run,
        thread_pool_size=10,
        saas_file_name=None,
        env_name=None,
        defer=None):
    saas_files = queries.get_saas_files(saas_file_name, env_name)
    if not saas_files:
        logging.error('no saas files found')
        sys.exit(1)

    instance = queries.get_gitlab_instance()
    desired_jenkins_instances = [s['instance']['name'] for s in saas_files]
    jenkins_map = jenkins_base.get_jenkins_map(
        desired_instances=desired_jenkins_instances)
    settings = queries.get_app_interface_settings()
    try:
        gl = GitLabApi(instance, settings=settings)
    except Exception:
        # allow execution without access to gitlab
        # as long as there are no access attempts.
        gl = None

    saasherder = SaasHerder(saas_files,
                            thread_pool_size=thread_pool_size,
                            gitlab=gl,
                            integration=QONTRACT_INTEGRATION,
                            integration_version=QONTRACT_INTEGRATION_VERSION,
                            settings=settings,
                            jenkins_map=jenkins_map)
    if not saasherder.valid:
        sys.exit(1)
    if len(saasherder.namespaces) == 0:
        logging.warning('no targets found')
        sys.exit(0)

    ri, oc_map = ob.fetch_current_state(
        namespaces=saasherder.namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION)
    defer(lambda: oc_map.cleanup())
    saasherder.populate_desired_state(ri)
    # if saas_file_name is defined, the integration
    # is being called from multiple running instances
    ob.realize_data(dry_run,
                    oc_map,
                    ri,
                    caller=saas_file_name,
                    wait_for_namespace=True,
                    no_dry_run_skip_compare=True,
                    take_over=saasherder.take_over)

    if ri.has_error_registered():
        sys.exit(1)
def run(dry_run: bool, thread_pool_size: int, internal: bool,
        use_jump_host: bool, defer=None) -> None:
    # verify blackbox-exporter modules
    settings = queries.get_app_interface_settings()
    allowed_modules = \
        set(settings["endpointMonitoringBlackboxExporterModules"])
    verification_errors = False
    if allowed_modules:
        for p in get_blackbox_providers():
            if p.blackboxExporter and \
                 p.blackboxExporter.module not in allowed_modules:
                LOG.error(
                    f"endpoint monitoring provider {p.name} uses "
                    f"blackbox-exporter module {p.blackboxExporter.module} "
                    f"which is not in the allow list {allowed_modules} of "
                    "app-interface-settings"
                )
                verification_errors = True
    if verification_errors:
        sys.exit(1)

    # prepare
    desired_endpoints = get_endpoints()
    namespaces = {
        p.blackboxExporter.namespace.get("name"):
        p.blackboxExporter.namespace
        for p in desired_endpoints
        if p.blackboxExporter
    }

    if namespaces:
        ri, oc_map = ob.fetch_current_state(
            namespaces.values(),
            thread_pool_size=thread_pool_size,
            internal=internal,
            use_jump_host=use_jump_host,
            integration=QONTRACT_INTEGRATION,
            integration_version=QONTRACT_INTEGRATION_VERSION,
            override_managed_types=["Probe"]
        )
        defer(oc_map.cleanup)

        # reconcile
        for provider, endpoints in desired_endpoints.items():
            fill_desired_state(provider, endpoints, ri)
        ob.realize_data(dry_run, oc_map, ri, thread_pool_size,
                        recycle_pods=False)

        if ri.has_error_registered():
            sys.exit(1)
Exemplo n.º 14
0
def run(
    dry_run: bool,
    thread_pool_size: int = 10,
    internal: Optional[bool] = None,
    use_jump_host: bool = True,
    saas_file_name: Optional[str] = None,
) -> None:

    tkn_providers = fetch_tkn_providers(saas_file_name)

    # TODO: This will need to be an error condition in the future
    if not tkn_providers:
        LOG.debug("No saas files found to be processed")
        sys.exit(0)

    # We need to start with the desired state to know the names of the
    # tekton objects that will be created in the providers' namespaces. We
    # need to make sure that this integration only manages its resources
    # and not the tekton resources already created via openshift-resources
    LOG.debug("Fetching desired resources")
    desired_resources = fetch_desired_resources(tkn_providers)

    tkn_namespaces = [tknp["namespace"] for tknp in tkn_providers.values()]
    LOG.debug("Fetching current resources")
    ri, oc_map = ob.fetch_current_state(
        namespaces=tkn_namespaces,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=["Pipeline", "Task"],
        internal=internal,
        use_jump_host=use_jump_host,
        thread_pool_size=thread_pool_size,
    )
    defer(oc_map.cleanup)

    LOG.debug("Adding desired resources to inventory")
    for desired_resource in desired_resources:
        ri.add_desired(**desired_resource)

    LOG.debug("Realizing data")
    ob.realize_data(dry_run, oc_map, ri, thread_pool_size)

    if ri.has_error_registered():
        sys.exit(ExitCodes.ERROR)

    sys.exit(0)
Exemplo n.º 15
0
def run(dry_run, thread_pool_size=10, internal=None,
        use_jump_host=True, defer=None):
    clusters = [cluster_info for cluster_info
                in queries.get_clusters()
                if cluster_info.get('managedClusterRoles')]
    ri, oc_map = ob.fetch_current_state(
        clusters=clusters,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['ClusterRoleBinding'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())
    fetch_desired_state(ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri)

    if ri.has_error_registered():
        sys.exit(1)
Exemplo n.º 16
0
def run(dry_run=False,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        defer=None):
    namespaces = [
        namespace_info for namespace_info in queries.get_namespaces()
        if namespace_info.get('managedRoles')
    ]
    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['RoleBinding'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())
    fetch_desired_state(ri, oc_map)
    ob.realize_data(dry_run, oc_map, ri)
def run(dry_run=False,
        thread_pool_size=10,
        saas_file_name=None,
        env_name=None,
        defer=None):
    saas_files = queries.get_saas_files(saas_file_name, env_name)
    if not saas_files:
        logging.error('no saas files found')
        sys.exit(1)

    instance = queries.get_gitlab_instance()
    settings = queries.get_app_interface_settings()
    try:
        gl = GitLabApi(instance, settings=settings)
    except Exception:
        # allow execution without access to gitlab
        # as long as there are no access attempts.
        gl = None

    saasherder = SaasHerder(saas_files,
                            thread_pool_size=thread_pool_size,
                            gitlab=gl,
                            integration=QONTRACT_INTEGRATION,
                            integration_version=QONTRACT_INTEGRATION_VERSION,
                            settings=settings)
    if not saasherder.valid:
        sys.exit(1)

    ri, oc_map = ob.fetch_current_state(
        namespaces=saasherder.namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION)
    defer(lambda: oc_map.cleanup())
    saasherder.populate_desired_state(ri)
    # if saas_file_name is defined, the integration
    # is being called from multiple running instances
    ob.realize_data(dry_run, oc_map, ri, caller=saas_file_name)

    if ri.has_error_registered():
        sys.exit(1)
Exemplo n.º 18
0
def run(dry_run, thread_pool_size=10, internal=None,
        use_jump_host=True, defer=None):

    try:
        namespaces = [
            namespace_info for namespace_info
            in queries.get_namespaces()
            if namespace_info.get('openshiftAcme')
            ]

        namespaces = construct_resources(namespaces)

        ri, oc_map = ob.fetch_current_state(
            namespaces=namespaces,
            thread_pool_size=thread_pool_size,
            integration=QONTRACT_INTEGRATION,
            integration_version=QONTRACT_INTEGRATION_VERSION,
            override_managed_types=[
                'Deployment',
                'Role',
                'RoleBinding',
                'ServiceAccount',
                'Secret'],
            internal=internal,
            use_jump_host=use_jump_host)
        add_desired_state(namespaces, ri, oc_map)

        defer(lambda: oc_map.cleanup())

        ob.realize_data(dry_run, oc_map, ri)

        if ri.has_error_registered():
            sys.exit(1)

    except Exception as e:
        msg = 'There was problem running openshift acme reconcile.'
        msg += ' Exception: {}'
        msg = msg.format(str(e))
        logging.error(msg)
        sys.exit(1)
Exemplo n.º 19
0
def run(dry_run=False, thread_pool_size=10, defer=None):
    instance = queries.get_gitlab_instance()
    settings = queries.get_app_interface_settings()
    aws_accounts = queries.get_aws_accounts()
    gl = GitLabApi(instance, settings=settings)

    saas_files = queries.get_saas_files()
    saasherder = SaasHerder(saas_files,
                            gitlab=gl,
                            integration=QONTRACT_INTEGRATION,
                            integration_version=QONTRACT_INTEGRATION_VERSION,
                            settings=settings)
    ri, oc_map = ob.fetch_current_state(
        namespaces=saasherder.namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION)
    defer(lambda: oc_map.cleanup())
    saasherder.populate_desired_state(ri)
    enable_deletion = False if ri.has_error_registered() else True
    ob.realize_data(dry_run, oc_map, ri, enable_deletion=enable_deletion)
    saasherder.slack_notify(dry_run, aws_accounts, ri)
def run(dry_run=False, thread_pool_size=10, internal=None,
        use_jump_host=True, take_over=True, defer=None):
    try:
        namespaces = [namespace_info for namespace_info
                      in queries.get_namespaces()
                      if namespace_info.get('quota')]
        ri, oc_map = ob.fetch_current_state(
            namespaces=namespaces,
            thread_pool_size=thread_pool_size,
            integration=QONTRACT_INTEGRATION,
            integration_version=QONTRACT_INTEGRATION_VERSION,
            override_managed_types=['ResourceQuota'],
            internal=internal,
            use_jump_host=use_jump_host)
        defer(lambda: oc_map.cleanup())
        fetch_desired_state(namespaces, ri, oc_map)
        ob.realize_data(dry_run, oc_map, ri)

        if ri.has_error_registered():
            sys.exit(1)

    except Exception as e:
        logging.error(f"Error during execution. Exception: {str(e)}")
        sys.exit(1)
Exemplo n.º 21
0
def run(dry_run,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        vault_throughput_path=None,
        defer=None):
    if not vault_throughput_path:
        logging.error('must supply vault throughput path')
        sys.exit(ExitCodes.ERROR)

    kafka_clusters = queries.get_kafka_clusters()
    if not kafka_clusters:
        logging.debug("No Kafka clusters found in app-interface")
        sys.exit(ExitCodes.SUCCESS)

    settings = queries.get_app_interface_settings()
    ocm_map = OCMMap(clusters=kafka_clusters,
                     integration=QONTRACT_INTEGRATION,
                     settings=settings)
    namespaces = []
    for kafka_cluster in kafka_clusters:
        namespaces.extend(kafka_cluster['namespaces'])
    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['Secret'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(oc_map.cleanup)

    current_state = ocm_map.kafka_cluster_specs()
    desired_state = fetch_desired_state(kafka_clusters)
    kafka_service_accounts = ocm_map.kafka_service_account_specs()

    for kafka_cluster in kafka_clusters:
        kafka_cluster_name = kafka_cluster['name']
        desired_cluster = [
            c for c in desired_state if kafka_cluster_name == c['name']
        ][0]
        current_cluster = [
            c for c in current_state if kafka_cluster_name == c['name']
        ]
        # check if cluster exists. if not - create it
        if not current_cluster:
            logging.info(['create_cluster', kafka_cluster_name])
            if not dry_run:
                ocm = ocm_map.get(kafka_cluster_name)
                ocm.create_kafka_cluster(desired_cluster)
            continue
        # there should only be one cluster
        current_cluster = current_cluster[0]
        # check if desired cluster matches current cluster. if not - error
        if not all(k in current_cluster.keys()
                   for k in desired_cluster.keys()):
            logging.error(
                '[%s] desired spec %s is different ' + 'from current spec %s',
                kafka_cluster_name, desired_cluster, current_cluster)
            ri.register_error()
            continue
        # check if cluster is ready. if not - wait
        status = current_cluster['status']
        if status != STATUS_READY:
            # check if cluster is failed
            if status == STATUS_FAILED:
                failed_reason = current_cluster['failed_reason']
                logging.error(
                    f'[{kafka_cluster_name}] cluster status is {status}. '
                    f'reason: {failed_reason}')
                ri.register_error()
            else:
                logging.warning(
                    f'[{kafka_cluster_name}] cluster status is {status}')
            continue
        # we have a ready cluster!
        # get a service account for the cluster
        kafka_service_account = get_kafa_service_account(
            kafka_service_accounts,
            kafka_cluster_name,
            vault_throughput_path,
            dry_run,
            ocm_map,
        )
        # let's create a Secret in all referencing namespaces
        kafka_namespaces = kafka_cluster['namespaces']
        secret_fields = ['bootstrap_server_host']
        data = {k: v for k, v in current_cluster.items() if k in secret_fields}
        data.update(kafka_service_account)
        resource = construct_oc_resource(data)
        for namespace_info in kafka_namespaces:
            ri.add_desired(namespace_info['cluster']['name'],
                           namespace_info['name'], resource.kind,
                           resource.name, resource)
        if not dry_run:
            write_output_to_vault(vault_throughput_path, kafka_cluster_name,
                                  resource.body['data'])

    ob.realize_data(dry_run, oc_map, ri, thread_pool_size)

    if ri.has_error_registered():
        sys.exit(ExitCodes.ERROR)
Exemplo n.º 22
0
def run(dry_run,
        thread_pool_size=10,
        internal=None,
        use_jump_host=True,
        defer=None):
    kafka_clusters = queries.get_kafka_clusters()
    if not kafka_clusters:
        logging.debug("No Kafka clusters found in app-interface")
        sys.exit(ExitCodes.SUCCESS)

    settings = queries.get_app_interface_settings()
    ocm_map = OCMMap(clusters=kafka_clusters,
                     integration=QONTRACT_INTEGRATION,
                     settings=settings)
    namespaces = []
    for kafka_cluster in kafka_clusters:
        namespaces.extend(kafka_cluster['namespaces'])
    ri, oc_map = ob.fetch_current_state(
        namespaces=namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        override_managed_types=['Secret'],
        internal=internal,
        use_jump_host=use_jump_host)
    defer(lambda: oc_map.cleanup())

    current_state = ocm_map.kafka_cluster_specs()
    desired_state = fetch_desired_state(kafka_clusters)

    error = False
    for kafka_cluster in kafka_clusters:
        kafka_cluster_name = kafka_cluster['name']
        desired_cluster = [
            c for c in desired_state if kafka_cluster_name == c['name']
        ][0]
        current_cluster = [
            c for c in current_state if kafka_cluster_name == c['name']
        ]
        # check if cluster exists. if not - create it
        if not current_cluster:
            logging.info(['create_cluster', kafka_cluster_name])
            if not dry_run:
                ocm = ocm_map.get(kafka_cluster_name)
                ocm.create_kafka_cluster(desired_cluster)
            continue
        # there should only be one cluster
        current_cluster = current_cluster[0]
        # check if desired cluster matches current cluster. if not - error
        if not all(k in current_cluster.keys()
                   for k in desired_cluster.keys()):
            logging.error(
                '[%s] desired spec %s is different ' + 'from current spec %s',
                kafka_cluster_name, desired_cluster, current_cluster)
            error = True
            continue
        # check if cluster is ready. if not - wait
        if current_cluster['status'] != 'complete':
            continue
        # we have a ready cluster!
        # let's create a Secret in all referencing namespaces
        kafka_namespaces = kafka_cluster['namespaces']
        secret_fields = ['bootstrapServerHost']
        data = {k: v for k, v in current_cluster.items() if k in secret_fields}
        resource = construct_oc_resource(data)
        for namespace_info in kafka_namespaces:
            ri.add_desired(namespace_info['cluster']['name'],
                           namespace_info['name'], resource.kind,
                           resource.name, resource)

    ob.realize_data(dry_run, oc_map, ri)

    if error:
        sys.exit(ExitCodes.ERROR)
Exemplo n.º 23
0
def run(dry_run,
        thread_pool_size=10,
        saas_file_name=None,
        env_name=None,
        defer=None):
    saas_files = queries.get_saas_files(saas_file_name, env_name)
    if not saas_files:
        logging.error('no saas files found')
        sys.exit(1)

    instance = queries.get_gitlab_instance()
    desired_jenkins_instances = [s['instance']['name'] for s in saas_files]
    jenkins_map = jenkins_base.get_jenkins_map(
        desired_instances=desired_jenkins_instances)
    settings = queries.get_app_interface_settings()
    try:
        gl = GitLabApi(instance, settings=settings)
    except Exception:
        # allow execution without access to gitlab
        # as long as there are no access attempts.
        gl = None

    saasherder = SaasHerder(saas_files,
                            thread_pool_size=thread_pool_size,
                            gitlab=gl,
                            integration=QONTRACT_INTEGRATION,
                            integration_version=QONTRACT_INTEGRATION_VERSION,
                            settings=settings,
                            jenkins_map=jenkins_map)
    if not saasherder.valid:
        sys.exit(1)
    if len(saasherder.namespaces) == 0:
        logging.warning('no targets found')
        sys.exit(0)

    ri, oc_map = ob.fetch_current_state(
        namespaces=saasherder.namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        init_api_resources=True)
    defer(lambda: oc_map.cleanup())
    saasherder.populate_desired_state(ri)
    # if saas_file_name is defined, the integration
    # is being called from multiple running instances
    actions = ob.realize_data(dry_run,
                              oc_map,
                              ri,
                              caller=saas_file_name,
                              wait_for_namespace=True,
                              no_dry_run_skip_compare=(not saasherder.compare),
                              take_over=saasherder.take_over)

    if ri.has_error_registered():
        sys.exit(1)

    # send human readable notifications to slack
    # we only do this if:
    # - this is not a dry run
    # - there is a single saas file deployed
    # - output is 'events'
    # - no errors were registered
    if not dry_run and len(saasherder.saas_files) == 1:
        saas_file = saasherder.saas_files[0]
        slack_info = saas_file.get('slack')
        if slack_info and actions and slack_info.get('output') == 'events':
            slack = init_slack(slack_info, QONTRACT_INTEGRATION)
            for action in actions:
                message = \
                    f"[{action['cluster']}] " + \
                    f"{action['kind']} {action['name']} {action['action']}"
                slack.chat_post_message(message)
def run(dry_run,
        thread_pool_size=10,
        io_dir='throughput/',
        saas_file_name=None,
        env_name=None,
        gitlab_project_id=None,
        defer=None):
    all_saas_files = queries.get_saas_files()
    saas_files = queries.get_saas_files(saas_file_name, env_name)
    if not saas_files:
        logging.error('no saas files found')
        sys.exit(ExitCodes.ERROR)

    instance = queries.get_gitlab_instance()
    desired_jenkins_instances = [s['instance']['name'] for s in saas_files]
    jenkins_map = jenkins_base.get_jenkins_map(
        desired_instances=desired_jenkins_instances)
    settings = queries.get_app_interface_settings()
    accounts = queries.get_aws_accounts()
    try:
        gl = GitLabApi(instance, settings=settings)
    except Exception:
        # allow execution without access to gitlab
        # as long as there are no access attempts.
        gl = None

    saasherder = SaasHerder(saas_files,
                            thread_pool_size=thread_pool_size,
                            gitlab=gl,
                            integration=QONTRACT_INTEGRATION,
                            integration_version=QONTRACT_INTEGRATION_VERSION,
                            settings=settings,
                            jenkins_map=jenkins_map,
                            accounts=accounts)
    if len(saasherder.namespaces) == 0:
        logging.warning('no targets found')
        sys.exit(ExitCodes.SUCCESS)

    ri, oc_map = ob.fetch_current_state(
        namespaces=saasherder.namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        init_api_resources=True)
    defer(lambda: oc_map.cleanup())
    saasherder.populate_desired_state(ri)

    # validate that this deployment is valid
    # based on promotion information in targets
    if not saasherder.validate_promotions():
        logging.error('invalid promotions')
        sys.exit(ExitCodes.ERROR)

    # if saas_file_name is defined, the integration
    # is being called from multiple running instances
    actions = ob.realize_data(dry_run,
                              oc_map,
                              ri,
                              caller=saas_file_name,
                              wait_for_namespace=True,
                              no_dry_run_skip_compare=(not saasherder.compare),
                              take_over=saasherder.take_over)

    if not dry_run:
        if saasherder.publish_job_logs:
            try:
                ob.follow_logs(oc_map, actions, io_dir)
            except Exception as e:
                logging.error(str(e))
                ri.register_error()
        try:
            ob.validate_data(oc_map, actions)
        except Exception as e:
            logging.error(str(e))
            ri.register_error()

    # publish results of this deployment
    # based on promotion information in targets
    success = not ri.has_error_registered()
    # only publish promotions for deployment jobs (a single saas file)
    if not dry_run and len(saasherder.saas_files) == 1:
        mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
        saasherder.publish_promotions(success, all_saas_files, mr_cli)

    if not success:
        sys.exit(ExitCodes.ERROR)

    # send human readable notifications to slack
    # we only do this if:
    # - this is not a dry run
    # - there is a single saas file deployed
    # - output is 'events'
    # - no errors were registered
    if not dry_run and len(saasherder.saas_files) == 1:
        saas_file = saasherder.saas_files[0]
        slack_info = saas_file.get('slack')
        if slack_info and actions and slack_info.get('output') == 'events':
            slack = init_slack(slack_info,
                               QONTRACT_INTEGRATION,
                               init_usergroups=False)
            for action in actions:
                message = \
                    f"[{action['cluster']}] " + \
                    f"{action['kind']} {action['name']} {action['action']}"
                slack.chat_post_message(message)
def run(
    dry_run,
    thread_pool_size=10,
    io_dir="throughput/",
    saas_file_name=None,
    env_name=None,
    gitlab_project_id=None,
    defer=None,
):
    all_saas_files = queries.get_saas_files(v1=True, v2=True)
    saas_files = queries.get_saas_files(saas_file_name, env_name, v1=True, v2=True)
    app_interface_settings = queries.get_app_interface_settings()
    if not saas_files:
        logging.error("no saas files found")
        sys.exit(ExitCodes.ERROR)

    # notify different outputs (publish results, slack notifications)
    # we only do this if:
    # - this is not a dry run
    # - there is a single saas file deployed
    notify = not dry_run and len(saas_files) == 1
    if notify:
        saas_file = saas_files[0]
        slack_info = saas_file.get("slack")
        if slack_info:
            slack = slackapi_from_slack_workspace(
                slack_info,
                app_interface_settings,
                QONTRACT_INTEGRATION,
                init_usergroups=False,
            )
            # support built-in start and end slack notifications
            # only in v2 saas files
            if saas_file["apiVersion"] == "v2":
                ri = ResourceInventory()
                console_url = compose_console_url(saas_file, saas_file_name, env_name)
                # deployment result notification
                defer(
                    lambda: slack_notify(
                        saas_file_name,
                        env_name,
                        slack,
                        ri,
                        console_url,
                        in_progress=False,
                    )
                )
                # deployment start notification
                slack_notifications = slack_info.get("notifications")
                if slack_notifications and slack_notifications.get("start"):
                    slack_notify(
                        saas_file_name,
                        env_name,
                        slack,
                        ri,
                        console_url,
                        in_progress=True,
                    )
        else:
            slack = None

    instance = queries.get_gitlab_instance()
    # instance exists in v1 saas files only
    desired_jenkins_instances = [
        s["instance"]["name"] for s in saas_files if s.get("instance")
    ]
    jenkins_map = jenkins_base.get_jenkins_map(
        desired_instances=desired_jenkins_instances
    )
    settings = queries.get_app_interface_settings()
    accounts = queries.get_aws_accounts()
    try:
        gl = GitLabApi(instance, settings=settings)
    except Exception:
        # allow execution without access to gitlab
        # as long as there are no access attempts.
        gl = None

    saasherder = SaasHerder(
        saas_files,
        thread_pool_size=thread_pool_size,
        gitlab=gl,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        settings=settings,
        jenkins_map=jenkins_map,
        accounts=accounts,
    )
    if len(saasherder.namespaces) == 0:
        logging.warning("no targets found")
        sys.exit(ExitCodes.SUCCESS)

    ri, oc_map = ob.fetch_current_state(
        namespaces=saasherder.namespaces,
        thread_pool_size=thread_pool_size,
        integration=QONTRACT_INTEGRATION,
        integration_version=QONTRACT_INTEGRATION_VERSION,
        init_api_resources=True,
        cluster_admin=saasherder.cluster_admin,
    )
    defer(oc_map.cleanup)
    saasherder.populate_desired_state(ri)

    # validate that this deployment is valid
    # based on promotion information in targets
    if not saasherder.validate_promotions():
        logging.error("invalid promotions")
        ri.register_error()
        sys.exit(ExitCodes.ERROR)

    # if saas_file_name is defined, the integration
    # is being called from multiple running instances
    actions = ob.realize_data(
        dry_run,
        oc_map,
        ri,
        thread_pool_size,
        caller=saas_file_name,
        wait_for_namespace=True,
        no_dry_run_skip_compare=(not saasherder.compare),
        take_over=saasherder.take_over,
    )

    if not dry_run:
        if saasherder.publish_job_logs:
            try:
                ob.follow_logs(oc_map, actions, io_dir)
            except Exception as e:
                logging.error(str(e))
                ri.register_error()
        try:
            ob.validate_data(oc_map, actions)
        except Exception as e:
            logging.error(str(e))
            ri.register_error()

    # publish results of this deployment
    # based on promotion information in targets
    success = not ri.has_error_registered()
    # only publish promotions for deployment jobs (a single saas file)
    if notify:
        # Auto-promote next stages only if there are changes in the
        # promoting stage. This prevents trigger promotions on job re-runs
        auto_promote = len(actions) > 0
        mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
        saasherder.publish_promotions(success, all_saas_files, mr_cli, auto_promote)

    if not success:
        sys.exit(ExitCodes.ERROR)

    # send human readable notifications to slack
    # we only do this if:
    # - this is not a dry run
    # - there is a single saas file deployed
    # - output is 'events'
    # - no errors were registered
    if notify and slack and actions and slack_info.get("output") == "events":
        for action in actions:
            message = (
                f"[{action['cluster']}] "
                + f"{action['kind']} {action['name']} {action['action']}"
            )
            slack.chat_post_message(message)