def initialize(interactive=False, dry_run=False): ckan_infra = CkanInfra(required=False) solr_config = config_manager.interactive_set( { 'self-hosted': True }, secret_name='solr-config', interactive=interactive ) if is_self_hosted(solr_config): initialize_self_hosted(interactive=interactive, dry_run=dry_run) else: config_manager.interactive_set( { 'http-endpoint': ckan_infra.SOLR_HTTP_ENDPOINT, }, secret_name='solr-config', interactive=interactive ) config_manager.interactive_set( { 'num-shards': ckan_infra.SOLR_NUM_SHARDS or '1', 'replication-factor': ckan_infra.SOLR_REPLICATION_FACTOR or '3' }, secret_name='solr-config', interactive=interactive )
def initialize(interactive=False): config_manager.interactive_set({ 'default-storage-bucket': 'ckan', }, secret_name='ckan-storage-config', interactive=interactive) logs.warning('Minio bucket policy was not applied!')
def initialize(interactive=False, provider_id=None, storage_suffix=None, use_existing_disk_name=None, dry_run=False): if not provider_id and interactive: config_manager.interactive_set({'use-cloud-native-storage': True}, secret_name=CONFIG_NAME, interactive=interactive) if config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME): provider_id = get_provider_id() logs.info(f'Storage init: chosen provider_id: {provider_id}') provider = get_provider(default=minio_provider_id, provider_id=provider_id).initialize( interactive=interactive, storage_suffix=storage_suffix, use_existing_disk_name=use_existing_disk_name, dry_run=dry_run) if provider: provider.initialize(interactive=interactive, storage_suffix=storage_suffix, use_existing_disk_name=use_existing_disk_name, dry_run=dry_run)
def initialize(interactive=False): ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'default-storage-bucket': ckan_infra.GCLOUD_STORAGE_BUCKET, }, secret_name='ckan-storage-config', interactive=interactive ) logs.warning('Minio bucket policy was not applied!')
def initialize(interactive=False, storage_suffix='', use_existing_disk_name=False, dry_run=False): default_zone = cluster_config_get('cluster-compute-zone') assert default_zone, 'No cluster region specified.' if interactive and not dry_run: config_manager.interactive_set({'storage-region': default_zone}, secret_name=CONFIG_NAME, interactive=interactive)
def config_interactive_set(submodule, provider_id=None, default_values=None, namespace=None, is_secret=False, suffix=None, from_file=False, interactive=True): resource_name = get_resource_name(submodule, provider_id=provider_id, suffix=suffix) config_manager.interactive_set( default_values, secret_name=resource_name if is_secret else None, configmap_name=None if is_secret else resource_name, namespace=namespace, from_file=from_file, interactive=interactive )
def initialize(interactive=False): ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'env-id': ckan_infra.ROUTERS_ENV_ID, 'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN, #'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL, #'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY }, configmap_name='routers-config', interactive=interactive) routers_manager.install_crds()
def config_interactive_set(singular, name, default_values=None, namespace=None, is_secret=False, from_file=False): resource_name = get_resource_name(singular, name) config_manager.interactive_set( default_values, secret_name=resource_name if is_secret else None, configmap_name=None if is_secret else resource_name, namespace=namespace, from_file=from_file)
def initialize(interactive=False): config_manager.interactive_set( { 'env-id': None, 'default-root-domain': None, 'dns-provider': 'route53' if cluster_manager.get_provider_id() == 'aws' else None }, configmap_name='routers-config', interactive=interactive) dns_provider = get_dns_provider() if dns_provider.lower() == 'none': if cluster_manager.get_provider_id() != 'minikube': return logs.info(dns_provider=dns_provider) if dns_provider == 'cloudflare': config_manager.interactive_set( { 'cloudflare-email': None, 'cloudflare-api-key': None }, configmap_name='routers-config', interactive=interactive) routers_manager.install_crds() infra_router_name = routers_manager.get_default_infra_router_name() default_root_domain = config_manager.get('default-root-domain', configmap_name='routers-config', required=True) logs.info('Creating infra router', infra_router_name=infra_router_name, default_root_domain=default_root_domain) routers_manager.create( infra_router_name, routers_manager.get_traefik_router_spec( default_root_domain, config_manager.get('cloudflare-email', configmap_name='routers-config', required=False, default=None), config_manager.get('cloudflare-api-key', configmap_name='routers-config', required=False, default=None), dns_provider=dns_provider))
def initialize(interactive=False): ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'env-id': ckan_infra.ROUTERS_ENV_ID, 'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN, 'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL, 'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY }, configmap_name='routers-config', interactive=interactive ) routers_manager.install_crds() routers_manager.create( routers_manager.get_default_infra_router_name(), routers_manager.get_traefik_router_spec( config_manager.get('default-root-domain', configmap_name='routers-config'), config_manager.get('cloudflare-email', configmap_name='routers-config'), config_manager.get('cloudflare-api-key', configmap_name='routers-config') ) )
def initialize(interactive=False): ckan_db_migration_manager.initialize(interactive=interactive) if config_manager.get('enable-deis-ckan', configmap_name='global-ckan-config') == 'y': ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG, }, from_file=True, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD, }, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER, 'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME, 'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD, 'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL, }, secret_name='ckan-docker-registry', interactive=interactive) crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX, INSTANCE_CRD_KIND_SUFFIX) crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR, INSTANCE_NAME_CRD_PLURAL_SUFFIX, INSTANCE_NAME_CRD_KIND_SUFFIX) if config_manager.get('disable-centralized-datapushers', configmap_name='global-ckan-config', required=False) != 'yes': from ckan_cloud_operator.datapushers import initialize as datapusher_initialize datapusher_initialize() from ckan_cloud_operator.routers import manager as routers_manager router_name = get_default_instances_router_name() wildcard_ssl_domain = routers_manager.get_default_root_domain() print(f'wildcard_ssl_domain={wildcard_ssl_domain}') allow_wildcard_ssl = routers_manager.get_env_id() == 'p' router = routers_manager.get(router_name, required=False) if router: assert ((allow_wildcard_ssl and router.get( 'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain) or (not allow_wildcard_ssl and not router.get('spec', {})) ), f'invalid router wildcard ssl config: {router}' else: routers_manager.create( router_name, routers_manager.get_traefik_router_spec( wildcard_ssl_domain=wildcard_ssl_domain)) from .storage.manager import initialize as ckan_storage_initialize ckan_storage_initialize(interactive=interactive)
def initialize(interactive=False, storage_suffix='', use_existing_disk_name=False, dry_run=False): default_zone = aws_config_get('aws-default-region', is_secret=True) assert default_zone, 'No cluster region specified.' if interactive and not dry_run: config_manager.interactive_set({'storage-region': default_zone}, secret_name=CONFIG_NAME, interactive=interactive) config_manager.interactive_set({'aws-storage-access-key': None}, secret_name=CONFIG_NAME, interactive=interactive) config_manager.interactive_set({'aws-storage-access-secret': None}, secret_name=CONFIG_NAME, interactive=interactive)
def initialize(interactive=False, storage_suffix='', use_existing_disk_name=False, dry_run=False): default_zone = cluster_config_get('azure-default-location') assert default_zone, 'No cluster region specified.' if interactive and not dry_run: config_manager.interactive_set( {'storage-region': default_zone}, secret_name=CONFIG_NAME, interactive=interactive ) config_manager.interactive_set( {'storage-account-name': ''}, secret_name=CONFIG_NAME, interactive=interactive ) config_manager.interactive_set( {'storage-account-key': ''}, secret_name=CONFIG_NAME, interactive=interactive )
def initialize(log_kwargs=None, interactive=False): logs.info('setting label-prefix: ckan-cloud', **(log_kwargs or {})) config_manager.set('label-prefix', 'ckan-cloud') config_manager.interactive_set({'short-label-prefix': 'cc'}, interactive=interactive)
def initialize(interactive=False): ckan_db_migration_manager.initialize(interactive=interactive) if config_manager.get('enable-deis-ckan', configmap_name='global-ckan-config') == 'y': ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG, }, from_file=True, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD, }, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER, 'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME, 'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD, 'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL, }, secret_name='ckan-docker-registry', interactive=interactive) crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX, INSTANCE_CRD_KIND_SUFFIX) crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR, INSTANCE_NAME_CRD_PLURAL_SUFFIX, INSTANCE_NAME_CRD_KIND_SUFFIX) from ckan_cloud_operator.providers.solr.manager import zk_list_configs, zk_put_configs print('Checking CKAN Solr config in ZooKeeper') if 'ckan_default' in zk_list_configs(): print('Found ckan_default Solr config') else: print( 'No default Solr config found. Putting CKAN 2.8 config for Solr to ZooKeeper as ckan_default...' ) file_path = os.path.dirname(os.path.abspath(__file__)) root_path = Path(file_path).parent.parent configs_dir = os.path.join(root_path, 'data', 'solr') zk_put_configs(configs_dir) if config_manager.get('disable-centralized-datapushers', configmap_name='global-ckan-config', required=False) != 'yes': from ckan_cloud_operator.datapushers import initialize as datapusher_initialize datapusher_initialize() from ckan_cloud_operator.routers import manager as routers_manager router_name = get_default_instances_router_name() wildcard_ssl_domain = routers_manager.get_default_root_domain() print(f'wildcard_ssl_domain={wildcard_ssl_domain}') allow_wildcard_ssl = routers_manager.get_env_id() == 'p' router = routers_manager.get(router_name, required=False) if router: assert ((allow_wildcard_ssl and router.get( 'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain) or (not allow_wildcard_ssl and not router.get('spec', {})) ), f'invalid router wildcard ssl config: {router}' else: routers_manager.create( router_name, routers_manager.get_traefik_router_spec( wildcard_ssl_domain=wildcard_ssl_domain)) from .storage.manager import initialize as ckan_storage_initialize ckan_storage_initialize(interactive=interactive) from .deployment.manager import initialize as ckan_deployment_initialize ckan_deployment_initialize(interactive=interactive)
def initialize(interactive=False, dry_run=False): if cluster_manager.get_provider_id() == 'minikube': config_manager.set( 'container-spec-overrides', '{"resources":{"limits":{"memory":"1Gi"}}}', configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config') # This is for setting solr's configurations and their defaults # This can be modified from interactive,yaml or with prompt answers on cluster initialize config_manager.interactive_set( { 'sc-cpu': '1', 'sc-mem': '1Gi', 'zk-cpu': '0.2', 'zk-mem': '200Mi', 'zn-cpu': '0.01', 'zn-mem': '0.01Gi', 'sc-cpu-limit': '2.5', 'sc-mem-limit': '8Gi', 'sc-image': 'solr:5.5.5', 'sc-init-image': 'alpine', 'zk-cpu-limit': '0.5', 'zk-mem-limit': '200Mi', 'zk-image': 'gcr.io/google_samples/k8szk:v3', 'zn-cpu-limit': '0.5', 'zn-mem-limit': '0.5Gi', }, secret_name='solr-config', interactive=interactive) registry_secrets = config_manager.interactive_set( {'private-registry': 'n'}, secret_name='ckan-docker-registry', interactive=interactive) if registry_secrets.get('private-registry') == 'y': registry_secrets = config_manager.interactive_set( { 'docker-server': None, 'docker-username': None, 'docker-password': None, 'docker-email': None, 'docker-image-pull-secret-name': 'container-registry', }, secret_name='ckan-docker-registry', interactive=interactive) helm_manager._create_private_container_registry_secret('ckan-cloud') zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run) _config_set('zk-host-names', yaml.dump(zk_host_names, default_flow_style=False)) logs.info(f'Initialized zookeeper: {zk_host_names}') zoonavigator_deployment_name = _apply_zoonavigator_deployment( dry_run=dry_run) logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}') sc_host_names = initialize_solrcloud(zk_host_names, pause_deployment=False, interactive=interactive, dry_run=dry_run) _config_set('sc-host-names', yaml.dump(sc_host_names, default_flow_style=False)) logs.info(f'Initialized solrcloud: {sc_host_names}') solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run) _config_set('sc-main-host-name', solrcloud_host_name) logs.info(f'Initialized solrcloud service: {solrcloud_host_name}') expected_running = len(sc_host_names) + len(zk_host_names) + 1 RETRIES = 40 # ~20 minutes for retry in range(RETRIES): pods = kubectl.get('pods') running = len([ x for x in pods['items'] if x['status']['phase'] == 'Running' and x['metadata']['labels']['app'].startswith( _get_resource_labels(for_deployment=True)['app']) ]) time.sleep(45) logs.info('Waiting for SolrCloud to start... %d/%d' % (running, expected_running)) for x in pods['items']: logs.info(' - %-10s | %s: %s' % (x['metadata'].get('labels', {}).get('app'), x['metadata']['name'], x['status']['phase'])) if running == expected_running: break assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud' _set_provider()
def initialize(interactive=False, dry_run=False): if cluster_manager.get_provider_id() == 'minikube': config_manager.set( 'container-spec-overrides', '{"resources":{"limits":{"memory":"1Gi"}}}', configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config') config_manager.interactive_set( { 'sc-cpu': '1', 'sc-mem': '1Gi', 'zk-cpu': '0.2', 'zk-mem': '200Mi', 'zn-cpu': '0.01', 'zn-mem': '0.01Gi', 'sc-cpu-limit': '2.5', 'sc-mem-limit': '8Gi', 'zk-cpu-limit': '0.5', 'zk-mem-limit': '200Mi', 'zn-cpu-limit': '0.5', 'zn-mem-limit': '0.5Gi', 'sc-suffixes': DEFAULT_SC_SUFFIXES, 'zk-suffixes': DEFAULT_ZK_SUFFIXES }, secret_name='solr-config', interactive=interactive) zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run) _config_set('zk-host-names', yaml.dump(zk_host_names, default_flow_style=False)) logs.info(f'Initialized zookeeper: {zk_host_names}') zoonavigator_deployment_name = _apply_zoonavigator_deployment( dry_run=dry_run) logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}') sc_host_names = initialize_solrcloud(zk_host_names, pause_deployment=False, interactive=interactive, dry_run=dry_run) _config_set('sc-host-names', yaml.dump(sc_host_names, default_flow_style=False)) logs.info(f'Initialized solrcloud: {sc_host_names}') solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run) _config_set('sc-main-host-name', solrcloud_host_name) logs.info(f'Initialized solrcloud service: {solrcloud_host_name}') expected_running = len(sc_host_names) + len(zk_host_names) + 1 RETRIES = 40 # ~20 minutes for retry in range(RETRIES): pods = kubectl.get('pods') running = len([ x for x in pods['items'] if x['status']['phase'] == 'Running' and x['metadata']['labels']['app'].startswith( _get_resource_labels(for_deployment=True)['app']) ]) time.sleep(45) logs.info('Waiting for SolrCloud to start... %d/%d' % (running, expected_running)) for x in pods['items']: logs.info(' - %-10s | %s: %s' % (x['metadata'].get('labels', {}).get('app'), x['metadata']['name'], x['status']['phase'])) if running == expected_running: break assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud' _set_provider()
def initialize(interactive=False): logs.info('env-id is a single character identifier of the environment') logs.info('.e.g p for production, s for staging, d for development') logs.info("For the first cluster it's recommended to use 'p'") default_dns_provider = { 'aws': 'route53', }.get(cluster_manager.get_provider_id(), 'cloudflare') if cluster_manager.get_provider_id() == 'kamatera': from ckan_cloud_operator.providers.cluster.kamatera.management import manager as kamatera_management_manager default_root_domain = kamatera_management_manager.get_management_machine_secrets('RootDomainName') else: default_root_domain = None config_manager.interactive_set( { 'env-id': 'p', 'default-root-domain': default_root_domain, 'dns-provider': default_dns_provider }, configmap_name='routers-config', interactive=interactive ) dns_provider = config_manager.get(key='dns-provider', configmap_name='routers-config') logs.info(dns_provider=dns_provider) if dns_provider == 'cloudflare': if cluster_manager.get_provider_id() == 'kamatera': cloudflare_api_key = config_manager.get( 'CloudflareApiKey', secret_name='cco-kamatera-management-server', namespace='ckan-cloud-operator', required=True ) cloudflare_email = config_manager.get( 'CloudflareEmail', secret_name='cco-kamatera-management-server', namespace='ckan-cloud-operator', required=True ) config_manager.set(values={ 'cloudflare-email': cloudflare_email, 'cloudflare-api-key': cloudflare_api_key }, secret_name='routers-secrets') else: config_manager.interactive_set( { 'cloudflare-email': None, 'cloudflare-api-key': None }, secret_name='routers-secrets', interactive=interactive ) routers_manager.install_crds() infra_router_name = routers_manager.get_default_infra_router_name() default_root_domain = config_manager.get('default-root-domain', configmap_name='routers-config', required=True) router_type = { 'kamatera': 'nginx' }.get(cluster_manager.get_provider_id(), 'traefik') logs.info('Creating infra router', infra_router_name=infra_router_name, default_root_domain=default_root_domain, router_type=router_type) if router_type == 'traefik': router_spec = routers_manager.get_traefik_router_spec( default_root_domain, config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None), config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None), dns_provider=dns_provider ) else: router_spec = routers_manager.get_nginx_router_spec( default_root_domain, config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None), config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None), dns_provider=dns_provider ) routers_manager.create(infra_router_name, router_spec)