def initialize(interactive=False, dry_run=False):
ckan_infra = CkanInfra(required=False)
solr_config = config_manager.interactive_set(
{
'self-hosted': True
},
secret_name='solr-config',
interactive=interactive
)
if is_self_hosted(solr_config):
initialize_self_hosted(interactive=interactive, dry_run=dry_run)
else:
config_manager.interactive_set(
{
'http-endpoint': ckan_infra.SOLR_HTTP_ENDPOINT,
},
secret_name='solr-config',
interactive=interactive
)
config_manager.interactive_set(
{
'num-shards': ckan_infra.SOLR_NUM_SHARDS or '1',
'replication-factor': ckan_infra.SOLR_REPLICATION_FACTOR or '3'
},
secret_name='solr-config',
interactive=interactive
)
def initialize(interactive=False):
config_manager.interactive_set({
'default-storage-bucket': 'ckan',
},
secret_name='ckan-storage-config',
interactive=interactive)
logs.warning('Minio bucket policy was not applied!')
def initialize(interactive=False,
provider_id=None,
storage_suffix=None,
use_existing_disk_name=None,
dry_run=False):
if not provider_id and interactive:
config_manager.interactive_set({'use-cloud-native-storage': True},
secret_name=CONFIG_NAME,
interactive=interactive)
if config_manager.get('use-cloud-native-storage',
secret_name=CONFIG_NAME):
provider_id = get_provider_id()
logs.info(f'Storage init: chosen provider_id: {provider_id}')
provider = get_provider(default=minio_provider_id,
provider_id=provider_id).initialize(
interactive=interactive,
storage_suffix=storage_suffix,
use_existing_disk_name=use_existing_disk_name,
dry_run=dry_run)
if provider:
provider.initialize(interactive=interactive,
storage_suffix=storage_suffix,
use_existing_disk_name=use_existing_disk_name,
dry_run=dry_run)
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'default-storage-bucket': ckan_infra.GCLOUD_STORAGE_BUCKET,
},
secret_name='ckan-storage-config',
interactive=interactive
)
logs.warning('Minio bucket policy was not applied!')
def initialize(interactive=False,
storage_suffix='',
use_existing_disk_name=False,
dry_run=False):
default_zone = cluster_config_get('cluster-compute-zone')
assert default_zone, 'No cluster region specified.'
if interactive and not dry_run:
config_manager.interactive_set({'storage-region': default_zone},
secret_name=CONFIG_NAME,
interactive=interactive)
def config_interactive_set(submodule, provider_id=None, default_values=None, namespace=None,
is_secret=False, suffix=None, from_file=False, interactive=True):
resource_name = get_resource_name(submodule, provider_id=provider_id, suffix=suffix)
config_manager.interactive_set(
default_values,
secret_name=resource_name if is_secret else None,
configmap_name=None if is_secret else resource_name,
namespace=namespace,
from_file=from_file,
interactive=interactive
)
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'env-id': ckan_infra.ROUTERS_ENV_ID,
'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN,
#'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL,
#'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY
},
configmap_name='routers-config',
interactive=interactive)
routers_manager.install_crds()
def config_interactive_set(singular,
name,
default_values=None,
namespace=None,
is_secret=False,
from_file=False):
resource_name = get_resource_name(singular, name)
config_manager.interactive_set(
default_values,
secret_name=resource_name if is_secret else None,
configmap_name=None if is_secret else resource_name,
namespace=namespace,
from_file=from_file)
def initialize(interactive=False):
config_manager.interactive_set(
{
'env-id':
None,
'default-root-domain':
None,
'dns-provider':
'route53' if cluster_manager.get_provider_id() == 'aws' else None
},
configmap_name='routers-config',
interactive=interactive)
dns_provider = get_dns_provider()
if dns_provider.lower() == 'none':
if cluster_manager.get_provider_id() != 'minikube':
return
logs.info(dns_provider=dns_provider)
if dns_provider == 'cloudflare':
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
configmap_name='routers-config',
interactive=interactive)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain',
configmap_name='routers-config',
required=True)
logs.info('Creating infra router',
infra_router_name=infra_router_name,
default_root_domain=default_root_domain)
routers_manager.create(
infra_router_name,
routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email',
configmap_name='routers-config',
required=False,
default=None),
config_manager.get('cloudflare-api-key',
configmap_name='routers-config',
required=False,
default=None),
dns_provider=dns_provider))
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'env-id': ckan_infra.ROUTERS_ENV_ID,
'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN,
'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL,
'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY
},
configmap_name='routers-config',
interactive=interactive
)
routers_manager.install_crds()
routers_manager.create(
routers_manager.get_default_infra_router_name(),
routers_manager.get_traefik_router_spec(
config_manager.get('default-root-domain', configmap_name='routers-config'),
config_manager.get('cloudflare-email', configmap_name='routers-config'),
config_manager.get('cloudflare-api-key', configmap_name='routers-config')
)
)
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
if config_manager.get('disable-centralized-datapushers',
configmap_name='global-ckan-config',
required=False) != 'yes':
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
print(f'wildcard_ssl_domain={wildcard_ssl_domain}')
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
def initialize(interactive=False,
storage_suffix='',
use_existing_disk_name=False,
dry_run=False):
default_zone = aws_config_get('aws-default-region', is_secret=True)
assert default_zone, 'No cluster region specified.'
if interactive and not dry_run:
config_manager.interactive_set({'storage-region': default_zone},
secret_name=CONFIG_NAME,
interactive=interactive)
config_manager.interactive_set({'aws-storage-access-key': None},
secret_name=CONFIG_NAME,
interactive=interactive)
config_manager.interactive_set({'aws-storage-access-secret': None},
secret_name=CONFIG_NAME,
interactive=interactive)
def initialize(interactive=False, storage_suffix='', use_existing_disk_name=False, dry_run=False):
default_zone = cluster_config_get('azure-default-location')
assert default_zone, 'No cluster region specified.'
if interactive and not dry_run:
config_manager.interactive_set(
{'storage-region': default_zone},
secret_name=CONFIG_NAME,
interactive=interactive
)
config_manager.interactive_set(
{'storage-account-name': ''},
secret_name=CONFIG_NAME,
interactive=interactive
)
config_manager.interactive_set(
{'storage-account-key': ''},
secret_name=CONFIG_NAME,
interactive=interactive
)
def initialize(log_kwargs=None, interactive=False):
logs.info('setting label-prefix: ckan-cloud', **(log_kwargs or {}))
config_manager.set('label-prefix', 'ckan-cloud')
config_manager.interactive_set({'short-label-prefix': 'cc'},
interactive=interactive)
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
from ckan_cloud_operator.providers.solr.manager import zk_list_configs, zk_put_configs
print('Checking CKAN Solr config in ZooKeeper')
if 'ckan_default' in zk_list_configs():
print('Found ckan_default Solr config')
else:
print(
'No default Solr config found. Putting CKAN 2.8 config for Solr to ZooKeeper as ckan_default...'
)
file_path = os.path.dirname(os.path.abspath(__file__))
root_path = Path(file_path).parent.parent
configs_dir = os.path.join(root_path, 'data', 'solr')
zk_put_configs(configs_dir)
if config_manager.get('disable-centralized-datapushers',
configmap_name='global-ckan-config',
required=False) != 'yes':
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
print(f'wildcard_ssl_domain={wildcard_ssl_domain}')
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
from .deployment.manager import initialize as ckan_deployment_initialize
ckan_deployment_initialize(interactive=interactive)
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
# This is for setting solr's configurations and their defaults
# This can be modified from interactive,yaml or with prompt answers on cluster initialize
config_manager.interactive_set(
{
'sc-cpu': '1',
'sc-mem': '1Gi',
'zk-cpu': '0.2',
'zk-mem': '200Mi',
'zn-cpu': '0.01',
'zn-mem': '0.01Gi',
'sc-cpu-limit': '2.5',
'sc-mem-limit': '8Gi',
'sc-image': 'solr:5.5.5',
'sc-init-image': 'alpine',
'zk-cpu-limit': '0.5',
'zk-mem-limit': '200Mi',
'zk-image': 'gcr.io/google_samples/k8szk:v3',
'zn-cpu-limit': '0.5',
'zn-mem-limit': '0.5Gi',
},
secret_name='solr-config',
interactive=interactive)
registry_secrets = config_manager.interactive_set(
{'private-registry': 'n'},
secret_name='ckan-docker-registry',
interactive=interactive)
if registry_secrets.get('private-registry') == 'y':
registry_secrets = config_manager.interactive_set(
{
'docker-server': None,
'docker-username': None,
'docker-password': None,
'docker-email': None,
'docker-image-pull-secret-name': 'container-registry',
},
secret_name='ckan-docker-registry',
interactive=interactive)
helm_manager._create_private_container_registry_secret('ckan-cloud')
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
expected_running = len(sc_host_names) + len(zk_host_names) + 1
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len([
x for x in pods['items'] if x['status']['phase'] == 'Running'
and x['metadata']['labels']['app'].startswith(
_get_resource_labels(for_deployment=True)['app'])
])
time.sleep(45)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
config_manager.interactive_set(
{
'sc-cpu': '1',
'sc-mem': '1Gi',
'zk-cpu': '0.2',
'zk-mem': '200Mi',
'zn-cpu': '0.01',
'zn-mem': '0.01Gi',
'sc-cpu-limit': '2.5',
'sc-mem-limit': '8Gi',
'zk-cpu-limit': '0.5',
'zk-mem-limit': '200Mi',
'zn-cpu-limit': '0.5',
'zn-mem-limit': '0.5Gi',
'sc-suffixes': DEFAULT_SC_SUFFIXES,
'zk-suffixes': DEFAULT_ZK_SUFFIXES
},
secret_name='solr-config',
interactive=interactive)
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
expected_running = len(sc_host_names) + len(zk_host_names) + 1
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len([
x for x in pods['items'] if x['status']['phase'] == 'Running'
and x['metadata']['labels']['app'].startswith(
_get_resource_labels(for_deployment=True)['app'])
])
time.sleep(45)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def initialize(interactive=False):
logs.info('env-id is a single character identifier of the environment')
logs.info('.e.g p for production, s for staging, d for development')
logs.info("For the first cluster it's recommended to use 'p'")
default_dns_provider = {
'aws': 'route53',
}.get(cluster_manager.get_provider_id(), 'cloudflare')
if cluster_manager.get_provider_id() == 'kamatera':
from ckan_cloud_operator.providers.cluster.kamatera.management import manager as kamatera_management_manager
default_root_domain = kamatera_management_manager.get_management_machine_secrets('RootDomainName')
else:
default_root_domain = None
config_manager.interactive_set(
{
'env-id': 'p',
'default-root-domain': default_root_domain,
'dns-provider': default_dns_provider
},
configmap_name='routers-config',
interactive=interactive
)
dns_provider = config_manager.get(key='dns-provider', configmap_name='routers-config')
logs.info(dns_provider=dns_provider)
if dns_provider == 'cloudflare':
if cluster_manager.get_provider_id() == 'kamatera':
cloudflare_api_key = config_manager.get(
'CloudflareApiKey',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True
)
cloudflare_email = config_manager.get(
'CloudflareEmail',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True
)
config_manager.set(values={
'cloudflare-email': cloudflare_email,
'cloudflare-api-key': cloudflare_api_key
}, secret_name='routers-secrets')
else:
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
secret_name='routers-secrets',
interactive=interactive
)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain', configmap_name='routers-config', required=True)
router_type = {
'kamatera': 'nginx'
}.get(cluster_manager.get_provider_id(), 'traefik')
logs.info('Creating infra router', infra_router_name=infra_router_name, default_root_domain=default_root_domain, router_type=router_type)
if router_type == 'traefik':
router_spec = routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None),
config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None),
dns_provider=dns_provider
)
else:
router_spec = routers_manager.get_nginx_router_spec(
default_root_domain,
config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None),
config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None),
dns_provider=dns_provider
)
routers_manager.create(infra_router_name, router_spec)
