def initialize(log_kwargs=None, interactive=False):
log_kwargs = log_kwargs or {}
logs.info(f'Installing crds', **log_kwargs)
crds_manager.install_crd(CRD_SINGULAR,
CRD_PLURAL,
CRD_KIND,
hash_names=True)
ckan_infra = CkanInfra(required=False)
if interactive:
providers_manager.config_interactive_set(
PROVIDER_SUBMODULE,
default_values={
'gcloud-storage-import-bucket':
ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
},
suffix='deis-migration')
else:
if not providers_manager.config_get(PROVIDER_SUBMODULE,
key='gcloud-storage-import-bucket',
suffix='deis-migration',
required=False):
providers_manager.config_set(
PROVIDER_SUBMODULE,
values={
'gcloud-storage-import-bucket':
ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
},
suffix='deis-migration')
def initialize(db_prefix=None, interactive=False):
_set_provider()
ckan_infra = CkanInfra(required=False)
default_values = {
'host': ckan_infra.POSTGRES_HOST,
'port': '5432',
'is-private-ip': True,
'admin-user': ckan_infra.POSTGRES_USER,
'admin-password': ckan_infra.POSTGRES_PASSWORD,
'gcloud-sql-instance-name': ckan_infra.GCLOUD_SQL_INSTANCE_NAME,
}
if interactive:
print(
"\n"
"Starting interactive initialization of the gcloudsql db provider\n"
"Please prepare the following values:\n"
"\n"
" - admin db host and credentials\n"
" - gcloudsql instance name and project details\n"
"\n")
_config_interactive_set(default_values,
**_get_config_credentials_kwargs(db_prefix))
config = _config_get(**_get_config_credentials_kwargs(db_prefix))
for key, default_value in default_values.items():
if key not in config and default_value:
_config_set(key, default_value,
**_get_config_credentials_kwargs(db_prefix))
elif not config.get(key):
raise Exception(f'missing key: {key}')
def initialize(interactive=False, dry_run=False):
ckan_infra = CkanInfra(required=False)
solr_config = config_manager.interactive_set(
{
'self-hosted': True
},
secret_name='solr-config',
interactive=interactive
)
if is_self_hosted(solr_config):
initialize_self_hosted(interactive=interactive, dry_run=dry_run)
else:
config_manager.interactive_set(
{
'http-endpoint': ckan_infra.SOLR_HTTP_ENDPOINT,
},
secret_name='solr-config',
interactive=interactive
)
config_manager.interactive_set(
{
'num-shards': ckan_infra.SOLR_NUM_SHARDS or '1',
'replication-factor': ckan_infra.SOLR_REPLICATION_FACTOR or '3'
},
secret_name='solr-config',
interactive=interactive
)
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
if config_manager.get('disable-centralized-datapushers',
configmap_name='global-ckan-config',
required=False) != 'yes':
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
print(f'wildcard_ssl_domain={wildcard_ssl_domain}')
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
def ckan_infra(self):
"""Initialize the infra object, fetching values from Kubernetes secrets
:return: CkanInfra
"""
ckan_infra = getattr(self, '_ckan_infra', None)
if not ckan_infra:
self._ckan_infra = ckan_infra = CkanInfra(required=False)
return ckan_infra
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'default-storage-bucket': ckan_infra.GCLOUD_STORAGE_BUCKET,
},
secret_name='ckan-storage-config',
interactive=interactive
)
logs.warning('Minio bucket policy was not applied!')
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'env-id': ckan_infra.ROUTERS_ENV_ID,
'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN,
#'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL,
#'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY
},
configmap_name='routers-config',
interactive=interactive)
routers_manager.install_crds()
def getstatusoutput(cmd,
project=None,
with_activate=True,
gsutil=False,
ckan_infra=None):
if with_activate: activate()
if not ckan_infra:
ckan_infra = CkanInfra()
if not project:
project = ckan_infra.GCLOUD_AUTH_PROJECT
compute_zone = ckan_infra.GCLOUD_COMPUTE_ZONE
bin = 'gsutil' if gsutil else f'CLOUDSDK_COMPUTE_ZONE={compute_zone} gcloud --project={project}'
return subprocess.getstatusoutput(f'{bin} {cmd}')
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'env-id': ckan_infra.ROUTERS_ENV_ID,
'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN,
'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL,
'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY
},
configmap_name='routers-config',
interactive=interactive
)
routers_manager.install_crds()
routers_manager.create(
routers_manager.get_default_infra_router_name(),
routers_manager.get_traefik_router_spec(
config_manager.get('default-root-domain', configmap_name='routers-config'),
config_manager.get('cloudflare-email', configmap_name='routers-config'),
config_manager.get('cloudflare-api-key', configmap_name='routers-config')
)
)
def deploy_storage_permissions_function():
"""Deploys a serverless function that sets all keys to private"""
ckan_infra = CkanInfra()
bucket_name = ckan_infra.GCLOUD_STORAGE_BUCKET
project_id = ckan_infra.GCLOUD_AUTH_PROJECT
function_name = bucket_name.replace('-', '') + 'permissions'
function_js = PERMISSIONS_FUNCTION_JS(function_name, project_id,
bucket_name)
package_json = PERMISSIONS_FUNCTION_PACKAGE_JSON
with tempfile.TemporaryDirectory() as tmpdir:
with open(f'{tmpdir}/package.json', 'w') as f:
f.write(package_json)
with open(f'{tmpdir}/index.js', 'w') as f:
f.write(function_js)
gcloud.check_call(
f'functions deploy {function_name} '
f'--runtime nodejs6 '
f'--trigger-resource {bucket_name} '
f'--trigger-event google.storage.object.finalize '
f'--source {tmpdir} '
f'--retry '
f'--timeout 30s ',
ckan_infra=ckan_infra)
def _init_router(router_name):
router = kubectl.get(f'CkanCloudRouter {router_name}')
assert router['spec']['type'] == 'nginx'
annotations = CkanRoutersAnnotations(router_name, router)
ckan_infra = CkanInfra()
return router, annotations, ckan_infra
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
from ckan_cloud_operator.providers.solr.manager import zk_list_configs, zk_put_configs
print('Checking CKAN Solr config in ZooKeeper')
if 'ckan_default' in zk_list_configs():
print('Found ckan_default Solr config')
else:
print(
'No default Solr config found. Putting CKAN 2.8 config for Solr to ZooKeeper as ckan_default...'
)
file_path = os.path.dirname(os.path.abspath(__file__))
root_path = Path(file_path).parent.parent
configs_dir = os.path.join(root_path, 'data', 'solr')
zk_put_configs(configs_dir)
if config_manager.get('disable-centralized-datapushers',
configmap_name='global-ckan-config',
required=False) != 'yes':
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
print(f'wildcard_ssl_domain={wildcard_ssl_domain}')
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
from .deployment.manager import initialize as ckan_deployment_initialize
ckan_deployment_initialize(interactive=interactive)
# @main.group()
# def users():
# """Manage ckan-cloud-operator users"""
# pass
#
# ckan_cloud_operator.providers.users.add_cli_commands(click, users, great_success)
@main.group()
def ckan_infra():
"""Manage the centralized infrastructure"""
pass
CkanInfra.add_cli_commands(click, ckan_infra, great_success)
@main.group()
def deis_instance():
"""Manage Deis CKAN instance resources"""
pass
DeisCkanInstance.add_cli_commands(click, deis_instance, great_success)
@main.group()
def routers():
"""Manage CKAN Cloud routers"""
pass