def _init_solr(instance_id):
solr_url = solr_manager.get_internal_http_endpoint()
solr_manager.create_collection(instance_id)
assert solr_url.startswith('https://') and solr_url.endswith('/solr')
host, port = solr_url.replace('https://', '').replace('/solr',
'').split(':')
return host, port
def check_envvars(envvars, deis_instance):
from ckan_cloud_operator.providers.db import manager as db_manager
from ckan_cloud_operator.deis_ckan.ckan import DeisCkanInstanceCKAN
logs.info('waiting for instance to be executable...')
for i in range(20):
try:
DeisCkanInstanceCKAN(deis_instance).exec('true')
break
except Exception:
traceback.print_exc()
logs.warning(f'waiting for instance exec ({i}/20)...')
time.sleep(5)
continue
for url_type, url in {
'site': envvars.get('CKAN_SITE_URL'),
'db': envvars.get('CKAN_SQLALCHEMY_URL'),
'datastore': envvars.get('CKAN__DATASTORE__WRITE_URL'),
'datastore-ro': envvars.get('CKAN__DATASTORE__READ_URL'),
'beaker': envvars.get('CKAN___BEAKER__SESSION__URL'),
'datapusher': envvars.get('CKAN__DATAPUSHER__URL'),
's3': envvars.get('CKANEXT__S3FILESTORE__HOST_NAME'),
'solr': envvars.get('CKAN_SOLR_URL'),
}.items():
logs.info(f'checking envvar: {url_type}')
assert url and 'None' not in url, f'invalid url: {url}'
if url_type in ['site', 'datapusher', 's3']:
check_cluster_url(url,
verify_site_url=url_type == 'site',
deis_instance=deis_instance)
elif url_type in ['db', 'datastore', 'datastore-ro', 'beaker']:
if '@ckan-cloud-provider-db-proxy-pgbouncer.ckan-cloud:5432' in url:
db_manager.check_connection_string(
url.replace(
'ckan-cloud-provider-db-proxy-pgbouncer.ckan-cloud',
'localhost'))
elif url_type == 'solr':
from ckan_cloud_operator.providers.solr import manager as solr_manager
assert url.startswith(solr_manager.get_internal_http_endpoint())
else:
raise Exception(f'unknown url type: {url_type}')
def _update(self):
spec = self.instance.spec
db_name = spec.db['name']
db_password = self.instance.annotations.get_secret('databasePassword')
datastore_name = spec.datastore['name']
datastore_password = self.instance.annotations.get_secret('datastorePassword')
datastore_ro_user = self.instance.annotations.get_secret('datastoreReadonlyUser')
datastore_ro_password = self.instance.annotations.get_secret('datatastoreReadonlyPassword')
db_no_db_proxy = spec.db.get('no-db-proxy') == 'yes'
datastore_no_db_proxy = spec.datastore.get('no-db-proxy') == 'yes'
if db_no_db_proxy or datastore_no_db_proxy:
assert db_no_db_proxy and datastore_no_db_proxy, 'must set both DB and datastore with no-db-proxy'
no_db_proxy = True
else:
no_db_proxy = False
from ckan_cloud_operator.providers.solr import manager as solr_manager
solr_http_endpoint = solr_manager.get_internal_http_endpoint()
solr_collection_name = spec.solrCloudCollection['name']
if 'fromSecret' in spec.envvars:
envvars = kubectl.get(f'secret {spec.envvars["fromSecret"]}')
envvars = yaml.load(kubectl.decode_secret(envvars, 'envvars.yaml'))
elif 'fromGitlab' in spec.envvars:
envvars = CkanGitlab().get_envvars(spec.envvars['fromGitlab'])
else:
raise Exception(f'invalid envvars spec: {spec.envvars}')
assert envvars['CKAN_SITE_ID'] and envvars['CKAN_SITE_URL'] and envvars['CKAN_SQLALCHEMY_URL']
from ckan_cloud_operator.providers.storage import manager as storage_manager
storage_hostname, storage_access_key, storage_secret_key = storage_manager.get_provider().get_credentials()
storage_path_parts = spec.storage['path'].strip('/').split('/')
storage_bucket = storage_path_parts[0]
storage_path = '/'.join(storage_path_parts[1:])
if no_db_proxy:
postgres_host, postgres_port = db_manager.get_internal_unproxied_db_host_port()
logs.info(f'Bypassing db proxy, connecting to DB directly: {postgres_host}:{postgres_port}')
else:
postgres_host, postgres_port = db_manager.get_internal_proxy_host_port()
logs.info(f'Connecting to DB proxy: {postgres_host}:{postgres_port}')
envvars.update(
CKAN_SQLALCHEMY_URL=f"postgresql://{db_name}:{db_password}@{postgres_host}:{postgres_port}/{db_name}",
CKAN___BEAKER__SESSION__URL=f"postgresql://{db_name}:{db_password}@{postgres_host}:{postgres_port}/{db_name}",
CKAN__DATASTORE__READ_URL=f"postgresql://{datastore_ro_user}:{datastore_ro_password}@{postgres_host}:{postgres_port}/{datastore_name}",
CKAN__DATASTORE__WRITE_URL=f"postgresql://{datastore_name}:{datastore_password}@{postgres_host}:{postgres_port}/{datastore_name}",
CKAN_SOLR_URL=f"{solr_http_endpoint}/{solr_collection_name}",
CKANEXT__S3FILESTORE__AWS_STORAGE_PATH=storage_path,
CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=storage_access_key,
CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=storage_secret_key,
CKANEXT__S3FILESTORE__AWS_BUCKET_NAME=storage_bucket,
CKANEXT__S3FILESTORE__HOST_NAME=f'https://{storage_hostname}/',
CKANEXT__S3FILESTORE__REGION_NAME='us-east-1',
CKANEXT__S3FILESTORE__SIGNATURE_VERSION='s3v4',
CKAN__DATAPUSHER__URL=datapusher.get_datapusher_url(envvars.get('CKAN__DATAPUSHER__URL')),
)
# print(yaml.dump(envvars, default_flow_style=False))
self._apply_instance_envvars_overrides(envvars)
envvars = {
k: ('' if v is None else v)
for k,v
in envvars.items()
}
kubectl.update_secret('ckan-envvars', envvars, namespace=self.instance.id)
self.site_url = envvars.get('CKAN_SITE_URL')