def _init_namespace(instance_id, dry_run=False):
logs.debug('Initializing helm-based instance deployment namespace',
namespace=instance_id)
if kubectl.get('ns', instance_id, required=False):
logs.info(f'instance namespace already exists ({instance_id})')
else:
logs.info(f'creating instance namespace ({instance_id})')
kubectl.apply(kubectl.get_resource('v1', 'Namespace', instance_id, {}),
dry_run=dry_run)
service_account_name = f'ckan-{instance_id}-operator'
logs.debug('Creating service account',
service_account_name=service_account_name)
if not dry_run:
kubectl_rbac_driver.update_service_account(
f'ckan-{instance_id}-operator', {}, namespace=instance_id)
role_name = f'ckan-{instance_id}-operator-role'
logs.debug('Creating role and binding to the service account',
role_name=role_name)
if not dry_run:
kubectl_rbac_driver.update_role(role_name, {}, [{
"apiGroups": ["*"],
"resources":
['secrets', 'pods', 'pods/exec', 'pods/portforward'],
"verbs": ["list", "get", "create"]
}],
namespace=instance_id)
kubectl_rbac_driver.update_role_binding(
name=f'ckan-{instance_id}-operator-rolebinding',
role_name=f'ckan-{instance_id}-operator-role',
namespace=instance_id,
service_account_name=f'ckan-{instance_id}-operator',
labels={})
def deploy():
"""Deploys a proxy inside the cluster which allows to access the centralized solr without authentication"""
labels = {'app': 'ckan-cloud-solrcloud-proxy'}
solr_url = parse_url(solr_manager.get_internal_http_endpoint())
scheme = solr_url.scheme
hostname = solr_url.hostname
port = solr_url.port
solr_user, solr_password = solr_url.auth.split(':')
if not port:
port = '443' if scheme == 'https' else '8983'
kubectl.update_secret(
'solrcloud-proxy', {
'SOLR_URL': f'{scheme}://{hostname}:{port}',
'SOLR_USER': solr_user,
'SOLR_PASSWORD': solr_password
})
kubectl.apply(
kubectl.get_deployment(
'solrcloud-proxy', labels, {
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {
'type': 'RollingUpdate',
},
'template': {
'metadata': {
'labels': labels,
'annotations': {
'ckan-cloud/operator-timestamp':
str(datetime.datetime.now())
}
},
'spec': {
'containers': [{
'name':
'solrcloud-proxy',
'image':
'viderum/ckan-cloud-operator-solrcloud-proxy',
'envFrom': [{
'secretRef': {
'name': 'solrcloud-proxy'
}
}],
'ports': [{
'containerPort': 8983
}],
}]
}
}
}))
service = kubectl.get_resource('v1', 'Service', 'solrcloud-proxy', labels)
service['spec'] = {
'ports': [{
'name': '8983',
'port': 8983
}],
'selector': labels
}
kubectl.apply(service)
def _apply_zookeeper_headless_service(dry_run=False):
headless_service_name = _get_resource_name('zk-headless')
kubectl.apply(kubectl.get_resource(
'v1',
'Service',
headless_service_name,
_get_resource_labels(suffix='zk-headless'),
spec={
'clusterIP':
'None',
'ports': [{
'name': 'client',
'port': 2181,
'protocol': 'TCP',
'targetPort': 2181
}, {
'name': 'server',
'port': 2888,
'protocol': 'TCP',
'targetPort': 2888
}, {
'name': 'leader-election',
'port': 3888,
'protocol': 'TCP',
'targetPort': 3888
}],
'selector': {
'app':
_get_resource_labels(for_deployment=True, suffix='zk')['app']
}
}),
dry_run=dry_run)
return headless_service_name
def _apply_solrcloud_headless_service(dry_run=False):
headless_service_name = _get_resource_name('sc-headless')
kubectl.apply(kubectl.get_resource(
'v1',
'Service',
headless_service_name,
_get_resource_labels(suffix='sc-headless'),
spec={
'clusterIP':
'None',
'ports': [{
'name': 'solr',
'port': 8983,
'protocol': 'TCP',
'targetPort': 8983
}, {
'name': 'stop',
'port': 7983,
'protocol': 'TCP',
'targetPort': 7983
}, {
'name': 'rmi',
'port': 18983,
'protocol': 'TCP',
'targetPort': 18983
}],
'selector': {
'app':
_get_resource_labels(for_deployment=True, suffix='sc')['app']
}
}),
dry_run=dry_run)
return headless_service_name
def get_resource(singular, name, extra_label_suffixes=None, **kwargs):
crd_group = get_crd_group()
_, kind_suffix = _get_plural_kind_suffix(singular)
resource = kubectl.get_resource(
f'{crd_group}/v1', get_resource_kind(singular),
get_resource_name(singular, name),
get_resource_labels(singular,
name,
extra_label_suffixes=extra_label_suffixes))
resource.update(**kwargs)
return resource
def create(name, image, config, router_name=None):
labels = _get_labels(name)
datapusher = kubectl.get_resource('stable.viderum.com/v1',
'CkanCloudDatapusher', name, labels)
datapusher['spec'] = {'image': image, 'config': config}
kubectl.apply(datapusher)
if router_name:
routers_manager.create_subdomain_route(router_name, {
'target-type': 'datapusher',
'datapusher-name': name
})
def create_subdomain_route(router_name, route_spec):
target_type = route_spec['target-type']
sub_domain = route_spec.get('sub-domain')
root_domain = route_spec.get('root-domain')
if target_type == 'datapusher':
target_resource_id = route_spec['datapusher-name']
elif target_type == 'deis-instance':
target_resource_id = route_spec['deis-instance-id']
elif target_type == 'backend-url':
target_resource_id = route_spec['target-resource-id']
else:
raise Exception(f'Invalid route spec: {route_spec}')
sub_domain, root_domain = _get_default_sub_root_domain(
sub_domain, root_domain, target_resource_id)
route_name = 'cc' + hashlib.sha3_224(
f'{target_type} {target_resource_id} {root_domain} {sub_domain}'.
encode()).hexdigest()
router, spec, router_type, annotations, labels, router_type_config = _init_router(
router_name)
route_type = f'{target_type}-subdomain'
labels.update(
**{
'ckan-cloud/route-type': route_type,
'ckan-cloud/route-root-domain': root_domain,
'ckan-cloud/route-sub-domain': sub_domain,
'ckan-cloud/route-target-type': target_type,
'ckan-cloud/route-target-resource-id': target_resource_id,
})
spec = {
'name': route_name,
'type': route_type,
'root-domain': root_domain,
'sub-domain': sub_domain,
'router_name': router_name,
'router_type': router_type,
'route-target-type': target_type,
'route-target-resource-id': target_resource_id,
}
if target_type == 'datapusher':
labels['ckan-cloud/route-datapusher-name'] = spec[
'datapusher-name'] = route_spec['datapusher-name']
elif target_type == 'deis-instance':
labels['ckan-cloud/route-deis-instance-id'] = spec[
'deis-instance-id'] = route_spec['deis-instance-id']
elif target_type == 'backend-url':
spec['backend-url'] = route_spec['backend-url']
route = kubectl.get_resource('stable.viderum.com/v1',
'CkanCloudRoute',
route_name,
labels,
spec=spec)
kubectl.apply(route)
def create(router_name, router_spec):
router_type = router_spec.get('type')
default_root_domain = router_spec.get('default-root-domain')
assert router_type in ROUTER_TYPES and default_root_domain, f'Invalid router spec: {router_spec}'
get(router_name, only_dns=True, failfast=True)
print(f'Creating CkanCloudRouter {router_name} {router_spec}')
labels = _get_labels(router_name, router_type)
router = kubectl.get_resource('stable.viderum.com/v1', 'CkanCloudRouter', router_name, labels,
spec=dict(router_spec, **{'type': router_type}))
router_manager = ROUTER_TYPES[router_type]['manager']
router = router_manager.create(router)
annotations = CkanRoutersAnnotations(router_name, router)
annotations.json_annotate('default-root-domain', default_root_domain)
def update_service(name, labels):
labels.update(**_get_labels(name))
service = kubectl.get_resource('v1', 'Service', get_service_name(name),
labels)
service['spec'] = {
'ports': [{
'name': '8000',
'port': 8000
}],
'selector': {
'ckan-cloud/datapusher-name': name,
'app': 'datapusher',
}
}
kubectl.apply(service)
def _apply_solrcloud_service():
service_name = _get_resource_name('sc')
kubectl.apply(kubectl.get_resource(
'v1', 'Service',
service_name,
_get_resource_labels(suffix='sc'),
spec={
'ports': [
{'name': 'solr', 'port': 8983, 'protocol': 'TCP', 'targetPort': 8983},
],
'selector': {
'app': _get_resource_labels(for_deployment=True, suffix='sc')['app']
}
}
))
return service_name
def deploy_ckan_infra_solr_proxy():
"""Deploys a proxy inside the cluster which allows to access the centralized solr without authentication"""
labels = {'app': 'ckan-cloud-solrcloud-proxy'}
infra = cls()
solr_url = urlparse(infra.SOLR_HTTP_ENDPOINT)
scheme = solr_url.scheme
hostname = solr_url.hostname
port = solr_url.port
if not port:
port = '443' if scheme == 'https' else '8983'
kubectl.update_secret('solrcloud-proxy', {
'SOLR_URL': f'{scheme}://{hostname}:{port}',
'SOLR_USER': infra.SOLR_USER,
'SOLR_PASSWORD': infra.SOLR_PASSWORD
})
kubectl.apply(kubectl.get_deployment('solrcloud-proxy', labels, {
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {'type': 'RollingUpdate', },
'template': {
'metadata': {
'labels': labels,
'annotations': {
'ckan-cloud/operator-timestamp': str(datetime.datetime.now())
}
},
'spec': {
'containers': [
{
'name': 'solrcloud-proxy',
'image': 'orihoch/ckan-cloud-operator-solrcloud-proxy',
'envFrom': [{'secretRef': {'name': 'solrcloud-proxy'}}],
'ports': [{'containerPort': 8983}],
}
]
}
}
}))
service = kubectl.get_resource('v1', 'Service', 'solrcloud-proxy', labels)
service['spec'] = {
'ports': [
{'name': '8983', 'port': 8983}
],
'selector': labels
}
kubectl.apply(service)
def create(router_name, router_spec):
from ckan_cloud_operator.providers.cluster.manager import get_or_create_multi_user_volume_claim
from ckan_cloud_operator.routers.traefik.deployment import get_label_suffixes
router_type = router_spec.get('type')
default_root_domain = router_spec.get('default-root-domain')
assert router_type in ROUTER_TYPES and default_root_domain, f'Invalid router spec: {router_spec}'
get(router_name, only_dns=True, failfast=True)
print(f'Creating CkanCloudRouter {router_name} {router_spec}')
labels = _get_labels(router_name, router_type)
router = kubectl.get_resource('stable.viderum.com/v1', 'CkanCloudRouter', router_name, labels,
spec=dict(router_spec, **{'type': router_type}))
router_manager = ROUTER_TYPES[router_type]['manager']
router = router_manager.create(router)
get_or_create_multi_user_volume_claim(get_label_suffixes(router_name, router_type))
annotations = CkanRoutersAnnotations(router_name, router)
annotations.json_annotate('default-root-domain', default_root_domain)
def _apply_service(storage_suffix=None, dry_run=False):
kubectl.apply(kubectl.get_resource(
'v1',
'Service',
_get_resource_name(suffix=storage_suffix),
_get_resource_labels(suffix=storage_suffix),
spec={
'ports': [{
'name': '9000',
'port': 9000
}],
'selector': {
'app':
_get_resource_labels(for_deployment=True,
suffix=storage_suffix)['app']
}
}),
dry_run=dry_run)
def pre_deployment_hook(route, labels):
name, spec = _init_route(route)
deis_instance_id = spec['deis-instance-id']
if kubectl.get(f'ns {deis_instance_id}', required=False):
print(
f'updating route name {name} for deis instance {deis_instance_id}')
route_service = kubectl.get_resource('v1',
'Service',
name,
labels,
namespace=deis_instance_id)
route_service['spec'] = {
'ports': [{
'name': '5000',
'port': 5000
}],
'selector': {
'app': 'ckan'
}
}
kubectl.apply(route_service)
def _init_namespace(instance_id):
if kubectl.get('ns', instance_id, required=False):
logs.info(f'instance namespace already exists ({instance_id})')
else:
logs.info(f'creating instance namespace ({instance_id})')
kubectl.apply(kubectl.get_resource('v1', 'Namespace', instance_id, {}))
kubectl_rbac_driver.update_service_account(
f'ckan-{instance_id}-operator', {}, namespace=instance_id)
kubectl_rbac_driver.update_role(
f'ckan-{instance_id}-operator-role', {}, [{
"apiGroups": ["*"],
"resources":
['secrets', 'pods', 'pods/exec', 'pods/portforward'],
"verbs": ["list", "get", "create"]
}],
namespace=instance_id)
kubectl_rbac_driver.update_role_binding(
name=f'ckan-{instance_id}-operator-rolebinding',
role_name=f'ckan-{instance_id}-operator-role',
namespace=instance_id,
service_account_name=f'ckan-{instance_id}-operator',
labels={})
def update_service_account(service_account_name, labels):
kubectl.apply(
kubectl.get_resource('v1', 'ServiceAccount', service_account_name,
labels))
def deploy_gcs_minio_proxy(router_name):
"""Deploys a minio proxy (AKA gateway) for access to google storage"""
labels = {'app': 'ckan-cloud-gcsminio-proxy'}
if not kubectl.get('secret gcsminio-proxy-credentials', required=False):
print('Creating minio credentials')
minio_access_key = binascii.hexlify(os.urandom(8)).decode()
minio_secret_key = binascii.hexlify(os.urandom(12)).decode()
kubectl.update_secret(
'gcsminio-proxy-credentials', {
'MINIO_ACCESS_KEY': minio_access_key,
'MINIO_SECRET_KEY': minio_secret_key,
})
kubectl.apply(
kubectl.get_deployment(
'gcsminio-proxy', labels, {
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {
'type': 'RollingUpdate',
},
'template': {
'metadata': {
'labels': labels,
'annotations': {
'ckan-cloud/operator-timestamp':
str(datetime.datetime.now())
}
},
'spec': {
'containers': [{
'name':
'minio',
'image':
'orihoch/ckan-cloud-operator-gcsminio-proxy',
'env': [{
'name':
'GOOGLE_APPLICATION_CREDENTIALS',
'value':
'/gcloud-credentials/credentials.json'
}],
'envFrom': [{
'secretRef': {
'name': 'gcsminio-proxy-credentials'
}
}],
'ports': [{
'containerPort': 9000
}],
'volumeMounts': [
{
'name': 'gcloud-credentials',
'mountPath':
'/gcloud-credentials/credentials.json',
'subPath': 'GCLOUD_SERVICE_ACCOUNT_JSON'
},
],
}],
'volumes': [
{
'name': 'gcloud-credentials',
'secret': {
'secretName': 'ckan-infra'
}
},
]
}
}
}))
service = kubectl.get_resource('v1', 'Service', 'gcsminio-proxy', labels)
service['spec'] = {
'ports': [{
'name': '9000',
'port': 9000
}],
'selector': labels
}
kubectl.apply(service)
if not routers_manager.get_backend_url_routes('gcs-minio'):
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'backend-url',
'target-resource-id': 'gcs-minio',
'backend-url': 'http://gcsminio-proxy.ckan-cloud:9000',
'sub-domain': 'default',
'root-domain': 'default',
})
routers_manager.update(router_name, wait_ready=True)
def _update(router_name, spec, annotations, routes):
resource_name = _get_resource_name(router_name)
router_type = spec['type']
cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials()
external_domains = spec.get('external-domains')
dns_provider = spec.get('dns-provider', 'cloudflare')
logs.info('updating traefik deployment',
resource_name=resource_name,
router_type=router_type,
cloudflare_email=cloudflare_email,
cloudflare_auth_key_len=len(cloudflare_auth_key)
if cloudflare_auth_key else 0,
external_domains=external_domains,
dns_provider=dns_provider)
kubectl.apply(
kubectl.get_configmap(
resource_name, get_labels(router_name, router_type), {
'traefik.toml':
toml.dumps(
traefik_router_config.get(
routes,
cloudflare_email,
enable_access_log=bool(spec.get('enable-access-log')),
wildcard_ssl_domain=spec.get('wildcard-ssl-domain'),
external_domains=external_domains,
dns_provider=dns_provider,
force=True))
}))
domains = {}
httpauth_secrets = []
for route in routes:
root_domain, sub_domain = routes_manager.get_domain_parts(route)
domains.setdefault(root_domain, []).append(sub_domain)
routes_manager.pre_deployment_hook(
route, get_labels(router_name, router_type))
if route['spec'].get('httpauth-secret') and route['spec'][
'httpauth-secret'] not in httpauth_secrets:
httpauth_secrets.append(route['spec']['httpauth-secret'])
load_balancer = kubectl.get_resource('v1', 'Service',
f'loadbalancer-{resource_name}',
get_labels(router_name, router_type))
load_balancer['spec'] = {
'ports': [
{
'name': '80',
'port': 80
},
{
'name': '443',
'port': 443
},
],
'selector': {
'app': get_labels(router_name, router_type,
for_deployment=True)['app']
},
'type': 'LoadBalancer'
}
kubectl.apply(load_balancer)
load_balancer_ip = get_load_balancer_ip(router_name)
print(f'load balancer ip: {load_balancer_ip}')
from ckan_cloud_operator.providers.routers import manager as routers_manager
if external_domains:
from ckan_cloud_operator.providers.routers import manager as routers_manager
external_domains_router_root_domain = routers_manager.get_default_root_domain(
)
env_id = routers_manager.get_env_id()
assert router_name.startswith(
'prod-'), f'invalid external domains router name: {router_name}'
external_domains_router_sub_domain = f'cc-{env_id}-{router_name}'
routers_manager.update_dns_record(dns_provider,
external_domains_router_sub_domain,
external_domains_router_root_domain,
load_balancer_ip, cloudflare_email,
cloudflare_auth_key)
else:
for root_domain, sub_domains in domains.items():
for sub_domain in sub_domains:
routers_manager.update_dns_record(dns_provider, sub_domain,
root_domain,
load_balancer_ip,
cloudflare_email,
cloudflare_auth_key)
kubectl.apply(
kubectl.get_deployment(
resource_name,
get_labels(router_name, router_type, for_deployment=True),
_get_deployment_spec(
router_name,
router_type,
annotations,
image=('traefik:1.7' if
(external_domains
or len(httpauth_secrets) > 0) else None),
httpauth_secrets=httpauth_secrets,
dns_provider=dns_provider)))
def update_service_account(service_account_name, labels, namespace=None):
service_account = kubectl.get_resource('v1', 'ServiceAccount',
service_account_name, labels)
if namespace:
service_account['metadata']['namespace'] = namespace
kubectl.apply(service_account)
def _update(router_name, spec, annotations, routes):
resource_name = _get_resource_name(router_name)
router_type = spec['type']
cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials()
external_domains = spec.get('external-domains')
kubectl.apply(
kubectl.get_configmap(
resource_name, get_labels(router_name, router_type), {
'traefik.toml':
toml.dumps(
traefik_router_config.get(
routes,
cloudflare_email,
wildcard_ssl_domain=spec.get('wildcard-ssl-domain'),
external_domains=external_domains))
}))
domains = {}
for route in routes:
root_domain, sub_domain = routes_manager.get_domain_parts(route)
domains.setdefault(root_domain, []).append(sub_domain)
routes_manager.pre_deployment_hook(
route, get_labels(router_name, router_type))
load_balancer = kubectl.get_resource('v1', 'Service',
f'loadbalancer-{resource_name}',
get_labels(router_name, router_type))
load_balancer['spec'] = {
'ports': [
{
'name': '80',
'port': 80
},
{
'name': '443',
'port': 443
},
],
'selector': {
'app': get_labels(router_name, router_type,
for_deployment=True)['app']
},
'type': 'LoadBalancer'
}
kubectl.apply(load_balancer)
load_balancer_ip = get_load_balancer_ip(router_name)
print(f'load balancer ip: {load_balancer_ip}')
if external_domains:
from ckan_cloud_operator.providers.routers import manager as routers_manager
external_domains_router_root_domain = routers_manager.get_default_root_domain(
)
env_id = routers_manager.get_env_id()
assert router_name.startswith(
'prod-'), f'invalid external domains router name: {router_name}'
external_domains_router_sub_domain = f'cc-{env_id}-{router_name}'
cloudflare.update_a_record(
cloudflare_email, cloudflare_auth_key,
external_domains_router_root_domain,
f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}',
load_balancer_ip)
else:
for root_domain, sub_domains in domains.items():
for sub_domain in sub_domains:
cloudflare.update_a_record(cloudflare_email,
cloudflare_auth_key, root_domain,
f'{sub_domain}.{root_domain}',
load_balancer_ip)
kubectl.apply(
kubectl.get_deployment(
resource_name,
get_labels(router_name, router_type, for_deployment=True),
_get_deployment_spec(
router_name,
router_type,
annotations,
image='traefik:1.7' if external_domains else None)))
