def solr_curl(path, required=False, debug=False, max_retries=15):
deployment_name = _get_resource_name(_get_sc_suffixes()[0])
if debug:
kubectl.check_call(
f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'',
use_first_pod=True)
else:
exitcode, output = kubectl.getstatusoutput(
f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'',
use_first_pod=True)
if exitcode == 0:
return output
elif required:
if max_retries > 0:
logs.info(
f'Failed to run solr curl: localhost:8983/solr{path} - retring in 30 seconds'
)
time.sleep(30)
solr_curl(path,
required=required,
debug=debug,
max_retries=max_retries - 1)
logs.critical(output)
raise Exception(
f'Failed to run solr curl: localhost:8983/solr{path}')
else:
logs.warning(output)
return False
def delete_routes(routes=None,
deis_instance_id=None,
backend_url_target_resource_id=None,
datapusher_name=None,
root_domain=None,
sub_domain=None):
if deis_instance_id:
assert not routes and not backend_url_target_resource_id and not datapusher_name and not root_domain and not sub_domain
routes = get_deis_instance_routes(deis_instance_id)
elif backend_url_target_resource_id:
assert not routes and not datapusher_name and not root_domain and not sub_domain
routes = get_backend_url_routes(backend_url_target_resource_id)
elif datapusher_name:
assert not routes and not root_domain and not sub_domain
routes = get_datapusher_routes(datapusher_name)
elif root_domain or sub_domain:
assert not routes
routes = get_domain_routes(root_domain, sub_domain)
assert routes
delete_route_names = set()
update_router_names = set()
for route in routes:
delete_route_names.add(route['metadata']['name'])
update_router_names.add(route['spec']['router_name'])
if len(delete_route_names) > 0:
delete_route_names = ' '.join(delete_route_names)
kubectl.check_call(f'delete CkanCloudRoute {delete_route_names}')
for router_name in update_router_names:
update(router_name)
def delete(singular, name):
config_delete(singular, name, by_labels=True)
crd_prefix = get_crd_prefix()
_, kind_suffix = _get_plural_kind_suffix(singular)
name = get_resource_name(singular, name)
kubectl.check_call(
f'delete --ignore-not-found {crd_prefix}{kind_suffix} {name}')
def reload():
deployment_app = _get_resource_labels(for_deployment=True)['app']
logs.info('Reloading pgbouncers...')
for pod_name in kubectl.check_output(
f'get pods -l app={deployment_app} --output=custom-columns=name:.metadata.name --no-headers'
).decode().splitlines():
kubectl.check_call(
f'exec {pod_name} -- pgbouncer -q -u pgbouncer -d -R /var/local/pgbouncer/pgbouncer.ini'
)
logs.info(f'{pod_name}: PgBouncer Reloaded')
def start():
print('\n'.join([
'', '\nadmin db credentials:\n' +
str(db_manager.get_admin_db_credentials()),
'\ninternal proxy host / port:\n' +
str(db_manager.get_internal_proxy_host_port()),
'\ninternal unproxied host / port:\n' +
str(db_manager.get_internal_unproxied_db_host_port()), '\n'
]))
deployment_name = _get_resource_name()
kubectl.check_call(f'port-forward deployment/{deployment_name} 8080')
def _annotate(resource, *annotations, overwrite=True):
label_prefix = labels_manager.get_label_prefix()
kind = resource['kind']
namespace = resource['metadata']['namespace']
name = resource['metadata']['name']
cmd = f'annotate {kind} {name}'
for annotation in annotations:
cmd += f' {label_prefix}/{annotation}'
if overwrite:
cmd += ' --overwrite'
kubectl.check_call(cmd, namespace)
def solr_curl(path, required=False, debug=False):
deployment_name = _get_resource_name(_get_sc_suffixes()[0])
if debug:
kubectl.check_call(f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'',
use_first_pod=True)
else:
exitcode, output = kubectl.getstatusoutput(f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'',
use_first_pod=True)
if exitcode == 0:
return output
elif required:
logs.critical(output)
raise Exception(f'Failed to run solr curl: localhost:8983/solr{path}')
else:
logs.warning(output)
return False
def _update_registry_secret():
secret = kubectl.get('secret datapushers-docker-registry', required=False)
if secret:
print(
'Secret already exists, delete to recreate: datapushers-registry-secret'
)
else:
print('Creating datapushers registry secret')
from ckan_cloud_operator.providers.ckan import manager as ckan_manager
docker_server, docker_username, docker_password, docker_email = ckan_manager.get_docker_credentials(
)
kubectl.check_call(
f'create secret docker-registry datapushers-docker-registry '
f'--docker-password={docker_password} '
f'--docker-server={docker_server} '
f'--docker-username={docker_username} '
f'--docker-email={docker_email}')
def delete(self, force=False, wait_deleted=False):
"""
Can run delete multiple time until successful deletion of all components.
Uses Kubernetes finalizers to ensure deletion is complete before applying the deletion.
"""
print(f'Deleting {self.kind} {self.id}')
try:
assert self.spec
has_spec = True
except Exception:
has_spec = False
# this updates deletion timestamp but doesn't delete the object until all finalizers are removed
subprocess.call(f'kubectl -n ckan-cloud delete --wait=false {self.kind} {self.id}', shell=True)
num_exceptions = 0
if has_spec:
for delete_id, delete_code in {
'deployment': lambda: DeisCkanInstanceDeployment(self).delete(),
'envvars': lambda: DeisCkanInstanceEnvvars(self).delete(),
'registry': lambda: DeisCkanInstanceRegistry(self).delete(),
'solr': lambda: DeisCkanInstanceSolr(self).delete(),
'storage': lambda: DeisCkanInstanceStorage(self).delete(),
'namespace': lambda: DeisCkanInstanceNamespace(self).delete(),
'envvars-secret': lambda: kubectl.check_call(f'delete --ignore-not-found secret/{self.id}-envvars'),
'routes': lambda: routers_manager.delete_routes(deis_instance_id=self.id),
'uptime-monitoring': lambda: DeisCkanInstanceUptime(self).delete(self.id)
}.items():
try:
delete_code()
except Exception as e:
logs.critical(f'deletion failed for instance {self.id}, submodule: {delete_id}')
num_exceptions += 1
else:
try:
routers_manager.delete_routes(deis_instance_id=self.id)
except Exception as e:
logs.critical(f'deletion failed for instance {self.id}, submodule: routes')
num_exceptions += 1
num_exceptions += 1
if num_exceptions != 0 and not force:
raise Exception('instance was not deleted, run with --force to force deletion with risk of remaining infra')
else:
print(f'Removing finalizers from {self.kind} {self.id}')
try:
subprocess.check_call(
f'kubectl -n ckan-cloud patch {self.kind} {self.id} -p \'{{"metadata":{{"finalizers":[]}}}}\' --type=merge',
shell=True
)
except Exception:
logs.critical(f'failed to remove finalizers: {self.id}')
num_exceptions += 1
if not force:
raise
if wait_deleted and has_spec:
logs.info('Waiting 30 seconds for instance to be deleted...')
time.sleep(30)
def delete(instance_id, instance):
namespace = instance["spec"]["namespace"]
storage_class = instance["spec"]["storageclass"]
kubectl.check_call("delete deployment %s" % instance_id,
namespace=namespace)
kubectl.check_call("delete RoleBinding leader-locking-%s" % instance_id,
namespace=namespace)
kubectl.check_call("delete Role leader-locking-%s" % instance_id,
namespace=namespace)
kubectl.check_call("delete ClusterRoleBinding run-%s" % instance_id,
namespace=namespace)
kubectl.check_call("delete ClusterRole %s-runner" % instance_id,
namespace=namespace)
kubectl.check_call("delete ServiceAccount %s" % instance_id,
namespace=namespace)
kubectl.check_call("delete StorageClass %s" % storage_class,
namespace=namespace)
def _delete(cache_key, exists_ok=False):
config_type, namespace, config_name = _parse_cache_key(cache_key)
assert config_type in ['configmap', 'secret'], f'Invalid config type: {config_type}'
ignore_not_found = ' --ignore-not-found' if exists_ok else ''
kubectl.check_call(f'delete{ignore_not_found} {config_type} {config_name}')
def kubectl_command(arg):
from ckan_cloud_operator import kubectl
kubectl.check_call(' '.join(arg))
