def get_provider(submodule,
required=True,
supported_provider_ids=None,
default=None,
verbose=False,
provider_id=None):
if default: required = False
if not provider_id:
provider_id = get_provider_id(submodule, required=required)
if not provider_id:
if default:
provider_id = default
else:
return None
if verbose:
logs.info(f'submodule {submodule} using provider_id {provider_id}')
if not supported_provider_ids or provider_id in supported_provider_ids:
provider_manager = _get_submodule_ids_provider_or_provider_ids(
submodule, provider_id)
else:
provider_manager = None
if provider_manager:
return provider_manager
else:
msg = f' (supported provider ids: {supported_provider_ids})' if supported_provider_ids else ''
logs.critical(
f'Invalid submodule / provider: {submodule} / {provider_id}{msg}')
raise Exception('failed to get provider')
def solr_curl(path, required=False, debug=False, max_retries=15):
deployment_name = _get_resource_name(_get_sc_suffixes()[0])
if debug:
kubectl.check_call(
f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'',
use_first_pod=True)
else:
exitcode, output = kubectl.getstatusoutput(
f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'',
use_first_pod=True)
if exitcode == 0:
return output
elif required:
if max_retries > 0:
logs.info(
f'Failed to run solr curl: localhost:8983/solr{path} - retring in 30 seconds'
)
time.sleep(30)
solr_curl(path,
required=required,
debug=debug,
max_retries=max_retries - 1)
logs.critical(output)
raise Exception(
f'Failed to run solr curl: localhost:8983/solr{path}')
else:
logs.warning(output)
return False
def port_forward(db_prefix, all_daemon):
if all_daemon:
assert not db_prefix and all_daemon == 'I know the risks'
subprocess.Popen(
['ckan-cloud-operator', 'db', 'proxy', 'port-forward'],
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL)
for db_prefix in manager.get_provider().get_all_db_prefixes():
subprocess.Popen([
'ckan-cloud-operator', 'db', 'proxy', 'port-forward',
'--db-prefix', db_prefix
],
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL)
else:
while True:
start_time = datetime.datetime.now()
try:
manager.start_port_forward(db_prefix=db_prefix)
except Exception:
traceback.print_exc()
end_time = datetime.datetime.now()
if (end_time - start_time).total_seconds() < 10:
logs.critical('DB Proxy failure')
logs.exit_catastrophic_failure()
else:
logs.warning('Restarting the DB proxy')
def port_forward():
while True:
start_time = datetime.datetime.now()
try:
manager.start_port_forward()
except Exception:
traceback.print_exc()
end_time = datetime.datetime.now()
if (end_time - start_time).total_seconds() < 10:
logs.critical('DB Proxy failure')
logs.exit_catastrophic_failure()
else:
logs.warning('Restarting the DB proxy')
def deis_instance_delete(instance_id, force):
"""Permanently delete the instances and all related infrastructure"""
failed_instance_ids = []
for id in instance_id:
try:
cls(id).delete(force)
except Exception:
traceback.print_exc()
failed_instance_ids.append(id)
if len(failed_instance_ids) > 0:
logs.critical(f'Instance deletion failed for the following instance ids: {failed_instance_ids}')
if not force:
logs.exit_catastrophic_failure()
great_success()
def solr_curl(path, required=False, debug=False):
deployment_name = _get_resource_name(_get_sc_suffixes()[0])
if debug:
kubectl.check_call(f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'',
use_first_pod=True)
else:
exitcode, output = kubectl.getstatusoutput(f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'',
use_first_pod=True)
if exitcode == 0:
return output
elif required:
logs.critical(output)
raise Exception(f'Failed to run solr curl: localhost:8983/solr{path}')
else:
logs.warning(output)
return False
def solr_curl(path, required=False, debug=False):
if is_self_hosted():
return get_provider().solr_curl(path, required=required, debug=debug)
else:
http_endpoint = get_internal_http_endpoint()
if debug:
subprocess.check_call(f'curl \'{http_endpoint}{path}\'')
else:
exitcode, output = subprocess.getstatusoutput(f'curl -s -f \'{http_endpoint}{path}\'')
if exitcode == 0:
return output
elif required:
logs.critical(output)
raise Exception(f'Failed to run solr curl: {http_endpoint}{path}')
else:
logs.warning(output)
return False
def curl(auth_email, auth_key, urlpart, data=None, method='GET'):
logs.info(f'Running Cloudflare curl: {urlpart} {data} {method}')
logs.debug(f'{auth_email} / {auth_key}')
cmd = [
'curl', '-s', '-X', method,
f'https://api.cloudflare.com/client/v4/{urlpart}'
]
cmd += [
'-H', f'X-Auth-Email: {auth_email}', '-H', f'X-Auth-Key: {auth_key}',
'-H', 'Content-Type: application/json'
]
if data:
cmd += ['--data', json.dumps(data)]
logs.debug(*cmd)
output = subprocess.check_output(cmd)
try:
return json.loads(output)
except Exception:
logs.critical(f'Got invalid data from cloudflare curl: {output}')
raise
def delete(self, force=False, wait_deleted=False):
"""
Can run delete multiple time until successful deletion of all components.
Uses Kubernetes finalizers to ensure deletion is complete before applying the deletion.
"""
print(f'Deleting {self.kind} {self.id}')
try:
assert self.spec
has_spec = True
except Exception:
has_spec = False
# this updates deletion timestamp but doesn't delete the object until all finalizers are removed
subprocess.call(f'kubectl -n ckan-cloud delete --wait=false {self.kind} {self.id}', shell=True)
num_exceptions = 0
if has_spec:
for delete_id, delete_code in {
'deployment': lambda: DeisCkanInstanceDeployment(self).delete(),
'envvars': lambda: DeisCkanInstanceEnvvars(self).delete(),
'registry': lambda: DeisCkanInstanceRegistry(self).delete(),
'solr': lambda: DeisCkanInstanceSolr(self).delete(),
'storage': lambda: DeisCkanInstanceStorage(self).delete(),
'namespace': lambda: DeisCkanInstanceNamespace(self).delete(),
'envvars-secret': lambda: kubectl.check_call(f'delete --ignore-not-found secret/{self.id}-envvars'),
'routes': lambda: routers_manager.delete_routes(deis_instance_id=self.id),
'uptime-monitoring': lambda: DeisCkanInstanceUptime(self).delete(self.id)
}.items():
try:
delete_code()
except Exception as e:
logs.critical(f'deletion failed for instance {self.id}, submodule: {delete_id}')
num_exceptions += 1
else:
try:
routers_manager.delete_routes(deis_instance_id=self.id)
except Exception as e:
logs.critical(f'deletion failed for instance {self.id}, submodule: routes')
num_exceptions += 1
num_exceptions += 1
if num_exceptions != 0 and not force:
raise Exception('instance was not deleted, run with --force to force deletion with risk of remaining infra')
else:
print(f'Removing finalizers from {self.kind} {self.id}')
try:
subprocess.check_call(
f'kubectl -n ckan-cloud patch {self.kind} {self.id} -p \'{{"metadata":{{"finalizers":[]}}}}\' --type=merge',
shell=True
)
except Exception:
logs.critical(f'failed to remove finalizers: {self.id}')
num_exceptions += 1
if not force:
raise
if wait_deleted and has_spec:
logs.info('Waiting 30 seconds for instance to be deleted...')
time.sleep(30)
