def get_provider(submodule, required=True, supported_provider_ids=None, default=None, verbose=False, provider_id=None): if default: required = False if not provider_id: provider_id = get_provider_id(submodule, required=required) if not provider_id: if default: provider_id = default else: return None if verbose: logs.info(f'submodule {submodule} using provider_id {provider_id}') if not supported_provider_ids or provider_id in supported_provider_ids: provider_manager = _get_submodule_ids_provider_or_provider_ids( submodule, provider_id) else: provider_manager = None if provider_manager: return provider_manager else: msg = f' (supported provider ids: {supported_provider_ids})' if supported_provider_ids else '' logs.critical( f'Invalid submodule / provider: {submodule} / {provider_id}{msg}') raise Exception('failed to get provider')
def solr_curl(path, required=False, debug=False, max_retries=15): deployment_name = _get_resource_name(_get_sc_suffixes()[0]) if debug: kubectl.check_call( f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'', use_first_pod=True) else: exitcode, output = kubectl.getstatusoutput( f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'', use_first_pod=True) if exitcode == 0: return output elif required: if max_retries > 0: logs.info( f'Failed to run solr curl: localhost:8983/solr{path} - retring in 30 seconds' ) time.sleep(30) solr_curl(path, required=required, debug=debug, max_retries=max_retries - 1) logs.critical(output) raise Exception( f'Failed to run solr curl: localhost:8983/solr{path}') else: logs.warning(output) return False
def port_forward(db_prefix, all_daemon): if all_daemon: assert not db_prefix and all_daemon == 'I know the risks' subprocess.Popen( ['ckan-cloud-operator', 'db', 'proxy', 'port-forward'], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) for db_prefix in manager.get_provider().get_all_db_prefixes(): subprocess.Popen([ 'ckan-cloud-operator', 'db', 'proxy', 'port-forward', '--db-prefix', db_prefix ], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) else: while True: start_time = datetime.datetime.now() try: manager.start_port_forward(db_prefix=db_prefix) except Exception: traceback.print_exc() end_time = datetime.datetime.now() if (end_time - start_time).total_seconds() < 10: logs.critical('DB Proxy failure') logs.exit_catastrophic_failure() else: logs.warning('Restarting the DB proxy')
def port_forward(): while True: start_time = datetime.datetime.now() try: manager.start_port_forward() except Exception: traceback.print_exc() end_time = datetime.datetime.now() if (end_time - start_time).total_seconds() < 10: logs.critical('DB Proxy failure') logs.exit_catastrophic_failure() else: logs.warning('Restarting the DB proxy')
def deis_instance_delete(instance_id, force): """Permanently delete the instances and all related infrastructure""" failed_instance_ids = [] for id in instance_id: try: cls(id).delete(force) except Exception: traceback.print_exc() failed_instance_ids.append(id) if len(failed_instance_ids) > 0: logs.critical(f'Instance deletion failed for the following instance ids: {failed_instance_ids}') if not force: logs.exit_catastrophic_failure() great_success()
def solr_curl(path, required=False, debug=False): deployment_name = _get_resource_name(_get_sc_suffixes()[0]) if debug: kubectl.check_call(f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'', use_first_pod=True) else: exitcode, output = kubectl.getstatusoutput(f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'', use_first_pod=True) if exitcode == 0: return output elif required: logs.critical(output) raise Exception(f'Failed to run solr curl: localhost:8983/solr{path}') else: logs.warning(output) return False
def solr_curl(path, required=False, debug=False): if is_self_hosted(): return get_provider().solr_curl(path, required=required, debug=debug) else: http_endpoint = get_internal_http_endpoint() if debug: subprocess.check_call(f'curl \'{http_endpoint}{path}\'') else: exitcode, output = subprocess.getstatusoutput(f'curl -s -f \'{http_endpoint}{path}\'') if exitcode == 0: return output elif required: logs.critical(output) raise Exception(f'Failed to run solr curl: {http_endpoint}{path}') else: logs.warning(output) return False
def curl(auth_email, auth_key, urlpart, data=None, method='GET'): logs.info(f'Running Cloudflare curl: {urlpart} {data} {method}') logs.debug(f'{auth_email} / {auth_key}') cmd = [ 'curl', '-s', '-X', method, f'https://api.cloudflare.com/client/v4/{urlpart}' ] cmd += [ '-H', f'X-Auth-Email: {auth_email}', '-H', f'X-Auth-Key: {auth_key}', '-H', 'Content-Type: application/json' ] if data: cmd += ['--data', json.dumps(data)] logs.debug(*cmd) output = subprocess.check_output(cmd) try: return json.loads(output) except Exception: logs.critical(f'Got invalid data from cloudflare curl: {output}') raise
def delete(self, force=False, wait_deleted=False): """ Can run delete multiple time until successful deletion of all components. Uses Kubernetes finalizers to ensure deletion is complete before applying the deletion. """ print(f'Deleting {self.kind} {self.id}') try: assert self.spec has_spec = True except Exception: has_spec = False # this updates deletion timestamp but doesn't delete the object until all finalizers are removed subprocess.call(f'kubectl -n ckan-cloud delete --wait=false {self.kind} {self.id}', shell=True) num_exceptions = 0 if has_spec: for delete_id, delete_code in { 'deployment': lambda: DeisCkanInstanceDeployment(self).delete(), 'envvars': lambda: DeisCkanInstanceEnvvars(self).delete(), 'registry': lambda: DeisCkanInstanceRegistry(self).delete(), 'solr': lambda: DeisCkanInstanceSolr(self).delete(), 'storage': lambda: DeisCkanInstanceStorage(self).delete(), 'namespace': lambda: DeisCkanInstanceNamespace(self).delete(), 'envvars-secret': lambda: kubectl.check_call(f'delete --ignore-not-found secret/{self.id}-envvars'), 'routes': lambda: routers_manager.delete_routes(deis_instance_id=self.id), 'uptime-monitoring': lambda: DeisCkanInstanceUptime(self).delete(self.id) }.items(): try: delete_code() except Exception as e: logs.critical(f'deletion failed for instance {self.id}, submodule: {delete_id}') num_exceptions += 1 else: try: routers_manager.delete_routes(deis_instance_id=self.id) except Exception as e: logs.critical(f'deletion failed for instance {self.id}, submodule: routes') num_exceptions += 1 num_exceptions += 1 if num_exceptions != 0 and not force: raise Exception('instance was not deleted, run with --force to force deletion with risk of remaining infra') else: print(f'Removing finalizers from {self.kind} {self.id}') try: subprocess.check_call( f'kubectl -n ckan-cloud patch {self.kind} {self.id} -p \'{{"metadata":{{"finalizers":[]}}}}\' --type=merge', shell=True ) except Exception: logs.critical(f'failed to remove finalizers: {self.id}') num_exceptions += 1 if not force: raise if wait_deleted and has_spec: logs.info('Waiting 30 seconds for instance to be deleted...') time.sleep(30)