def apply_yaml_from_template(self, namespace, k8s_template_file, dict_env): yaml_path, _ = util.load_yaml_config_template(k8s_template_file, dict_env) hiss.sub_echo('Create %s successfully' % yaml_path) # Execute yaml hiss.echo('Apply yaml file') stream = open(yaml_path, 'r') docs = yaml.safe_load_all(stream) success = True for doc in docs: try: if doc['kind'] == 'Service': self.coreApi.create_namespaced_service(namespace, body=doc) continue except ApiException as e: print("Service already deployed!") continue try: if doc['kind'] == 'StatefulSet': self.appsApi.create_namespaced_stateful_set( namespace, body=doc) self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace) if doc['kind'] == 'Deployment': self.appsApi.create_namespaced_deployment( namespace, body=doc) self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace) if doc['kind'] == 'Job': self.batchApi.create_namespaced_job(namespace, body=doc) self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace, check_job_success=True) except ApiException as e: print("Exception when apply yaml: %s\n" % e) success = False # self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace) return success
def setup_explorer(): hiss.echo('Generate explorer config') config = generate_explorer_config() create_explorer_config_in_efs(json.dumps(config)) hiss.echo('Deploy explorer') # Get domain explorer_namespace = get_namespace() # Create temp folder & namespace settings.k8s.prereqs(explorer_namespace) dict_env = { 'DOMAIN': explorer_namespace, 'DATABASE_PASSWORD': '******', 'EFS_SERVER': settings.EFS_SERVER, 'EFS_PATH': settings.EFS_PATH, 'EFS_EXTEND': settings.EFS_EXTEND } # Deploy explorer db sts explorer_template = '%s/explorer/explorer-deployment.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=explorer_namespace, k8s_template_file=explorer_template, dict_env=dict_env) # Deploy explorer db svc explorer_svc_template = '%s/explorer/explorer-service.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=explorer_namespace, k8s_template_file=explorer_svc_template, dict_env=dict_env)
def mamba(config, set_default): # Setup all shared global utilities in settings module settings.init(config, set_default) if mamba.invoke_without_command: hiss.rattle('Initialize mamba') hiss.echo('Successfully with \'%s\' on the %s!' % (config, settings.EKS_CLUSTER_NAME))
def load_yaml_config_template(yaml_template_path, dict_env): new_yaml_file_name = yaml_template_path.split('/').pop() # Load template file with open(yaml_template_path, 'r') as sources: lines = sources.readlines() out_data = [] # Replace variable for line in lines: out_line = line for key, value in dict_env.items(): out_line = re.sub(r'{{%s}}' % key, value, out_line) out_data.append(out_line) # Get time now current_time = split_timenow_utc() # Create yaml_path hiss.echo('Create yaml file') yaml_path = '%s/%s/%s_%s' % (get_temp_path(), current_time[0], current_time[1], new_yaml_file_name) # Write yaml -> yaml_path with open(yaml_path, "w") as sources: for line in out_data: sources.write(line) # Read new yaml with open(yaml_path, 'r') as ymlfile: config = yaml.load_all(ymlfile) return yaml_path, config
def setup_prometheus(): hiss.echo('Generate prometheus config') config_path = generate_prom_config() if not cp_config_efs(config_path): return util.Result(success=False, msg='Cannot copy prom config to efs pod') hiss.echo('Deploy prometheus') # Get domain prom_namespace = get_namespace() # Create temp folder & namespace settings.k8s.prereqs(prom_namespace) dict_env = { 'DOMAIN': prom_namespace, 'EFS_SERVER': settings.EFS_SERVER, 'EFS_PATH': settings.EFS_PATH, 'EFS_EXTEND': settings.EFS_EXTEND, 'PVS_PATH': settings.PVS_PATH } # Deploy prometheus sts prom_template = '%s/prometheus/prometheus-stateful.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=prom_namespace, k8s_template_file=prom_template, dict_env=dict_env) prom_svc_template = '%s/prometheus/prometheus-service-stateful.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=prom_namespace, k8s_template_file=prom_svc_template, dict_env=dict_env)
def delete_stateful(self, name, namespace, delete_pvc=False): action = 'Delete' if delete_pvc == True: action = 'Terminate' read_result = self.read_stateful_set(name, namespace) if read_result.success == False: return read_result volume_claim_templates = read_result.data.spec.volume_claim_templates if volume_claim_templates != None: for pvt in volume_claim_templates: pvt_name = pvt.metadata.name list_pvc = self.find_pvc(namespace, keyword=pvt_name) for pvc in list_pvc: hiss.echo('Delete pvc %s ' % pvc) self.delete_persistent_volume_claim( name=pvc, namespace=namespace) try: body = client.V1DeleteOptions(propagation_policy='Background') api_response = self.appsApi.delete_namespaced_stateful_set( name, namespace, body=body) self.check_pod_status_by_keyword(keyword=name, namespace=namespace, is_delete=True) hiss.echo('%s stateful set %s on namespace %s success' % (action, name, namespace)) return util.Result(success=True, data=api_response) except ApiException as e: err_msg = "Exception when calling AppsV1Api->delete_namespaced_stateful_set: %s\n" % e return util.Result(success=hiss.hiss(err_msg), msg=err_msg)
def setup_peer(peer, index): # Get domain domain = util.get_domain(peer) # Create temp folder & namespace settings.k8s.prereqs(domain) dict_env = { 'PEER_ORG': peer, 'PEER_DOMAIN': domain, 'PEER_INDEX': index, 'EFS_SERVER': settings.EFS_SERVER, 'EFS_PATH': settings.EFS_PATH, 'EFS_EXTEND': settings.EFS_EXTEND } peer_stateful = '%s/peer-sts/peer-stateful.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=domain, k8s_template_file=peer_stateful, dict_env=dict_env) peer_service = '%s/peer-sts/peer-service-stateful.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=domain, k8s_template_file=peer_service, dict_env=dict_env) num_peers = int(settings.NUM_PEERS) if (num_peers > 2): hiss.echo('NUM_PEER > 2: Does not support deployment nlb service') if (util.get_peer_external_domain(peer, index) != ''): peer_service_nlb = '%s/peer-sts/peer-service-nlb.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=domain, k8s_template_file=peer_service_nlb, dict_env=dict_env)
def detect_deployed_efs_pod(): hiss.echo('Looking for deployed efs pod...') efs_pod = '' efs_pod_cmd = 'kubectl get pod -n default | grep test-efs | awk \'{print $1}\'' detected_efs_pod = subprocess.check_output(efs_pod_cmd, shell=True) if detected_efs_pod: efs_pod = detected_efs_pod.decode().strip() return efs_pod
def detect_deployed_efs_path(): hiss.echo('Looking for deployed efs path...') efs_path = '' efs_path_cmd = 'kubectl get pvc | grep efs | awk \'{print $3}\'' detected_efs_path = subprocess.check_output(efs_path_cmd, shell=True) if detected_efs_path: efs_path = detected_efs_path.decode().strip() return efs_path
def delete_job(self, name, namespace): try: body = client.V1DeleteOptions(propagation_policy='Background') api_response = self.batchApi.delete_namespaced_job(name, namespace, body=body) hiss.echo('Delete job %s on namespace %s success' % (name, namespace)) return util.Result(success=True, data=api_response) except ApiException as e: err_msg = "Exception when calling BatchV1Api->delete_namespaced_job: %s\n" % e return util.Result(success=hiss.hiss(err_msg), msg=err_msg)
def create_namespace(self, name): hiss.echo('Create Namespace %s' % name) ns = Namespace(name) # ns.create() if not ns.get(): hiss.sub_echo('Namespace %s does not exist. Creating...' % name) ns.create() else: hiss.sub_echo('Namespace %s already exists' % name)
def make_temp_folder(): hiss.echo('Create Folder temp') temp_path = get_temp_path() make_folder(temp_path) # Get current time current_time = split_timenow_utc() # Make folder temp if it not exists yaml_path = '%s/%s' % (temp_path, current_time[0]) make_folder(yaml_path)
def setup_admin(): domains = settings.ORDERER_DOMAINS.split(' ') if len(domains) == 0: domains = settings.PEER_DOMAINS.split(' ') # Create application artifact folder hiss.echo('Create application artifact folder') ## Find efs pod pods = settings.k8s.find_pod(namespace="default", keyword="test-efs") if not pods: return hiss.hiss('cannot find tiller pod') mkdir_cmd = ('mkdir -p ' + settings.EFS_ROOT + '/admin/artifacts;' 'mkdir -p ' + settings.EFS_ROOT + '/admin/crypto-path;' 'mkdir -p ' + settings.EFS_ROOT + '/admin/crypto-store;') ## Exec command exec_command = ['/bin/bash', '-c', '%s' % (mkdir_cmd)] result_get_folder = settings.k8s.exec_pod(podName=pods[0], namespace="default", command=exec_command) hiss.sub_echo(result_get_folder.data) # Create temp folder & namespace settings.k8s.prereqs(domains[0]) dict_env = { 'ORDERER_DOMAIN': domains[0], 'EFS_SERVER': settings.EFS_SERVER, 'EFS_PATH': settings.EFS_PATH, 'EFS_EXTEND': settings.EFS_EXTEND, 'PVS_PATH': settings.PVS_PATH } # Apply deployment k8s_template_file = '%s/admin-v1/admin-deployment.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=domains[0], k8s_template_file=k8s_template_file, dict_env=dict_env) # Apply service k8s_template_file = '%s/admin-v1/admin-service.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=domains[0], k8s_template_file=k8s_template_file, dict_env=dict_env)
def setup_peer(peer, index): # Get domain domain = util.get_domain(peer) # Create temp folder & namespace settings.k8s.prereqs(domain) if settings.K8S_TYPE == 'minikube': storage_class = 'standard' else: storage_class = 'gp2' dict_env = { 'PEER_ORG': peer, 'PEER_DOMAIN': domain, 'PEER_INDEX': index, 'FABRIC_TAG': settings.FABRIC_TAG, 'COUCHDB_TAG': settings.COUCHDB_TAG, 'EFS_SERVER': settings.EFS_SERVER, 'EFS_PATH': settings.EFS_PATH, 'EFS_EXTEND': settings.EFS_EXTEND, 'PVS_PATH': settings.PVS_PATH, 'STORAGE_CLASS': storage_class } peer_stateful = '%s/peer-sts/peer-stateful.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=domain, k8s_template_file=peer_stateful, dict_env=dict_env) peer_service = '%s/peer-sts/peer-service-stateful.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=domain, k8s_template_file=peer_service, dict_env=dict_env) num_peers = int(settings.NUM_PEERS) if (num_peers > 2): hiss.echo('NUM_PEER > 2: Does not support deployment nlb service') if (util.get_peer_external_domain(peer, index) != ''): peer_service_nlb = '%s/peer-sts/peer-service-nlb.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template( namespace=domain, k8s_template_file=peer_service_nlb, dict_env=dict_env)
def setup_grafana(): hiss.echo('Deploy grafana') namespace = get_namespace() # Create temp folder & namespace settings.k8s.prereqs(namespace) dict_env = { 'DOMAIN': namespace } # Deploy grafana sts grafana_template = '%s/grafana/grafana-stateful.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=namespace, k8s_template_file=grafana_template, dict_env=dict_env) # Deploy grafana svc grafana_sv_template = '%s/grafana/grafana-service-stateful.yaml' % util.get_k8s_template_path() settings.k8s.apply_yaml_from_template( namespace=namespace, k8s_template_file=grafana_sv_template, dict_env=dict_env)
def detect_deployed_efs_server(k8s_type): hiss.echo('Looking for deployed efs server...') efs_server = '' if k8s_type == 'eks': # Find efs-efs-provisioner pod pods = KubeHelper().find_pod(namespace="default", keyword="efs-efs-provisioner") if not pods: return efs_server efs_server_cmd = 'kubectl describe deployments -n default efs-efs-provisioner | grep Server' detected_efs_server = subprocess.check_output(efs_server_cmd, shell=True) if detected_efs_server: efs_server = detected_efs_server.decode().split(':')[1].strip() elif k8s_type == 'minikube': efs_server_cmd = 'kubectl get svc -n default | grep nfs-server | awk \'{print $3}\'' detected_efs_server = subprocess.check_output(efs_server_cmd, shell=True) if detected_efs_server: efs_server = detected_efs_server.decode().strip() return efs_server
def extract(force_all, force_config, force_script, force_template, force_other, dev_mode): # Extract config mamba_config = os.path.dirname(DEFAULT_CONFIG_PATH) if not os.path.isdir(mamba_config) or force_all or force_config: extract_cfg(mamba_config, dev_mode) # Extract scripts if not os.path.isdir(DEFAULT_SCRIPT_PATH) or force_all or force_script: hiss.echo('Extract scripts to default scripts path: %s ' % DEFAULT_SCRIPT_PATH) script_path = util.get_package_resource('', 'scripts') if os.path.isdir(DEFAULT_SCRIPT_PATH): shutil.rmtree(DEFAULT_SCRIPT_PATH) shutil.copytree(script_path, DEFAULT_SCRIPT_PATH) # Extract template if not os.path.isdir(DEFAULT_TEMPLATE_PATH) or force_all or force_template: hiss.echo('Extract template to default template path: %s ' % DEFAULT_TEMPLATE_PATH) template_path = util.get_package_resource('', 'template') if os.path.isdir(DEFAULT_TEMPLATE_PATH): shutil.rmtree(DEFAULT_TEMPLATE_PATH) shutil.copytree(template_path, DEFAULT_TEMPLATE_PATH) # Extract other if not os.path.isdir(DEFAULT_OTHER_PATH) or force_all or force_other: hiss.echo('Extract other to default other path: %s ' % DEFAULT_OTHER_PATH) other_path = util.get_package_resource('k8s', '') if os.path.isdir(DEFAULT_OTHER_PATH): shutil.rmtree(DEFAULT_OTHER_PATH) shutil.copytree(other_path, DEFAULT_OTHER_PATH)
def extract_cfg(mamba_config, dev_mode): hiss.echo('Extract config to default config path: %s ' % DEFAULT_CONFIG_PATH) dotenv_path = get_template_env() if not os.path.isdir(mamba_config): os.makedirs(mamba_config) shutil.copy(dotenv_path, DEFAULT_CONFIG_PATH) load_dotenv(DEFAULT_CONFIG_PATH) default_cluster_name = os.getenv('EKS_CLUSTER_NAME') default_k8s_type = os.getenv('K8S_TYPE') deployment_mode = os.getenv('DEPLOYMENT_ENV') # Input cluster_name = input( f'Cluster name ({default_cluster_name}): ') or default_cluster_name k8s_type = input( f'Kubenetes type - support eks or minikube ({default_k8s_type}): ' ) or default_k8s_type if dev_mode: deployment_mode = 'develop' # Detect current environment setting # EFS_SERVER efs_server = detect_deployed_efs_server(k8s_type) # EFS_SERVER_ID efs_server_id = efs_server.split('.')[0] print('efs_server: ', efs_server) # EFS_PATH efs_path = detect_deployed_efs_path() # EFS_POD efs_pod = detect_deployed_efs_pod() if not efs_pod and k8s_type == 'eks': retry = 3 while retry > 0: efs_server = input('EFS server (*):') if efs_server: break retry -= 1 if not efs_server: hiss.hiss('Must specify EFS_SERVER!') exit() with open(DEFAULT_CONFIG_PATH, "r") as sources: lines = sources.readlines() with open(DEFAULT_CONFIG_PATH, "w") as sources: for line in lines: newline = re.sub(r'EKS_CLUSTER_NAME=.*', f'EKS_CLUSTER_NAME=\"{cluster_name}\"', line) newline = re.sub(r'K8S_TYPE=.*', f'K8S_TYPE=\"{k8s_type}\"', newline) if efs_server: newline = re.sub(r'EFS_SERVER=.*', f'EFS_SERVER=\"{efs_server}\"', newline) if efs_server_id: newline = re.sub(r'EFS_SERVER_ID=.*', f'EFS_SERVER_ID=\"{efs_server_id}\"', newline) if efs_path: newline = re.sub(r'EFS_PATH=.*', f'EFS_PATH=\"efs-{efs_path}\"', newline) if efs_pod: newline = re.sub(r'EFS_POD=.*', f'EFS_POD=\"{efs_pod}\"', newline) newline = re.sub(r'DEPLOYMENT_ENV=.*', f'DEPLOYMENT_ENV=\"{deployment_mode}\"', newline) sources.write(newline) hiss.rattle("See more config in %s" % DEFAULT_CONFIG_PATH)
def setup_admin(org): # Get domain domain = util.get_domain(org) # Get orderer information orderer_names = settings.ORDERER_ORGS.split(' ') orderer_domains = settings.ORDERER_DOMAINS.split(' ') if orderer_names == '' and settings.REMOTE_ORDERER_NAME != '': orderer_names = settings.REMOTE_ORDERER_NAME.split(' ') orderer_domains = settings.REMOTE_ORDERER_DOMAIN.split(' ') # Build endorsement config peer_orgs = '%s %s' % (settings.PEER_ORGS, settings.ENDORSEMENT_ORG_NAME) peer_domains = '%s %s' % (settings.PEER_DOMAINS, settings.ENDORSEMENT_ORG_DOMAIN) print(peer_orgs) # Create application artifact folder hiss.echo('Create wallet folder') ## Find efs pod pods = settings.k8s.find_pod(namespace="default", keyword="test-efs") if not pods: return hiss.hiss('cannot find tiller pod') mkdir_cmd = ('mkdir -p ' + settings.EFS_ROOT + '/admin-v2/wallet;') # Exec command exec_command = ['/bin/bash', '-c', '%s' % (mkdir_cmd)] result_get_folder = settings.k8s.exec_pod(podName=pods[0], namespace="default", command=exec_command) hiss.sub_echo(result_get_folder.data) # Create temp folder & namespace settings.k8s.prereqs(domain) dict_env = { 'ORG_NAME': org, 'ORG_DOMAIN': domain, 'PEER_NAMES': peer_orgs, 'PEER_DOMAINS': peer_domains, 'ORDERER_DOMAIN': orderer_domains[0], 'ORGDERER_NAME': orderer_names[0], 'EFS_SERVER': settings.EFS_SERVER, 'EFS_PATH': settings.EFS_PATH, 'EFS_EXTEND': settings.EFS_EXTEND, 'PVS_PATH': settings.PVS_PATH } # Apply deployment k8s_template_file = '%s/admin/admin-deployment.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=domain, k8s_template_file=k8s_template_file, dict_env=dict_env) # Apply service k8s_template_file = '%s/admin/admin-service.yaml' % util.get_k8s_template_path( ) settings.k8s.apply_yaml_from_template(namespace=domain, k8s_template_file=k8s_template_file, dict_env=dict_env)