def apply_yaml_from_template(self, namespace, k8s_template_file, dict_env):
yaml_path, _ = util.load_yaml_config_template(k8s_template_file, dict_env)
hiss.sub_echo('Create %s successfully' % yaml_path)
# Execute yaml
hiss.echo('Apply yaml file')
stream = open(yaml_path, 'r')
docs = yaml.safe_load_all(stream)
success = True
for doc in docs:
try:
if doc['kind'] == 'Service':
self.coreApi.create_namespaced_service(namespace, body=doc)
continue
except ApiException as e:
print("Service already deployed!")
continue
try:
if doc['kind'] == 'StatefulSet':
self.appsApi.create_namespaced_stateful_set(
namespace, body=doc)
self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace)
if doc['kind'] == 'Deployment':
self.appsApi.create_namespaced_deployment(
namespace, body=doc)
self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace)
if doc['kind'] == 'Job':
self.batchApi.create_namespaced_job(namespace, body=doc)
self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace, check_job_success=True)
except ApiException as e:
print("Exception when apply yaml: %s\n" % e)
success = False
# self.check_pod_status_by_keyword(keyword=doc['metadata']['name'], namespace=namespace)
return success
def setup_explorer():
hiss.echo('Generate explorer config')
config = generate_explorer_config()
create_explorer_config_in_efs(json.dumps(config))
hiss.echo('Deploy explorer')
# Get domain
explorer_namespace = get_namespace()
# Create temp folder & namespace
settings.k8s.prereqs(explorer_namespace)
dict_env = {
'DOMAIN': explorer_namespace,
'DATABASE_PASSWORD': '******',
'EFS_SERVER': settings.EFS_SERVER,
'EFS_PATH': settings.EFS_PATH,
'EFS_EXTEND': settings.EFS_EXTEND
}
# Deploy explorer db sts
explorer_template = '%s/explorer/explorer-deployment.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=explorer_namespace, k8s_template_file=explorer_template, dict_env=dict_env)
# Deploy explorer db svc
explorer_svc_template = '%s/explorer/explorer-service.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=explorer_namespace, k8s_template_file=explorer_svc_template, dict_env=dict_env)
def mamba(config, set_default):
# Setup all shared global utilities in settings module
settings.init(config, set_default)
if mamba.invoke_without_command:
hiss.rattle('Initialize mamba')
hiss.echo('Successfully with \'%s\' on the %s!' %
(config, settings.EKS_CLUSTER_NAME))
def load_yaml_config_template(yaml_template_path, dict_env):
new_yaml_file_name = yaml_template_path.split('/').pop()
# Load template file
with open(yaml_template_path, 'r') as sources:
lines = sources.readlines()
out_data = []
# Replace variable
for line in lines:
out_line = line
for key, value in dict_env.items():
out_line = re.sub(r'{{%s}}' % key, value, out_line)
out_data.append(out_line)
# Get time now
current_time = split_timenow_utc()
# Create yaml_path
hiss.echo('Create yaml file')
yaml_path = '%s/%s/%s_%s' % (get_temp_path(), current_time[0],
current_time[1], new_yaml_file_name)
# Write yaml -> yaml_path
with open(yaml_path, "w") as sources:
for line in out_data:
sources.write(line)
# Read new yaml
with open(yaml_path, 'r') as ymlfile:
config = yaml.load_all(ymlfile)
return yaml_path, config
def setup_prometheus():
hiss.echo('Generate prometheus config')
config_path = generate_prom_config()
if not cp_config_efs(config_path):
return util.Result(success=False,
msg='Cannot copy prom config to efs pod')
hiss.echo('Deploy prometheus')
# Get domain
prom_namespace = get_namespace()
# Create temp folder & namespace
settings.k8s.prereqs(prom_namespace)
dict_env = {
'DOMAIN': prom_namespace,
'EFS_SERVER': settings.EFS_SERVER,
'EFS_PATH': settings.EFS_PATH,
'EFS_EXTEND': settings.EFS_EXTEND,
'PVS_PATH': settings.PVS_PATH
}
# Deploy prometheus sts
prom_template = '%s/prometheus/prometheus-stateful.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=prom_namespace,
k8s_template_file=prom_template,
dict_env=dict_env)
prom_svc_template = '%s/prometheus/prometheus-service-stateful.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=prom_namespace,
k8s_template_file=prom_svc_template,
dict_env=dict_env)
def delete_stateful(self, name, namespace, delete_pvc=False):
action = 'Delete'
if delete_pvc == True:
action = 'Terminate'
read_result = self.read_stateful_set(name, namespace)
if read_result.success == False:
return read_result
volume_claim_templates = read_result.data.spec.volume_claim_templates
if volume_claim_templates != None:
for pvt in volume_claim_templates:
pvt_name = pvt.metadata.name
list_pvc = self.find_pvc(namespace, keyword=pvt_name)
for pvc in list_pvc:
hiss.echo('Delete pvc %s ' % pvc)
self.delete_persistent_volume_claim(
name=pvc, namespace=namespace)
try:
body = client.V1DeleteOptions(propagation_policy='Background')
api_response = self.appsApi.delete_namespaced_stateful_set(
name, namespace, body=body)
self.check_pod_status_by_keyword(keyword=name,
namespace=namespace,
is_delete=True)
hiss.echo('%s stateful set %s on namespace %s success' %
(action, name, namespace))
return util.Result(success=True, data=api_response)
except ApiException as e:
err_msg = "Exception when calling AppsV1Api->delete_namespaced_stateful_set: %s\n" % e
return util.Result(success=hiss.hiss(err_msg), msg=err_msg)
def setup_peer(peer, index):
# Get domain
domain = util.get_domain(peer)
# Create temp folder & namespace
settings.k8s.prereqs(domain)
dict_env = {
'PEER_ORG': peer,
'PEER_DOMAIN': domain,
'PEER_INDEX': index,
'EFS_SERVER': settings.EFS_SERVER,
'EFS_PATH': settings.EFS_PATH,
'EFS_EXTEND': settings.EFS_EXTEND
}
peer_stateful = '%s/peer-sts/peer-stateful.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=domain, k8s_template_file=peer_stateful, dict_env=dict_env)
peer_service = '%s/peer-sts/peer-service-stateful.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=domain, k8s_template_file=peer_service, dict_env=dict_env)
num_peers = int(settings.NUM_PEERS)
if (num_peers > 2):
hiss.echo('NUM_PEER > 2: Does not support deployment nlb service')
if (util.get_peer_external_domain(peer, index) != ''):
peer_service_nlb = '%s/peer-sts/peer-service-nlb.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=domain, k8s_template_file=peer_service_nlb, dict_env=dict_env)
def detect_deployed_efs_pod():
hiss.echo('Looking for deployed efs pod...')
efs_pod = ''
efs_pod_cmd = 'kubectl get pod -n default | grep test-efs | awk \'{print $1}\''
detected_efs_pod = subprocess.check_output(efs_pod_cmd, shell=True)
if detected_efs_pod:
efs_pod = detected_efs_pod.decode().strip()
return efs_pod
def detect_deployed_efs_path():
hiss.echo('Looking for deployed efs path...')
efs_path = ''
efs_path_cmd = 'kubectl get pvc | grep efs | awk \'{print $3}\''
detected_efs_path = subprocess.check_output(efs_path_cmd, shell=True)
if detected_efs_path:
efs_path = detected_efs_path.decode().strip()
return efs_path
def delete_job(self, name, namespace):
try:
body = client.V1DeleteOptions(propagation_policy='Background')
api_response = self.batchApi.delete_namespaced_job(name, namespace, body=body)
hiss.echo('Delete job %s on namespace %s success' % (name, namespace))
return util.Result(success=True, data=api_response)
except ApiException as e:
err_msg = "Exception when calling BatchV1Api->delete_namespaced_job: %s\n" % e
return util.Result(success=hiss.hiss(err_msg), msg=err_msg)
def create_namespace(self, name):
hiss.echo('Create Namespace %s' % name)
ns = Namespace(name)
# ns.create()
if not ns.get():
hiss.sub_echo('Namespace %s does not exist. Creating...' % name)
ns.create()
else:
hiss.sub_echo('Namespace %s already exists' % name)
def make_temp_folder():
hiss.echo('Create Folder temp')
temp_path = get_temp_path()
make_folder(temp_path)
# Get current time
current_time = split_timenow_utc()
# Make folder temp if it not exists
yaml_path = '%s/%s' % (temp_path, current_time[0])
make_folder(yaml_path)
def setup_admin():
domains = settings.ORDERER_DOMAINS.split(' ')
if len(domains) == 0:
domains = settings.PEER_DOMAINS.split(' ')
# Create application artifact folder
hiss.echo('Create application artifact folder')
## Find efs pod
pods = settings.k8s.find_pod(namespace="default", keyword="test-efs")
if not pods:
return hiss.hiss('cannot find tiller pod')
mkdir_cmd = ('mkdir -p ' + settings.EFS_ROOT + '/admin/artifacts;'
'mkdir -p ' + settings.EFS_ROOT + '/admin/crypto-path;'
'mkdir -p ' + settings.EFS_ROOT + '/admin/crypto-store;')
## Exec command
exec_command = ['/bin/bash', '-c', '%s' % (mkdir_cmd)]
result_get_folder = settings.k8s.exec_pod(podName=pods[0],
namespace="default",
command=exec_command)
hiss.sub_echo(result_get_folder.data)
# Create temp folder & namespace
settings.k8s.prereqs(domains[0])
dict_env = {
'ORDERER_DOMAIN': domains[0],
'EFS_SERVER': settings.EFS_SERVER,
'EFS_PATH': settings.EFS_PATH,
'EFS_EXTEND': settings.EFS_EXTEND,
'PVS_PATH': settings.PVS_PATH
}
# Apply deployment
k8s_template_file = '%s/admin-v1/admin-deployment.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=domains[0],
k8s_template_file=k8s_template_file,
dict_env=dict_env)
# Apply service
k8s_template_file = '%s/admin-v1/admin-service.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=domains[0],
k8s_template_file=k8s_template_file,
dict_env=dict_env)
def setup_peer(peer, index):
# Get domain
domain = util.get_domain(peer)
# Create temp folder & namespace
settings.k8s.prereqs(domain)
if settings.K8S_TYPE == 'minikube':
storage_class = 'standard'
else:
storage_class = 'gp2'
dict_env = {
'PEER_ORG': peer,
'PEER_DOMAIN': domain,
'PEER_INDEX': index,
'FABRIC_TAG': settings.FABRIC_TAG,
'COUCHDB_TAG': settings.COUCHDB_TAG,
'EFS_SERVER': settings.EFS_SERVER,
'EFS_PATH': settings.EFS_PATH,
'EFS_EXTEND': settings.EFS_EXTEND,
'PVS_PATH': settings.PVS_PATH,
'STORAGE_CLASS': storage_class
}
peer_stateful = '%s/peer-sts/peer-stateful.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=domain,
k8s_template_file=peer_stateful,
dict_env=dict_env)
peer_service = '%s/peer-sts/peer-service-stateful.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=domain,
k8s_template_file=peer_service,
dict_env=dict_env)
num_peers = int(settings.NUM_PEERS)
if (num_peers > 2):
hiss.echo('NUM_PEER > 2: Does not support deployment nlb service')
if (util.get_peer_external_domain(peer, index) != ''):
peer_service_nlb = '%s/peer-sts/peer-service-nlb.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(
namespace=domain,
k8s_template_file=peer_service_nlb,
dict_env=dict_env)
def setup_grafana():
hiss.echo('Deploy grafana')
namespace = get_namespace()
# Create temp folder & namespace
settings.k8s.prereqs(namespace)
dict_env = {
'DOMAIN': namespace
}
# Deploy grafana sts
grafana_template = '%s/grafana/grafana-stateful.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=namespace, k8s_template_file=grafana_template, dict_env=dict_env)
# Deploy grafana svc
grafana_sv_template = '%s/grafana/grafana-service-stateful.yaml' % util.get_k8s_template_path()
settings.k8s.apply_yaml_from_template(
namespace=namespace, k8s_template_file=grafana_sv_template, dict_env=dict_env)
def detect_deployed_efs_server(k8s_type):
hiss.echo('Looking for deployed efs server...')
efs_server = ''
if k8s_type == 'eks':
# Find efs-efs-provisioner pod
pods = KubeHelper().find_pod(namespace="default",
keyword="efs-efs-provisioner")
if not pods:
return efs_server
efs_server_cmd = 'kubectl describe deployments -n default efs-efs-provisioner | grep Server'
detected_efs_server = subprocess.check_output(efs_server_cmd,
shell=True)
if detected_efs_server:
efs_server = detected_efs_server.decode().split(':')[1].strip()
elif k8s_type == 'minikube':
efs_server_cmd = 'kubectl get svc -n default | grep nfs-server | awk \'{print $3}\''
detected_efs_server = subprocess.check_output(efs_server_cmd,
shell=True)
if detected_efs_server:
efs_server = detected_efs_server.decode().strip()
return efs_server
def extract(force_all, force_config, force_script, force_template, force_other,
dev_mode):
# Extract config
mamba_config = os.path.dirname(DEFAULT_CONFIG_PATH)
if not os.path.isdir(mamba_config) or force_all or force_config:
extract_cfg(mamba_config, dev_mode)
# Extract scripts
if not os.path.isdir(DEFAULT_SCRIPT_PATH) or force_all or force_script:
hiss.echo('Extract scripts to default scripts path: %s ' %
DEFAULT_SCRIPT_PATH)
script_path = util.get_package_resource('', 'scripts')
if os.path.isdir(DEFAULT_SCRIPT_PATH):
shutil.rmtree(DEFAULT_SCRIPT_PATH)
shutil.copytree(script_path, DEFAULT_SCRIPT_PATH)
# Extract template
if not os.path.isdir(DEFAULT_TEMPLATE_PATH) or force_all or force_template:
hiss.echo('Extract template to default template path: %s ' %
DEFAULT_TEMPLATE_PATH)
template_path = util.get_package_resource('', 'template')
if os.path.isdir(DEFAULT_TEMPLATE_PATH):
shutil.rmtree(DEFAULT_TEMPLATE_PATH)
shutil.copytree(template_path, DEFAULT_TEMPLATE_PATH)
# Extract other
if not os.path.isdir(DEFAULT_OTHER_PATH) or force_all or force_other:
hiss.echo('Extract other to default other path: %s ' %
DEFAULT_OTHER_PATH)
other_path = util.get_package_resource('k8s', '')
if os.path.isdir(DEFAULT_OTHER_PATH):
shutil.rmtree(DEFAULT_OTHER_PATH)
shutil.copytree(other_path, DEFAULT_OTHER_PATH)
def extract_cfg(mamba_config, dev_mode):
hiss.echo('Extract config to default config path: %s ' %
DEFAULT_CONFIG_PATH)
dotenv_path = get_template_env()
if not os.path.isdir(mamba_config):
os.makedirs(mamba_config)
shutil.copy(dotenv_path, DEFAULT_CONFIG_PATH)
load_dotenv(DEFAULT_CONFIG_PATH)
default_cluster_name = os.getenv('EKS_CLUSTER_NAME')
default_k8s_type = os.getenv('K8S_TYPE')
deployment_mode = os.getenv('DEPLOYMENT_ENV')
# Input
cluster_name = input(
f'Cluster name ({default_cluster_name}): ') or default_cluster_name
k8s_type = input(
f'Kubenetes type - support eks or minikube ({default_k8s_type}): '
) or default_k8s_type
if dev_mode:
deployment_mode = 'develop'
# Detect current environment setting
# EFS_SERVER
efs_server = detect_deployed_efs_server(k8s_type)
# EFS_SERVER_ID
efs_server_id = efs_server.split('.')[0]
print('efs_server: ', efs_server)
# EFS_PATH
efs_path = detect_deployed_efs_path()
# EFS_POD
efs_pod = detect_deployed_efs_pod()
if not efs_pod and k8s_type == 'eks':
retry = 3
while retry > 0:
efs_server = input('EFS server (*):')
if efs_server:
break
retry -= 1
if not efs_server:
hiss.hiss('Must specify EFS_SERVER!')
exit()
with open(DEFAULT_CONFIG_PATH, "r") as sources:
lines = sources.readlines()
with open(DEFAULT_CONFIG_PATH, "w") as sources:
for line in lines:
newline = re.sub(r'EKS_CLUSTER_NAME=.*',
f'EKS_CLUSTER_NAME=\"{cluster_name}\"', line)
newline = re.sub(r'K8S_TYPE=.*', f'K8S_TYPE=\"{k8s_type}\"',
newline)
if efs_server:
newline = re.sub(r'EFS_SERVER=.*',
f'EFS_SERVER=\"{efs_server}\"', newline)
if efs_server_id:
newline = re.sub(r'EFS_SERVER_ID=.*',
f'EFS_SERVER_ID=\"{efs_server_id}\"', newline)
if efs_path:
newline = re.sub(r'EFS_PATH=.*',
f'EFS_PATH=\"efs-{efs_path}\"', newline)
if efs_pod:
newline = re.sub(r'EFS_POD=.*', f'EFS_POD=\"{efs_pod}\"',
newline)
newline = re.sub(r'DEPLOYMENT_ENV=.*',
f'DEPLOYMENT_ENV=\"{deployment_mode}\"', newline)
sources.write(newline)
hiss.rattle("See more config in %s" % DEFAULT_CONFIG_PATH)
def setup_admin(org):
# Get domain
domain = util.get_domain(org)
# Get orderer information
orderer_names = settings.ORDERER_ORGS.split(' ')
orderer_domains = settings.ORDERER_DOMAINS.split(' ')
if orderer_names == '' and settings.REMOTE_ORDERER_NAME != '':
orderer_names = settings.REMOTE_ORDERER_NAME.split(' ')
orderer_domains = settings.REMOTE_ORDERER_DOMAIN.split(' ')
# Build endorsement config
peer_orgs = '%s %s' % (settings.PEER_ORGS, settings.ENDORSEMENT_ORG_NAME)
peer_domains = '%s %s' % (settings.PEER_DOMAINS,
settings.ENDORSEMENT_ORG_DOMAIN)
print(peer_orgs)
# Create application artifact folder
hiss.echo('Create wallet folder')
## Find efs pod
pods = settings.k8s.find_pod(namespace="default", keyword="test-efs")
if not pods:
return hiss.hiss('cannot find tiller pod')
mkdir_cmd = ('mkdir -p ' + settings.EFS_ROOT + '/admin-v2/wallet;')
# Exec command
exec_command = ['/bin/bash', '-c', '%s' % (mkdir_cmd)]
result_get_folder = settings.k8s.exec_pod(podName=pods[0],
namespace="default",
command=exec_command)
hiss.sub_echo(result_get_folder.data)
# Create temp folder & namespace
settings.k8s.prereqs(domain)
dict_env = {
'ORG_NAME': org,
'ORG_DOMAIN': domain,
'PEER_NAMES': peer_orgs,
'PEER_DOMAINS': peer_domains,
'ORDERER_DOMAIN': orderer_domains[0],
'ORGDERER_NAME': orderer_names[0],
'EFS_SERVER': settings.EFS_SERVER,
'EFS_PATH': settings.EFS_PATH,
'EFS_EXTEND': settings.EFS_EXTEND,
'PVS_PATH': settings.PVS_PATH
}
# Apply deployment
k8s_template_file = '%s/admin/admin-deployment.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=domain,
k8s_template_file=k8s_template_file,
dict_env=dict_env)
# Apply service
k8s_template_file = '%s/admin/admin-service.yaml' % util.get_k8s_template_path(
)
settings.k8s.apply_yaml_from_template(namespace=domain,
k8s_template_file=k8s_template_file,
dict_env=dict_env)
