def test_integration_dev(self): # Get options opts = load_config(self.CONFIG) # TODO: There should be a more elegant way of obtaining all the releases releases = ( [key for key in opts["cas"].keys()] + [key + "-pg" for key in opts["cas"].keys()] + list(opts["msps"]["AlphaMSP"]["orderers"]["nodes"].keys()) + [("cdb-" + key) for key in opts["msps"]["BetaMSP"]["peers"]["nodes"].keys()] + [key for key in opts["msps"]["BetaMSP"]["peers"]["nodes"].keys()]) # Run Fabric script check_cluster( self.CONTEXT ) # Dangerous operation, recheck we have not shifted context runner_fabric(opts) # Delete all deployments from Helm check_cluster( self.CONTEXT ) # Dangerous operation, recheck we have not shifted context execute(f"helm delete --purge {' '.join(releases)}") # Delete the namespaces check_cluster( self.CONTEXT ) # Dangerous operation, recheck we have not shifted context execute("kubectl delete ns alpha beta")
def test_execute(self, mock_log, mock_check_output): execute("ls") mock_log.info.assert_called_once() mock_log.info.assert_called_with("ls") mock_log.debug.assert_called_once() mock_check_output.assert_called_once() mock_check_output.assert_called_with("ls", shell=True, stderr=-2)
def genesis_block(opts, verbose=False): """Create and save Genesis Block to K8S. Args: opts (dict): Nephos options dict. verbose (bool) Verbosity. False by default. """ ord_namespace = get_namespace(opts, opts["orderers"]["msp"]) # Change to blockchain materials directory chdir(opts["core"]["dir_config"]) # Create the genesis block genesis_key = "genesis.block" genesis_file = join(opts["core"]["dir_crypto"], genesis_key) if not exists(genesis_file): # Genesis block creation and storage execute( "configtxgen -profile OrdererGenesis -outputBlock {genesis_file}". format(genesis_file=genesis_file), verbose=verbose, ) else: print("{} already exists".format(genesis_file)) # Create the genesis block secret secret_from_file( secret=opts["orderers"]["secret_genesis"], namespace=ord_namespace, key=genesis_key, filename=genesis_file, verbose=verbose, ) # Return to original directory chdir(PWD)
def helm_upgrade(repo, app, release, namespace, config_yaml=None, env_vars=None, preserve=None, verbose=False, pod_num=1): ls_res = execute('helm status {release}'.format(release=release)) # Get Helm Env-Vars env_vars_string = helm_env_vars(namespace, env_vars, preserve, verbose=verbose) if ls_res: command = 'helm upgrade {name} {repo}/{app}'.format(app=app, name=release, repo=repo) if config_yaml: command += ' -f {}'.format(config_yaml) command += env_vars_string # Execute execute(command, verbose=verbose) else: raise Exception('Cannot update a Helm release that is not running') helm_check(app, release, namespace, pod_num)
def genesis_block(opts): """Create and save Genesis Block to K8S. Args: opts (dict): Nephos options dict. """ ord_namespace = get_namespace(opts, opts["orderers"]["msp"]) # Change to blockchain materials directory chdir(opts["core"]["dir_config"]) # Create the genesis block genesis_key = "genesis.block" genesis_file = join(opts["core"]["dir_crypto"], genesis_key) if not exists(genesis_file): # Genesis block creation and storage execute( f"configtxgen -profile OrdererGenesis -outputBlock {genesis_file}", ) else: logging.info(f"{genesis_file} already exists") # Create the genesis block secret secret_from_file( secret=opts["orderers"]["secret_genesis"], namespace=ord_namespace, key=genesis_key, filename=genesis_file, ) # Return to original directory chdir(PWD)
def channel_tx(opts): """Create and save Channel Transaction to K8S. Args: opts (dict): Nephos options dict. """ peer_namespace = get_namespace(opts, opts["peers"]["msp"]) # Change to blockchain materials directory chdir(opts["core"]["dir_config"]) # Create Channel Tx channel_key = f"{opts['peers']['channel_name']}.tx" channel_file = join(opts["core"]["dir_crypto"], channel_key) if not exists(channel_file): # Channel transaction creation and storage execute( f"configtxgen -profile {opts['peers']['channel_profile']} -channelID {opts['peers']['channel_name']} -outputCreateChannelTx {channel_file}", ) else: logging.info(f"{channel_file} already exists") # Create the channel transaction secret secret_from_file( secret=opts["peers"]["secret_channel"], namespace=peer_namespace, key=channel_key, filename=channel_file, ) # Return to original directory chdir(PWD)
def channel_tx(opts, verbose=False): peer_namespace = get_namespace(opts, opts['peers']['msp']) # Change to blockchain materials directory chdir(opts['core']['dir_config']) # Create Channel Tx channel_file = '{channel}.tx'.format(channel=opts['peers']['channel_name']) if not path.exists(channel_file): # Channel transaction creation and storage execute( 'configtxgen -profile {channel_profile} -channelID {channel} -outputCreateChannelTx {channel_file}' .format(channel_profile=opts['peers']['channel_profile'], channel=opts['peers']['channel_name'], channel_file=channel_file), verbose=verbose) else: print('{channel}.tx already exists'.format( channel=opts['peers']['channel_name'])) # Create the channel transaction secret secret_from_file(secret=opts['peers']['secret_channel'], namespace=peer_namespace, key=channel_file, filename=channel_file, verbose=verbose) # Return to original directory chdir(PWD)
def helm_check(app, name, namespace, pod_num=None): print(t.yellow('Ensuring that all pods are running ')) running = False first_pass = True while not running: # TODO: Best to generate a function that checks app state states_list = execute( 'kubectl get pods -n {ns} -l "app={app},release={name}" -o jsonpath="{{.items[*].status.phase}}"' .format(app=app, name=name, ns=namespace), show_command=first_pass).split() # Let us also check the number of pods we have pod_list = execute( 'kubectl get pods -n {ns} -l "app={app},release={name}" -o jsonpath="{{.items[*].metadata.name}}"' .format(app=app, name=name, ns=namespace), show_command=first_pass).split() first_pass = False # We keep checking the state of the pods until they are running states = set(states_list) if len(states) == 1 and 'Running' in states and ( pod_num is None or len(pod_list) == pod_num): print(t.green('All pods in {} are running'.format(name))) running = True else: print(t.red('.'), end='', flush=True) sleep(15)
def channel_tx(opts, verbose=False): """Create and save Channel Transaction to K8S. Args: opts (dict): Nephos options dict. verbose (bool) Verbosity. False by default. """ peer_namespace = get_namespace(opts, opts["peers"]["msp"]) # Change to blockchain materials directory chdir(opts["core"]["dir_config"]) # Create Channel Tx channel_key = "{channel}.tx".format(channel=opts["peers"]["channel_name"]) channel_file = join(opts["core"]["dir_crypto"], channel_key) if not exists(channel_file): # Channel transaction creation and storage execute( "configtxgen -profile {channel_profile} -channelID {channel} -outputCreateChannelTx {channel_file}".format( channel_profile=opts["peers"]["channel_profile"], channel=opts["peers"]["channel_name"], channel_file=channel_file, ), verbose=verbose, ) else: print("{} already exists".format(channel_file)) # Create the channel transaction secret secret_from_file( secret=opts["peers"]["secret_channel"], namespace=peer_namespace, key=channel_key, filename=channel_file, verbose=verbose, ) # Return to original directory chdir(PWD)
def test_execute_verbose(self, mock_print, mock_check_output): # Add some side effects mock_check_output.side_effect = ["output".encode("ascii")] execute("ls", verbose=True) # First check_output mock_check_output.assert_called_once() mock_check_output.assert_called_with("ls", shell=True, stderr=-2) # Then print mock_print.assert_has_calls([call("ls"), call("output")])
def test_execute_verbose(self, mock_print, mock_check_output): # Add some side effects mock_check_output.side_effect = ['output'.encode('ascii')] execute('ls', verbose=True) # First check_output mock_check_output.assert_called_once() mock_check_output.assert_called_with('ls', shell=True, stderr=-2) # Then print mock_print.assert_has_calls([call('ls'), call('output')])
def helm_init(): """Initialise Helm on cluster, using RBAC.""" res, _ = execute("helm list") if res is not None: print(t.green("Helm is already installed!")) else: execute("kubectl create -f {}/../extras/helm-rbac.yaml".format( CURRENT_DIR)) execute("helm init --service-account tiller") # Fix issue with automountServiceToken res, _ = execute( "kubectl -n kube-system get deployment tiller-deploy " + '-o jsonpath="{.spec.template.spec.automountServiceAccountToken}"') if res == "false": execute( "kubectl -n kube-system patch deployment tiller-deploy " + '-p \'{"spec": {"template": {"spec": {"automountServiceAccountToken": true}}}}\'' ) # We keep checking the state of helm until everything is running running = False while not running: res, _ = execute("helm list") if res is not None: running = True else: print(t.red("."), end="", flush=True) sleep(15)
def helm_init(): res = execute('helm list') if res is not None: print(t.green('Helm is already installed!')) else: execute('kubectl create -f {}/../extras/helm-rbac.yaml'.format( CURRENT_DIR)) execute('helm init --service-account tiller') # Fix issue with automountServiceToken res = execute( 'kubectl -n kube-system get deployment tiller-deploy ' + '-o jsonpath="{.spec.template.spec.automountServiceAccountToken}"') if res == 'false': execute( "kubectl -n kube-system patch deployment tiller-deploy " + "-p '{\"spec\": {\"template\": {\"spec\": {\"automountServiceAccountToken\": true}}}}'" ) # We keep checking the state of helm until everything is running running = False while not running: res = execute('helm list') if res is not None: running = True else: print(t.red('.'), end='', flush=True) sleep(15)
def test_execute_error_quiet(self, mock_print, mock_check_output): # Add some side effects mock_check_output.side_effect = CalledProcessError(cmd='lst', returncode=127, output='/bin/sh: lst: command not found'.encode('ascii')) execute('lst', show_command=False, show_errors=False) # First check_output mock_check_output.assert_called_once() mock_check_output.assert_called_with('lst', shell=True, stderr=-2) # Then print mock_print.assert_not_called()
def test_execute_error(self, mock_print, mock_check_output): # Add some side effects mock_check_output.side_effect = CalledProcessError(cmd='lst', returncode=127, output='/bin/sh: lst: command not found'.encode('ascii')) execute('lst') # First check_output mock_check_output.assert_called_once() mock_check_output.assert_called_with('lst', shell=True, stderr=-2) # Then print mock_print.assert_has_calls([call('lst'), call('Command failed with CalledProcessError:'), call('/bin/sh: lst: command not found')])
def create_admin(opts, msp_name, verbose=False): """Create an admin identity. Args: opts (dict): Nephos options dict. msp_name (str): Name of Membership Service Provider. verbose (bool) Verbosity. False by default. """ dir_config = opts["core"]["dir_config"] dir_crypto = opts["core"]["dir_crypto"] msp_values = opts["msps"][msp_name] ca_values = opts["cas"][msp_values["ca"]] # TODO: Refactor this into its own function ca_name = msp_values["ca"] ca_namespace = get_namespace(opts, ca=ca_name) # Get CA ingress ingress_urls = ingress_read( ca_name + "-hlf-ca", namespace=ca_namespace, verbose=verbose ) ca_ingress = ingress_urls[0] # Register the Organisation with the CAs register_id( ca_namespace, msp_values["ca"], msp_values["org_admin"], msp_values["org_adminpw"], admin=True, verbose=verbose, ) # TODO: Can we reuse the Enroll function above? # If our keystore does not exist or is empty, we need to enroll the identity... keystore = join(dir_crypto, msp_name, "keystore") if not isdir(keystore) or not listdir(keystore): execute( ( "FABRIC_CA_CLIENT_HOME={dir} fabric-ca-client enroll " + "-u https://{id}:{pw}@{ingress} -M {msp_dir} --tls.certfiles {ca_server_tls}" ).format( dir=dir_config, id=msp_values["org_admin"], pw=msp_values["org_adminpw"], ingress=ca_ingress, msp_dir=join(dir_crypto, msp_name), ca_server_tls=abspath(ca_values["tls_cert"]), ), verbose=verbose, )
def pod_check(namespace, identifier, sleep_interval=10, pod_num=None): """Check if a set of pods exist and are functional. Args: namespace (str): Namespace where Helm deployment is located. identifier (str): Name of pod, or a label descriptor. sleep_interval (int): Number of seconds to sleep between attempts. pod_num (int): Number of pods expected to exist in the release. None by default. """ logging.info("Ensuring that all pods are running ") running = False while not running: states, _ = execute( f'kubectl get pods -n {namespace} {identifier} -o jsonpath="{{.items[*].status.phase}}"' ) states_list = states.split() # Let us also check the number of pods we have # We keep checking the state of the pods until they are running states = set(states_list) if (len(states) == 1 and "Running" in states and (pod_num is None or len(states_list) == pod_num)): logging.info(TERM.green("All pods are running")) running = True else: print(TERM.red("."), end="", flush=True) sleep(sleep_interval)
def test_execute_error(self, mock_print, mock_check_output): # Add some side effects mock_check_output.side_effect = CalledProcessError( cmd="lst", returncode=127, output="/bin/sh: lst: command not found".encode("ascii"), ) execute("lst") # First check_output mock_check_output.assert_called_once() mock_check_output.assert_called_with("lst", shell=True, stderr=-2) # Then print mock_print.assert_has_calls([ call("lst"), call("Command failed with CalledProcessError:"), call("/bin/sh: lst: command not found"), ])
def create_admin(opts, msp_name, verbose=False): dir_config = opts['core']['dir_config'] msp_values = opts['msps'][msp_name] ca_values = opts['cas'][msp_values['ca']] # TODO: Refactor this into its own function ca_name = msp_values['ca'] ca_namespace = get_namespace(opts, ca=ca_name) # Obtain CA pod pod_exec = get_pod(namespace=ca_namespace, release=ca_name, app='hlf-ca', verbose=verbose) # Get CA ingress ingress_urls = ingress_read(ca_name + '-hlf-ca', namespace=ca_namespace, verbose=verbose) ca_ingress = ingress_urls[0] # Register the Organisation with the CAs admin_id = pod_exec.execute( ('fabric-ca-client identity list --id {id}').format( id=msp_values['org_admin'])) # If we cannot find the identity, we must create it if not admin_id: pod_exec.execute(( "fabric-ca-client register --id.name {id} --id.secret {pw} --id.attrs 'admin=true:ecert'" ).format(id=msp_values['org_admin'], pw=msp_values['org_adminpw'])) # If our keystore does not exist or is empty, we need to enroll the identity... keystore = path.join(dir_config, msp_name, 'keystore') if not path.isdir(keystore) or not listdir(keystore): execute(( 'FABRIC_CA_CLIENT_HOME={dir} fabric-ca-client enroll ' + '-u https://{id}:{pw}@{ingress} -M {msp_dir} --tls.certfiles {ca_server_tls}' ).format(dir=dir_config, id=msp_values['org_admin'], pw=msp_values['org_adminpw'], ingress=ca_ingress, msp_dir=msp_name, ca_server_tls=ca_values['tls_cert']), verbose=verbose)
def helm_install(repo, app, release, namespace, extra_vars=""): """Install Helm chart. Args: repo (str): Repository or folder from which to install Helm chart. app (str): Helm application name. release (str): Release name on K8S. namespace (str): Namespace where to deploy Helm Chart. extra_vars (str): Extra variables for Helm including version, values files and environmental variables. """ ls_res, _ = execute(f"helm status {release}") if not ls_res: command = f"helm install {repo}/{app} -n {release} --namespace {namespace}" command += extra_vars # Execute execute(command)
def helm_install(repo, app, release, namespace, extra_vars="", verbose=False): """Install Helm chart. Args: repo (str): Repository or folder from which to install Helm chart. app (str): Helm application name. release (str): Release name on K8S. namespace (str): Namespace where to deploy Helm Chart. extra_vars (str): Extra variables for Helm including version, values files and environmental variables. verbose (bool): Verbosity. False by default. """ ls_res, _ = execute("helm status {release}".format(release=release)) if not ls_res: command = "helm install {repo}/{app} -n {name} --namespace {ns}".format( app=app, name=release, ns=namespace, repo=repo) command += extra_vars # Execute execute(command, verbose=verbose)
def helm_upgrade( repo, app, release, namespace, config_yaml=None, env_vars=None, preserve=None, verbose=False, pod_num=1, ): """Upgrade Helm chart. Args: repo (str): Repository or folder from which to install Helm chart. app (str): Helm application name. release (str): Release name on K8S. namespace (str): Namespace where to deploy Helm Chart. config_yaml (str): Values file to ovverride defaults. env_vars (tuple): Environmental variables we wish to store in Helm. preserve (tuple): Set of secrets we wish to get data from to assign to the Helm Chart. verbose (bool): Verbosity. False by default. pod_num (int): Number of pods we wish to have. """ ls_res, _ = execute("helm status {release}".format(release=release)) # Get Helm Env-Vars env_vars_string = helm_env_vars(env_vars) env_vars_string += helm_preserve(namespace, preserve, verbose=verbose) if ls_res: command = "helm upgrade {name} {repo}/{app}".format(app=app, name=release, repo=repo) if config_yaml: command += " -f {}".format(config_yaml) command += env_vars_string # Execute execute(command, verbose=verbose) else: raise Exception("Cannot update a Helm release that is not running") helm_check(app, release, namespace, pod_num)
def genesis_block(opts, verbose=False): ord_namespace = get_namespace(opts, opts['orderers']['msp']) # Change to blockchain materials directory chdir(opts['core']['dir_config']) # Create the genesis block if not path.exists('genesis.block'): # Genesis block creation and storage execute( 'configtxgen -profile OrdererGenesis -outputBlock genesis.block', verbose=verbose) else: print('genesis.block already exists') # Create the genesis block secret secret_from_file(secret=opts['orderers']['secret_genesis'], namespace=ord_namespace, key='genesis.block', filename='genesis.block', verbose=verbose) # Return to original directory chdir(PWD)
def get_pod(namespace, release, app, verbose=False): node_pod = execute( ('kubectl get pods -n {ns} -l "app={app},release={release}" ' + '-o jsonpath="{{.items[0].metadata.name}}"').format(ns=namespace, app=app, release=release), verbose=verbose) if not node_pod: raise ValueError('"node_pod" should contain a value') pod_ex = Executer(node_pod, namespace=namespace, verbose=verbose) return pod_ex
def helm_upgrade(repo, app, release, extra_vars=""): """Upgrade Helm chart. Args: repo (str): Repository or folder from which to install Helm chart. app (str): Helm application name. release (str): Release name on K8S. extra_vars (str): Extra variables for Helm including version, values files and environmental variables. """ ls_res, _ = execute(f"helm status {release}") if ls_res: command = f"helm upgrade {release} {repo}/{app}" command += extra_vars or "" # Execute execute(command) else: raise Exception("Cannot update a Helm release that is not running")
def helm_upgrade(repo, app, release, extra_vars="", verbose=False): """Upgrade Helm chart. Args: repo (str): Repository or folder from which to install Helm chart. app (str): Helm application name. release (str): Release name on K8S. extra_vars (str): Extra variables for Helm including version, values files and environmental variables. verbose (bool): Verbosity. False by default. """ ls_res, _ = execute("helm status {release}".format(release=release)) if ls_res: command = "helm upgrade {name} {repo}/{app}".format(app=app, name=release, repo=repo) command += extra_vars or "" # Execute execute(command, verbose=verbose) else: raise Exception("Cannot update a Helm release that is not running")
def create_admin(opts, msp_name): """Create an admin identity. Args: opts (dict): Nephos options dict. msp_name (str): Name of Membership Service Provider. """ dir_config = opts["core"]["dir_config"] dir_crypto = opts["core"]["dir_crypto"] msp_values = opts["msps"][msp_name] ca_values = opts["cas"][msp_values["ca"]] # TODO: Refactor this into its own function ca_name = msp_values["ca"] ca_namespace = get_namespace(opts, ca=ca_name) # Get CA ingress ingress_urls = ingress_read(ca_name + "-hlf-ca", namespace=ca_namespace) ca_ingress = ingress_urls[0] # Register the Organisation with the CAs register_id( ca_namespace, msp_values["ca"], msp_values["org_admin"], msp_values["org_adminpw"], admin=True, ) # TODO: Can we reuse the Enroll function above? # If our keystore does not exist or is empty, we need to enroll the identity... keystore = join(dir_crypto, msp_name, "keystore") if not isdir(keystore) or not listdir(keystore): execute(( f"FABRIC_CA_CLIENT_HOME={dir_config} fabric-ca-client enroll " + f"-u https://{msp_values['org_admin']}:{msp_values['org_adminpw']}@{ca_ingress} " + f"-M {join(dir_crypto, msp_name)} --tls.certfiles {abspath(ca_values['tls_cert'])}" ), )
def helm_install(repo, app, release, namespace, config_yaml=None, env_vars=None, verbose=False, pod_num=1): ls_res = execute('helm status {release}'.format(release=release)) # Get Helm Env-Vars env_vars_string = helm_env_vars(namespace, env_vars, verbose=verbose) if not ls_res: command = 'helm install {repo}/{app} -n {name} --namespace {ns}'.format( app=app, name=release, ns=namespace, repo=repo) if config_yaml: command += ' -f {}'.format(config_yaml) command += env_vars_string # Execute execute(command, verbose=verbose) helm_check(app, release, namespace, pod_num)
def execute(self, command): """Execute a command in pod. Args: command (str): Command to execute. Returns: tuple: 2-tuple of execution info: 1) result of the command, if successful, None if not; 2) and error, if command failed, None if not. """ result, error = execute(self.prefix_exec + command) return result, error
def logs(self, tail=-1, since_time=None): """Get logs from pod. Args: tail (int): How many lines of logs to obtain? Returns: str: Logs contained in pod. """ command = f"--tail={tail}" if since_time: command += f" --since-time='{since_time}'" result, _ = execute(self.prefix_logs + command) return result