def uninstall(ctx):
"""
Removes resources and deployments created by this installer from a Kubernetes cluster.
"""
if click.confirm(("Are you sure you wish to uninstall WALKOFF from your Kubernetes cluster? "
"(This will only uninstall components that WALKOFF created through this installer.)")):
k8s_api = None
try:
config_dir = os.environ.get('KUBECONFIG', os.path.join(os.path.expanduser("~"), ".kube", "config"))
config_dir = click.prompt("Enter location of kubernetes config",
default=config_dir)
contexts, current = config.list_kube_config_contexts(config_file=config_dir)
contexts = [context["name"] for context in contexts]
current = current["name"]
context = click.prompt("Available contexts: {}\nEnter context to uninstall WALKOFF from: ".format(contexts),
default=current)
config.load_kube_config(config_file=config_dir, context=context)
k8s_api = k8s_client.CoreV1Api()
except IOError as e:
print("Could not open config: {}".format(e))
namespaces = k8s_api.list_namespace()
namespaces = [ns.metadata.name for ns in namespaces.items]
namespace = click.prompt(
"Available namespaces: {}\nEnter namespace to uninstall WALKOFF from".format(namespaces),
default="default")
tiller_namespace = click.prompt('Enter the namespace your Tiller service resides in',
default='kube-system')
kubectl_command(['delete', 'cert', 'walkoff-cert'], namespace, exit_on_err=False)
kubectl_command(['delete', 'issuer', 'walkoff-ca-issuer'], namespace, exit_on_err=False)
kubectl_command(['delete', 'secrets',
'walkoff-redis-secret',
'walkoff-zmq-private-keys', 'walkoff-zmq-public-keys',
'walkoff-postgres-execution-secret', 'walkoff-postgres-secret',
'walkoff-ca-key-pair'],
namespace, exit_on_err=False)
kubectl_command(['delete', 'pvc',
'data-execution-db-postgresql-0',
'redis-data-walkoff-redis-master-0',
'data-walkoff-db-postgresql-0'],
namespace, exit_on_err=False)
kubectl_command(['delete', 'crd',
'certificates.certmanager.k8s.io',
'clusterissuers.certmanager.k8s.io',
'issuers.certmanager.k8s.io'],
namespace, exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-redis'], tiller_namespace, exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-db'], tiller_namespace, exit_on_err=False)
helm_command(['del', '--purge', 'execution-db'], tiller_namespace, exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-cert-manager'], tiller_namespace, exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-deployment'], tiller_namespace, exit_on_err=False)
def __init__(self):
'''
Initialize connection to Kubernetes
'''
config.load_kube_config()
self.client = client.CoreV1Api()
self.api_client = client.BatchV1Api()
def apply_specs(namespace,
sql_servers,
ag_services,
operator_yaml_file=OPERATOR_YAML_FILENAME,
sql_secrets_yaml_file=SQL_SECRETS_YAML_FILENAME,
pv_yaml_file=PV_YAML_FILENAME,
sqlserver_yaml_file=SQLSERVER_YAML_FILENAME,
ag_services_yaml_file=AG_SERVICES_YAML_FILENAME):
kconfig.load_kube_config()
operator_deployed = deploy_operator(namespace, operator_yaml_file)
if operator_deployed:
log(LogLevel.ALL, "Successfully deployed mssql-operator")
log(LogLevel.ALL)
kubectl(["apply", "-f", sql_secrets_yaml_file])
if pv_yaml_file:
pvs_deployed = deploy_pv(namespace, sql_servers, pv_yaml_file)
log(LogLevel.ALL, "Successfully deployed Persistent Volumes",
pvs_deployed)
log(LogLevel.ALL)
sql_pods_deployed = (deploy_sqlservers(namespace, sql_servers,
sqlserver_yaml_file))
log(LogLevel.ALL, "Successfully deployed SQL pods", sql_pods_deployed)
log(LogLevel.ALL)
ag_services_deployed = (deploy_ag_services(namespace, ag_services,
ag_services_yaml_file))
log(LogLevel.ALL, "Successfully deployed AG services",
ag_services_deployed)
log(LogLevel.ALL)
def main():
global_options = {}
logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(message)s')
logging.getLogger('kubernetes').setLevel(logging.WARNING)
try:
k8s_config.load_kube_config()
_, context = k8s_config.list_kube_config_contexts()
region = context['context']['cluster']
domain = 'cc.{}.cloud.sap'.format(region)
global_options['own_namespace'] = 'kube-system' #context['context']['namespace']
except IOError:
from os import environ
environ['KUBERNETES_SERVICE_HOST'] = 'kubernetes.default'
k8s_config.load_incluster_config()
with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as f:
global_options['own_namespace'] = f.read()
with open('/etc/resolv.conf', 'r') as f:
for l in f:
if re.match('^search\s+', l):
_, domain = l.rsplit(' ', 1)
domain = domain.strip()
configurator = Configurator(domain, global_options)
configurator.poll_config()
discovery = DnsDiscovery(domain, configurator.global_options)
discovery.register(re.compile(six.b('\Avc-[a-z]+-?\d+\Z')), configurator)
while True:
discovery.discover()
configurator.poll()
sleep(10)
def __init__(self, config):
self._labels = config['labels']
self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
self._namespace = config.get('namespace') or 'default'
self._role_label = config.get('role_label', 'role')
config['namespace'] = ''
super(Kubernetes, self).__init__(config)
self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
retry_exceptions=(KubernetesRetriableException, HTTPException,
HTTPError, socket.error, socket.timeout))
self._ttl = None
try:
k8s_config.load_incluster_config()
except k8s_config.ConfigException:
k8s_config.load_kube_config(context=config.get('context', 'local'))
self.__subsets = None
use_endpoints = config.get('use_endpoints') and (config.get('patronictl') or 'pod_ip' in config)
if use_endpoints:
addresses = [k8s_client.V1EndpointAddress(ip=config['pod_ip'])]
ports = []
for p in config.get('ports', [{}]):
port = {'port': int(p.get('port', '5432'))}
port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
ports.append(k8s_client.V1EndpointPort(**port))
self.__subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
self._api = CoreV1ApiProxy(use_endpoints)
self.set_retry_timeout(config['retry_timeout'])
self.set_ttl(config.get('ttl') or 30)
self._leader_observed_record = {}
self._leader_observed_time = None
self._leader_resource_version = None
self._leader_observed_subsets = []
self.__do_not_watch = False
def k8s_sdk():
from kubernetes import client, config
config.load_kube_config(config_file='../samples/0000-0000-0000-0000.k8s.yaml')
v1 = client.CoreV1Api()
ret = v1.list_pod_for_all_namespaces(watch=False)
for i in ret.items:
print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
return "GET /k8s/sdk"
def _load_kube_config(in_cluster):
from kubernetes import config, client
if in_cluster:
config.load_incluster_config()
return client.CoreV1Api()
else:
config.load_kube_config()
return client.CoreV1Api()
def create_namespace(namespace):
log(LogLevel.INFO, "Creating namespace:", namespace)
kconfig.load_kube_config()
core_v1_api = kclient.CoreV1Api()
v1_ns = kclient.V1Namespace(metadata=kclient.V1ObjectMeta(name=namespace))
core_v1_api.create_namespace(body=v1_ns)
return v1_ns
def _load_kube_config(in_cluster, cluster_context, config_file):
if not has_kubernetes:
raise _import_err
if in_cluster:
config.load_incluster_config()
else:
config.load_kube_config(config_file=config_file, context=cluster_context)
return client.CoreV1Api()
def client(self, group, version):
client_config = Configuration()
config.load_kube_config(self.config_file, client_configuration=client_config)
client_config.proxy = self.http_proxy
api_client = ApiClient(configuration=client_config)
log.debug('connecting to %s' % (api_client.configuration.host))
# e.g. client.CoreV1Api()
return getattr(client, '%s%sApi' % (group, version))(api_client)
def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
k8s_client = client.ApiClient()
k8s_api = utils.create_from_yaml(k8s_client, "nginx-deployment.yaml")
deps = k8s_api.read_namespaced_deployment("nginx-deployment", "default")
print("Deployment {0} created".format(deps.metadata.name))
def init_openshift_client(self):
"""
Method to get a OpenShift client connected to remote or local OpenShift
"""
kubecfg_path = os.environ.get('KUBECFG_PATH')
if kubecfg_path is None:
config.load_kube_config()
else:
config.load_kube_config(config_file=kubecfg_path)
self.kube_client = k_client.CoreV1Api()
self.kube_v1_batch_client = k_client.BatchV1Api()
def __init__(self):
self.kubecfg_path = os.environ.get('KUBECFG_PATH')
if self.kubecfg_path is None:
config.load_kube_config()
else:
config.load_kube_config(config_file='/tmp/.kube/config')
self.kube_client = client.CoreV1Api()
self.kube_v1_batch_client = client.BatchV1Api()
self.kube_v1_delete = client.V1DeleteOptions()
self.namespace = os.getenv("OPENSHIFTMGR_PROJECT", "myproject")
self.job_id = os.getenv("SWIFT_KEY")
def init_kube_client(self):
"""
Method to get a kube client connected to remote or local kube api
"""
kubecfg_path = os.environ.get('KUBECFG_PATH')
if kubecfg_path is None:
config.load_kube_config()
else:
config.load_kube_config(config_file='/tmp/.kube/config')
self.kube_client = k_client.CoreV1Api()
self.kube_v1_batch_client = k_client.BatchV1Api()
self.kube_v1_delete = k_client.V1DeleteOptions()
def __init__(self, context):
super(KubernetesController, self).__init__(context)
self._namespace = 'default'
self._labels = {"application": "patroni"}
self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
from kubernetes import client as k8s_client, config as k8s_config
k8s_config.load_kube_config(context='local')
self._client = k8s_client
self._api = self._client.CoreV1Api()
def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
with open(path.join(path.dirname(__file__), "nginx-deployment.yaml")) as f:
dep = yaml.load(f)
k8s_beta = client.ExtensionsV1beta1Api()
resp = k8s_beta.create_namespaced_deployment(
body=dep, namespace="default")
print("Deployment created. status='%s'" % str(resp.status))
def _load_kube_config(in_cluster, cluster_context):
from kubernetes import config, client
if in_cluster:
config.load_incluster_config()
return client.CoreV1Api()
else:
if cluster_context is None:
config.load_kube_config()
return client.CoreV1Api()
else:
return client.CoreV1Api(
api_client=config.new_client_from_config(context=cluster_context))
def serve(self):
# For deployed clusters, we should always be running inside
# a Rook cluster. For development convenience, also support
# running outside (reading ~/.kube config)
if self._in_cluster():
config.load_incluster_config()
cluster_name = os.environ['ROOK_CLUSTER_NAME']
else:
self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~")
config.load_kube_config()
cluster_name = "rook"
# So that I can do port forwarding from my workstation - jcsp
from kubernetes.client import configuration
configuration.verify_ssl = False
self._k8s = client.CoreV1Api()
try:
# XXX mystery hack -- I need to do an API call from
# this context, or subsequent API usage from handle_command
# fails with SSLError('bad handshake'). Suspect some kind of
# thread context setup in SSL lib?
self._k8s.list_namespaced_pod(cluster_name)
except ApiException:
# Ignore here to make self.available() fail with a proper error message
pass
self._rook_cluster = RookCluster(
self._k8s,
cluster_name)
# In case Rook isn't already clued in to this ceph
# cluster's existence, initialize it.
# self._rook_cluster.init_rook()
self._initialized.set()
while not self._shutdown.is_set():
# XXX hack (or is it?) to kick all completions periodically,
# in case we had a caller that wait()'ed on them long enough
# to get persistence but not long enough to get completion
global all_completions
self.wait(all_completions)
all_completions = filter(lambda x: not x.is_complete,
all_completions)
self._shutdown.wait(5)
def _load_kube_config(in_cluster, cluster_context, config_file):
from kubernetes import config, client
if in_cluster:
config.load_incluster_config()
else:
config.load_kube_config(config_file=config_file, context=cluster_context)
if PY2:
# For connect_get_namespaced_pod_exec
from kubernetes.client import Configuration
configuration = Configuration()
configuration.assert_hostname = False
Configuration.set_default(configuration)
return client.CoreV1Api()
def __init__(self,
job_to_label_mapping=None,
tf_server_port=8470,
rpc_layer='grpc',
override_client=None):
"""Initializes a new KubernetesClusterResolver.
This initializes a new Kubernetes Cluster Resolver. The Cluster Resolver
will attempt to talk to the Kubernetes master to retrieve all the instances
of pods matching a label selector.
Args:
job_to_label_mapping: A mapping of TensorFlow jobs to label selectors.
This allows users to specify many TensorFlow jobs in one Cluster
Resolver, and each job can have pods belong with different label
selectors. For example, a sample mapping might be
```
{'worker': ['job-name=worker-cluster-a', 'job-name=worker-cluster-b'],
'ps': ['job-name=ps-1', 'job-name=ps-2']}
```
tf_server_port: The port the TensorFlow server is listening on.
rpc_layer: (Optional) The RPC layer TensorFlow should use to communicate
between tasks in Kubernetes. Defaults to 'grpc'.
override_client: The Kubernetes client (usually automatically retrieved
using `from kubernetes import client as k8sclient`). If you pass this
in, you are responsible for setting Kubernetes credentials manually.
Raises:
ImportError: If the Kubernetes Python client is not installed and no
`override_client` is passed in.
RuntimeError: If autoresolve_task is not a boolean or a callable.
"""
if _KUBERNETES_API_CLIENT_INSTALLED:
k8sconfig.load_kube_config()
if not job_to_label_mapping:
job_to_label_mapping = {'worker': ['job-name=tensorflow']}
if not override_client and not _KUBERNETES_API_CLIENT_INSTALLED:
raise ImportError('The Kubernetes Python client must be installed before'
'using the Kubernetes Cluster Resolver. To install the'
'Kubernetes Python client, run `pip install '
'kubernetes` on your command line.')
self._job_to_label_mapping = job_to_label_mapping
self._tf_server_port = tf_server_port
self._override_client = override_client
self.task_type = None
self.task_id = None
self.rpc_layer = rpc_layer
def kube_apis(cli_arguments) -> KubeApis:
"""
Set up kubernets-client to operate in cluster.
:param cli_arguments: a set of command-line arguments
:return: KubeApis
"""
context_name = cli_arguments['context']
kubeconfig = cli_arguments['kubeconfig']
config.load_kube_config(config_file=kubeconfig, context=context_name, persist_config=False)
v1 = client.CoreV1Api()
extensions_v1_beta1 = client.ExtensionsV1beta1Api()
rbac_v1_beta1 = client.RbacAuthorizationV1beta1Api()
return KubeApis(v1, extensions_v1_beta1, rbac_v1_beta1)
def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
extensions_v1beta1 = client.ExtensionsV1beta1Api()
# Create a deployment object with client-python API. The deployment we
# created is same as the `nginx-deployment.yaml` in the /examples folder.
deployment = create_deployment_object()
create_deployment(extensions_v1beta1, deployment)
update_deployment(extensions_v1beta1, deployment)
delete_deployment(extensions_v1beta1)
def __init__(self, **kwargs):
self.svcaccount = kwargs.get('svcaccount','default')
self.namespace = kwargs.get('namespace','default')
if kwargs.get('kubeconfig') == 'incluster':
log.info('load incluster config')
config.load_incluster_config()
else:
cfg = kwargs.get('kubeconfig')
log.info('load config %s', cfg)
if not cfg:
config.load_kube_config()
else:
config.load_kube_config(cfg)
import urllib3
urllib3.disable_warnings()
def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
v1 = client.CoreV1Api()
count = 10
w = watch.Watch()
for event in w.stream(v1.list_namespace, timeout_seconds=10):
print("Event: %s %s" % (event['type'], event['object'].metadata.name))
count -= 1
if not count:
w.stop()
print("Ended.")
def __init__(self, api_client=None, config_file=None):
config = client.Configuration()
if api_client:
self.api_client = api_client
else:
if not config.api_client:
if config_file is not None and config_file != "":
konfig.load_kube_config(config_file=config_file)
else:
konfig.load_incluster_config()
config.api_client = klient.ApiClient()
# K8S python client doesn't provide any way to configure the
# client pool size, so we inject the value here
config.api_client.rest_client.pool_manager.connection_pool_kw[
'maxsize'] = 20
self.api_client = config.api_client
self._watch = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Load kubernetes config here, since this is a Singleton and
# so this __init__ will be run way before anything else gets run.
try:
config.load_incluster_config()
except config.ConfigException:
config.load_kube_config()
self.api = shared_client(self.api_group_name)
# FIXME: Protect against malicious labels?
self.label_selector = ','.join(['{}={}'.format(k, v) for k, v in self.labels.items()])
self.field_selector = ','.join(['{}={}'.format(k, v) for k, v in self.fields.items()])
self.first_load_future = Future()
self._stop_event = threading.Event()
self.start()
def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
print("Supported APIs (* is preferred version):")
print("%-20s %s" %
("core", ",".join(client.CoreApi().get_api_versions().versions)))
for api in client.ApisApi().get_api_versions().groups:
versions = []
for v in api.versions:
name = ""
if v.version == api.preferred_version.version and len(
api.versions) > 1:
name += "*"
name += v.version
versions.append(name)
print("%-40s %s" % (api.name, ",".join(versions)))
def kube_client(request, kube_ns):
"""fixture for the Kubernetes client object.
skips test that require kubernetes if kubernetes cannot be contacted
"""
load_kube_config()
client = shared_client('CoreV1Api')
try:
namespaces = client.list_namespace(_request_timeout=3)
except Exception as e:
pytest.skip("Kubernetes not found: %s" % e)
if not any(ns.metadata.name == kube_ns for ns in namespaces.items):
print("Creating namespace %s" % kube_ns)
client.create_namespace(V1Namespace(metadata=dict(name=kube_ns)))
else:
print("Using existing namespace %s" % kube_ns)
# delete the test namespace when we finish
request.addfinalizer(lambda: client.delete_namespace(kube_ns, {}))
return client
def __init__(self, config_file='/etc/kubernetes/admin.conf', logger=None, cluster=None):
if cluster:
config_file = cluster['kube_config_file']
cfg = client.Configuration()
config.load_kube_config(config_file=config_file,
client_configuration=cfg)
cfg.assert_hostname = False
if cluster:
(proto, _, port) = cfg.host.split(':')
host = proto + '://' + cluster['master_public_ip'] + ':' + port
cfg.host = host
cfg.verify_ssl = False
self.api_client = client.ApiClient(cfg)
#self.res_v1_beta1_h = client.ApiextensionsV1beta1Api(self.api_client)
self.res_v1_obj_h = client.CustomObjectsApi(self.api_client)
self.v1_h = client.CoreV1Api(self.api_client)
self.v1_h.read_namespace('default')
self.v1_beta_h = client.ExtensionsV1beta1Api(self.api_client)
self.v1_networking = client.NetworkingV1Api(self.api_client)
self.apps_v1_beta1_h = client.AppsV1beta1Api(self.api_client)
self.logger = logger or contrail_logging.getLogger(__name__)
def main():
"""
Change labels of the "minikube" node:
- Add label "foo" with value "bar". This will overwrite the "foo" label
if it already exists.
- Remove the label "baz" from the node.
"""
config.load_kube_config()
api_instance = client.CoreV1Api()
body = {
"metadata": {
"labels": {
"foo": "bar",
"baz": None}
}
}
api_response = api_instance.patch_node("minikube", body)
pprint(api_response)
def main():
#construct & parse args
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--file", required=True,
help="Name of input file containing test params")
ap.add_argument("-pName", "--promPod", required=True,
help="Name of prometheus operator pod inside k8 cluster")
ap.add_argument("-k8url", required=True,
help="URL & port number of k8 cluster we are accessing")
ap.add_argument("-locustF", required=True,
help="Name of locust file to be used for this round of testing")
# add more args here
args = vars(ap.parse_args())
#kubernetes setup
config.load_kube_config()
extensions_v1beta1 = client.ExtensionsV1beta1Api()
batch_v1beta1 = client.BatchV1Api()
clusterConfs = clusterInfo()
# -------- Main testing loop Start ----------
for line in open(args["file"]):
lnArgs = [x.strip() for x in line.split(',')]
if len(lnArgs) != 11: # change val to appropriate cnt later
print("Skipping experiment %s, wrong number of args" % lnArgs[0])
continue
exp_Nm = lnArgs[0]
runtime = lnArgs[1]
clientCnt = lnArgs[3]
hatchRate = lnArgs[2]
clusterConfs.interferenceZone = lnArgs[4]
clusterConfs.interferenceLvl = int(lnArgs[5])
clusterConfs.workflowDeplList['cart'] = lnArgs[6]
clusterConfs.workflowDeplList['catalogue'] = lnArgs[7]
clusterConfs.workflowDeplList['shipping'] = lnArgs[8]
start_po = lnArgs[9]
end_po = lnArgs[10]
# add more var defs here ^ if more args get added to lines (like node color interference is on)
print("Current running experiment: %s\n" % exp_Nm)
testDirPath = testDirInit(exp_Nm) #Create current test's directory
locustDur = runtime + "s"
# setup cluster using input params
print("Configuring cluster to match experiment input\n")
clusterSetup(extensions_v1beta1, batch_v1beta1,clusterConfs)
print("5 second grace period\n")
time.sleep(20)
# build locust command to run locust
locustCmd = "locust --host http://" + args["k8url"] + " -f " + args["locustF"] + " -c " + clientCnt + " -r " + hatchRate + " -t " + locustDur + " --no-web --only-summary --csv=locust"
locustArgs = shlex.split(locustCmd)
print("locust Command: %s\n" % locustCmd)
print("locust CMD args: %s\n" % locustArgs)
# TODO: Later: confirm cluster is setup properly
# Get time-stamp before running test
print("Test %s start\n" % exp_Nm)
# TODO: add perfstat shell cmd in
# Exec perfstat using params passed in through testParam file
# #format: python <script_name> exp_name total_duration output_dir
perfstatCmdString = "python perfstat_driver.py {} {} {}".format(exp_Nm,runtime,testDirPath)
perfstatArgs = shlex.split(perfstatCmdString)
perfstatResultFNm = testDirPath + "/perfstatLog.txt"
with open(perfstatResultFNm, 'w+') as perfstat_f:
p1 = subprocess.call(perfstatArgs, stdout=perfstat_f, stderr=perfstat_f, shell=False)
# Exec vmstat using params passed
vmstatCmdString = "python vmstat_driver.py {} {} {} {} {}".format(exp_Nm,runtime,testDirPath,start_po,end_po)
vmstatArgs = shlex.split(vmstatCmdString)
vmstatResultFNm = testDirPath + "/vmstatLog.txt"
with open(vmstatResultFNm, 'w+') as vmstat_f:
p2 = subprocess.call(vmstatArgs, stdout=vmstat_f, stderr=vmstat_f, shell=False)
print("[debug] start time {}".format(datetime.datetime.now()))
startT = time.time()
# Exec locust command, exporting to CSV file & using params passed in through testParam file
locustResultFNm = testDirPath + "/LocustLog.txt"
with open(locustResultFNm, 'w+') as locust_f:
p = subprocess.call(locustArgs, stdout=locust_f, stderr=locust_f, shell=False)
# Once locust command finishes, get end timestamp
stopT = time.time()
print ("[debug] test is completed. post processing")
# delete the batch jobs
if clusterConfs.interferenceLvl > 0:
deletebatchJobs(batch_v1beta1,clusterConfs)
moveLocustResults(testDirPath)
# TODO: Exec kubectl port forward to prometheus pod ?? (currently, command is being run in separate terminal window)
# Exec Prometheus API query(s) to gather metrics & build resulting csv files
promQueries(startT, stopT, testDirPath)
time.sleep(60)
print("[debug] end time {}".format(datetime.datetime.now()))
print ("[debug] End of Test")
def k8s_init():
config.load_kube_config()
api_instance = client.CoreV1Api()
return api_instance
def k8s_conf():
config_file="%s/../conf/k8s.conf" % app.root_path
contexts, active_context = config.list_kube_config_contexts(config_file)
contexts = [context['name'] for context in contexts]
config.load_kube_config(config_file, context=active_context['name'])
return(config,contexts,config_file)
def initialize_clients(kubeconfig_path):
global cli
config.load_kube_config(kubeconfig_path)
cli = client.CoreV1Api()
#!/usr/bin/python
import os
from kubernetes import client, config
ns = "oneinfo"
def print_endpoint_member(endpoint, ns):
print endpoint
endp = v1.read_namespaced_endpoints(endpoint, ns)
for s in endp.subsets:
if s.addresses == None:
continue
for addr in s.addresses:
print " ", addr.ip
for port in s.ports:
print " ", port.port, port.protocol
config.load_kube_config( os.path.join(os.environ["HOME"], '.kube/config'))
v1 = client.CoreV1Api()
endpts = v1.list_endpoints_for_all_namespaces()
for endpt in endpts.items:
print_endpoint_member(endpt.metadata.name, endpt.metadata.namespace)
def __get_out_of_cluster_url(self, label_selector, k8s_namespace,
load_config):
"""Get the API url for the dashboard when running out of a cluster """
if load_config:
config.load_kube_config()
configuration = client.Configuration()
k8s_client = client.CoreV1Api(_CustomApiClient(configuration))
ret = k8s_client.list_service_for_all_namespaces(
label_selector=label_selector)
if len(ret.items) == 0:
raise ValueError("Couldn't find a deployed dashboard service")
if len(ret.items) > 1:
raise ValueError("Found multiple deployed dashboard services")
service = ret.items[0]
service_type = service.spec.type
if service_type == "ClusterIP":
# cluster ip: up to user to grant access
ip = service.spec.cluster_ip
self.port = service.spec.ports[0].port
elif service_type == "NodePort":
self.port = next(p.node_port for p in service.spec.ports
if p.port == 80)
if service.spec.external_i_ps and len(
service.spec.external_i_ps) > 0:
ip = service.spec.external_i_ps[0]
else:
# try and get public node ip
namespaced_pods = k8s_client.list_namespaced_pod(
k8s_namespace, label_selector=label_selector)
assert len(namespaced_pods.items) == 1
master_pod = namespaced_pods.items[0]
node_name = master_pod.spec.node_name
node = k8s_client.read_node(node_name)
ip = None
internal_ip = None
for address in node.status.addresses:
if address.type == "ExternalIP":
ip = address.address
break
if address.type == "InternalIP":
internal_ip = address.address
if not ip:
# no external IP found, maybe internal works
logging.warning(
"No ExternalIP found for NodePort "
"Service. Trying InternalIP. This only works "
"if the cluster internal IP's are reachable.")
ip = internal_ip
elif service_type == "LoadBalancer":
self.port = service.spec.ports[0].port
if service.status.load_balancer.ingress is None:
raise NotImplementedError(
"Service Type 'LoadBalancer' only works with an "
"'Ingress' ip defined")
ip = service.status.load_balancer.ingress.ip
else:
raise NotImplementedError
return "{ip}:{port}".format(ip=ip, port=self.port)
def create_config(in_cluster=False):
if in_cluster:
logging.info("Loading in-cluster config")
return config.load_incluster_config()
else:
return config.load_kube_config()
def main():
args = get_args()
key_dir = os.path.join(args.tezos_dir, "client")
os.makedirs(key_dir, exist_ok=True)
timestamp = datetime.utcnow().replace(tzinfo=timezone.utc).isoformat()
bootstrap_peers = []
bootstrap_accounts = [
"baker", "bootstrap_account_1", "bootstrap_account_2"
]
k8s_templates = ["common.yaml"]
genesis_key = None
genesis_block = None
try:
genesis_key = get_key(args.docker_image, key_dir, "genesis")
except:
pass
zerotier_network = args.zerotier_network
zerotier_token = args.zerotier_token
if args.create:
if args.cluster in ["minikube", "docker-desktop"]:
k8s_templates.append("pv.yaml")
elif args.cluster == "eks":
k8s_templates.append("eks.yaml")
k8s_templates.append("activate.yaml")
if genesis_key is None:
bootstrap_accounts.append("genesis")
for account in bootstrap_accounts:
gen_key(args.docker_image, key_dir, account)
genesis_key = get_key(args.docker_image, key_dir, "genesis")
bootstrap_peers = []
if args.invite:
k8s_config.load_kube_config()
v1 = k8s_client.CoreV1Api()
bootstrap_peer = args.bootstrap_peer
tezos_config = json.loads(
v1.read_namespaced_config_map("tezos-config",
"tqtezos").data["config.json"])
service = v1.read_namespaced_service("tezos-net", "tqtezos")
node_port = (v1.read_namespaced_service(
"tezos-net", "tqtezos").spec.ports[0].node_port)
bootstrap_peers = [f"{bootstrap_peer}:{node_port}"]
genesis_key = tezos_config["network"]["genesis_parameters"]["values"][
"genesis_pubkey"]
genesis_block = tezos_config["network"]["genesis"]["block"]
timestamp = tezos_config["network"]["genesis"]["timestamp"]
zerotier_config = v1.read_namespaced_config_map(
'zerotier-config', 'tqtezos')
zerotier_network = zerotier_config.data['NETWORK_IDS']
zerotier_token = zerotier_config.data['ZTAUTHTOKEN']
minikube_gw = None
if args.cluster == "minikube":
try:
minikube_route = (subprocess.check_output(
'''minikube ssh "ip route show default"''',
shell=True).decode("utf-8").split())
minikube_gw, minikube_iface = minikube_route[2], minikube_route[4]
minikube_ip = (subprocess.check_output(
'''minikube ssh "ip addr show %s|awk /^[[:space:]]+inet/'{print \$2}'"'''
% minikube_iface,
shell=True,
).decode("utf-8").split("/")[0])
if args.create:
print(
"Add the following line to /etc/exports and restart nfsd.",
file=sys.stderr,
)
print(
'"%s" -alldirs -mapall=%s:%s %s' %
(args.tezos_dir, os.getuid(), os.getgid(), minikube_ip),
file=sys.stderr,
)
except subprocess.CalledProcessError as e:
print("failed to get minikube route %r" % e)
if zerotier_network:
k8s_templates.append("zerotier.yaml")
k8s_objects = []
for template in k8s_templates:
with open(os.path.join(my_path, "deployment", template),
"r") as yaml_template:
k8s_resources = yaml.load_all(yaml_template,
Loader=yaml.FullLoader)
for k in k8s_resources:
if safeget(k, "metadata", "name") == "tezos-var-volume":
if args.cluster == "docker_desktop":
k["spec"]["hostPath"] = {"path": args.tezos_dir}
elif args.cluster == "minikube":
k["spec"]["nfs"] = {
"path": args.tezos_dir,
"server": minikube_gw,
}
if safeget(k, "metadata", "name") == "tezos-pv-claim":
if args.cluster == "eks":
k["spec"]["storageClassName"] = "ebs-sc"
if safeget(k, "metadata", "name") == "tezos-config":
k["data"] = {
"parameters.json":
json.dumps(
get_parameters_config(
args.docker_image,
key_dir,
bootstrap_accounts,
args.bootstrap_mutez,
)),
"config.json":
json.dumps(
get_node_config(
args.chain_name,
genesis_key,
timestamp,
bootstrap_peers,
genesis_block,
)),
}
if safeget(k, "metadata", "name") == "tezos-node":
# set the docker image for the node
k["spec"]["template"]["spec"]["containers"][0][
"image"] = args.docker_image
# if you are the chain creator use a lower bootstrap threshold
if args.create:
node_args = k["spec"]["template"]["spec"][
"containers"][0]["args"]
new_node_args = node_args[:1]
new_node_args.extend(["--bootstrap-threshold", "0"])
new_node_args.extend(node_args[1:])
k["spec"]["template"]["spec"]["containers"][0][
"args"] = new_node_args
# if not os.path.isfile(
# os.path.join(args.tezos_dir, "node", "identity.json")
# ):
# # add the identity job
# k["spec"]["template"]["spec"][
# "initContainers"
# ] = get_identity_job(args.docker_image)
# add the identity job
k["spec"]["template"]["spec"][
"initContainers"] = get_identity_job(args.docker_image)
if args.baker:
k["spec"]["template"]["spec"]["containers"].append(
get_baker(args.docker_image, args.baker_command))
if safeget(k, "metadata", "name") == "activate-job":
k["spec"]["template"]["spec"]["initContainers"][1][
"image"] = args.docker_image
k["spec"]["template"]["spec"]["initContainers"][1][
"args"] = [
"-A",
"tezos-rpc",
"-P",
"8732",
"-d",
"/var/tezos/client",
"-l",
"--block",
"genesis",
"activate",
"protocol",
args.protocol_hash,
"with",
"fitness",
"-1",
"and",
"key",
"genesis",
"and",
"parameters",
"/etc/tezos/parameters.json",
]
k["spec"]["template"]["spec"]["initContainers"][2][
"image"] = args.docker_image
if safeget(k, "metadata", "name") == "zerotier-config":
k["data"]["NETWORK_IDS"] = zerotier_network
k["data"]["ZTAUTHTOKEN"] = zerotier_token
zt_hostname = str(uuid.uuid4())
print(f"zt_hostname: {zt_hostname}", file=sys.stderr)
k["data"]["ZTHOSTNAME"] = zt_hostname
k8s_objects.append(k)
yaml.dump_all(k8s_objects, sys.stdout)
def cli():
"""kuku: Kubernetes templating tool.
Usage:
kuku apply [-v] [-f <FILE>]... [-s <key=value>]... <TEMPLATES_DIR>
kuku delete [-v] [-f <FILE>]... [-s <key=value>]... <TEMPLATES_DIR>
kuku render [-v] [-f <FILE>]... [-s <key=value>]... <TEMPLATES_DIR>
kuku (-h | --help)
kuku --version
Options:
-h --help Show this screen.
-v --verbose Dump debug info to stderr.
--version Show version.
-s KEY=VALUE --set KEY=VALUE Set values on the command line (accepts multiple options or separate values with commas: Ex: -s key1=val1,key2=val2).
-f FILE --file FILE Specify values in a YAML file (accepts multiple options).
Notes:
Resolution of values: --set overrides values in --file by merging. The last value wins.
"""
# Parse cli arguments from docstring
arguments = docopt(str(cli.__doc__), version=__version__)
# Load k8s config -- needed to the correct API versions of k8s
config.load_kube_config()
# Find all templates
try:
templates = find(arguments["<TEMPLATES_DIR>"])
except ValueError as e:
print(e)
exit(1)
raise
try:
# Resolve values
context = resolve(arguments["--set"], arguments["--file"])
except ValueError as e:
print(e)
exit(1)
raise
# Render templates with resolved context
rendering = render(context, templates)
output = dump(rendering)
if arguments["render"]:
# print yaml
print(output)
if arguments["apply"]:
print("Not implemented yet. Use kuku render ... | kubectl apply -f-")
exit(1)
if arguments["delete"]:
print("Not implemented yet. Use kuku render ... | kubectl delete -f-")
exit(1)
if arguments["--verbose"]:
print(output, file=sys.stderr)
def __init__(self):
config.load_kube_config()
self.api = client.CustomObjectsApi()
self.coreApi = client.CoreV1Api()
def deploy_kubernetes(self):
"""
Deploy the Redis Pub/Sub service in Kubernetes orchestrator.
"""
if 'namespace' not in self.params.orchestrator_params:
self.params.orchestrator_params['namespace'] = "default"
from kubernetes import client, config
container = client.V1Container(
name=self.redis_server_name,
image='redis:4-alpine',
resources=client.V1ResourceRequirements(limits={
"cpu": "8",
"memory": "4Gi"
# "nvidia.com/gpu": "0",
}),
)
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
labels={'app': self.redis_server_name}),
spec=client.V1PodSpec(containers=[container]))
deployment_spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector=client.V1LabelSelector(
match_labels={'app': self.redis_server_name}))
deployment = client.V1Deployment(
api_version='apps/v1',
kind='Deployment',
metadata=client.V1ObjectMeta(
name=self.redis_server_name,
labels={'app': self.redis_server_name}),
spec=deployment_spec)
config.load_kube_config()
api_client = client.AppsV1Api()
try:
print(self.params.orchestrator_params)
api_client.create_namespaced_deployment(
self.params.orchestrator_params['namespace'], deployment)
except client.rest.ApiException as e:
print("Got exception: %s\n while creating redis-server", e)
return False
core_v1_api = client.CoreV1Api()
service = client.V1Service(
api_version='v1',
kind='Service',
metadata=client.V1ObjectMeta(name=self.redis_service_name),
spec=client.V1ServiceSpec(selector={'app': self.redis_server_name},
ports=[
client.V1ServicePort(
protocol='TCP',
port=6379,
target_port=6379)
]))
try:
core_v1_api.create_namespaced_service(
self.params.orchestrator_params['namespace'], service)
self.params.redis_address = '{}.{}.svc'.format(
self.redis_service_name,
self.params.orchestrator_params['namespace'])
self.params.redis_port = 6379
return True
except client.rest.ApiException as e:
print(
"Got exception: %s\n while creating a service for redis-server",
e)
return False
def do_test():
kubeconfig_path = shared_test_code.get_kubeconfig_path()
print("Loading k8s config: {}".format(kubeconfig_path))
config.load_kube_config(config_file=kubeconfig_path)
# Get kubectl command
kubectl_cmd = shared_test_code.get_kubectl_command()
# Ensure Helm Akri installation applied CRDs and set up agent and controller
print("Checking for CRDs")
if not shared_test_code.crds_applied():
print("CRDs not applied by helm chart")
return False
print("Checking for initial Akri state")
if not shared_test_code.check_akri_state(1, 1, 2, 2, 1, 2):
print("Akri not running in expected state")
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
#
# Check agent responds to dynamic offline/online resource
#
print("Writing to Agent pod {} that device offline".format(
shared_test_code.agent_pod_name))
os.system(
'sudo {} exec -i {} -- /bin/sh -c "echo "OFFLINE" > /tmp/debug-echo-availability.txt"'
.format(kubectl_cmd, shared_test_code.agent_pod_name))
print("Checking Akri state after taking device offline")
if not shared_test_code.check_akri_state(1, 1, 0, 0, 0, 0):
print("Akri not running in expected state after taking device offline")
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
# Do back online scenario
print("Writing to Agent pod {} that device online".format(
shared_test_code.agent_pod_name))
os.system(
'sudo {} exec -i {} -- /bin/sh -c "echo "ONLINE" > /tmp/debug-echo-availability.txt"'
.format(kubectl_cmd, shared_test_code.agent_pod_name))
print("Checking Akri state after bringing device back online")
if not shared_test_code.check_akri_state(1, 1, 2, 2, 1, 2):
print(
"Akri not running in expected state after bringing device back online"
)
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
#
# Check that slot reconciliation is working on agent
#
print("Check logs for Agent slot-reconciliation for pod {}".format(
shared_test_code.agent_pod_name))
temporary_agent_log_path = "/tmp/agent_log.txt"
for x in range(3):
log_result = subprocess.run('sudo {} logs {} > {}'.format(
kubectl_cmd, shared_test_code.agent_pod_name,
temporary_agent_log_path),
shell=True)
if log_result.returncode == 0:
print("Successfully stored Agent logs in {}".format(
temporary_agent_log_path))
break
print(
"Failed to get logs from {} pod with result {} on attempt {} of 3".
format(shared_test_code.agent_pod_name, log_result, x))
if x == 2:
return False
grep_result = subprocess.run([
'grep', "get_node_slots - crictl called successfully",
temporary_agent_log_path
])
if grep_result.returncode != 0:
print(
"Akri failed to successfully connect to crictl via the CRI socket with return value of {}",
grep_result)
# Log information to understand why error occurred
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('grep get_node_slots {}'.format(temporary_agent_log_path))
return False
#
# Check that broker is recreated if it is deleted
#
broker_pod_selector = "{}={}".format(
shared_test_code.CONFIGURATION_LABEL_NAME,
shared_test_code.DEBUG_ECHO_NAME)
brokers_info = shared_test_code.get_running_pod_names_and_uids(
broker_pod_selector)
if len(brokers_info) != 2:
print("Expected to find 2 broker pods but found: {}",
len(brokers_info))
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
# There is a possible race condition here between when the `kubectl delete pod` returns,
# when check_broker_pod_state validates that the pod is gone, and when the check_akri_state
# validates that the broker pod has been restarted
broker_pod_name = sorted(brokers_info.keys())[0]
delete_pod_command = 'sudo {} delete pod {}'.format(
kubectl_cmd, broker_pod_name)
print("Deleting broker pod: {}".format(delete_pod_command))
os.system(delete_pod_command)
# Create kube client
v1 = client.CoreV1Api()
# Wait for there to be 2 brokers pods again
if not shared_test_code.check_broker_pods_state(v1, 2):
print(
"Akri not running in expected state after broker pod restoration should have happened"
)
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
restored_brokers_info = shared_test_code.get_running_pod_names_and_uids(
broker_pod_selector)
if len(restored_brokers_info) != 2:
print("Expected to find 2 broker pods but found: {}",
len(restored_brokers_info))
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
# Make sure that the deleted broker uid is different from the restored broker pod uid ... signifying
# that the Pod was restarted
print("Restored broker pod uid should differ from original broker pod uid")
if brokers_info[broker_pod_name] == restored_brokers_info[broker_pod_name]:
print(
"Restored broker pod uid [{}] should differ from original broker pod uid [{}]"
.format(brokers_info[broker_pod_name],
restored_brokers_info[broker_pod_name]))
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
# Do cleanup scenario
print("Deleting Akri configuration: {}".format(
shared_test_code.DEBUG_ECHO_NAME))
os.system("sudo {} delete akric {}".format(
kubectl_cmd, shared_test_code.DEBUG_ECHO_NAME))
print("Checking Akri state after deleting configuration")
if not shared_test_code.check_akri_state(1, 1, 0, 0, 0, 0):
print(
"Akri not running in expected state after deleting configuration")
os.system('sudo {} get pods,services,akric,akrii --show-labels'.format(
kubectl_cmd))
os.system('sudo {} describe pod'.format(kubectl_cmd))
return False
return True
#!/usr/bin/env python3
import os
import sys
import pprint
from kubernetes import client, config, watch
try:
name = sys.argv[1]
except:
name = ""
#Configs can be set in Configuration class directly or using helper utility
config.load_kube_config()
v1 = client.CoreV1Api()
ret = v1.list_pod_for_all_namespaces(watch=False)
for pod in ret.items:
if (name.lower() in pod.metadata.name.lower()):
print("=" * (len(pod.metadata.name) + 20))
print(pod.metadata.name, )
print("=" * (len(pod.metadata.name) + 20))
try:
response = v1.read_namespaced_pod_log(
name=pod.metadata.name, namespace=pod.metadata.namespace)
for line in response.splitlines():
print(" ", line)
except:
pass
print("")
def load_kube_config():
if "AWS_WEB_IDENTITY_TOKEN_FILE" in os.environ and "eks.amazonaws.com" in os.environ[
"AWS_WEB_IDENTITY_TOKEN_FILE"]:
k8_config.load_incluster_config()
else:
k8_config.load_kube_config()
def init():
check_env()
config.load_kube_config()
def _load_config(self):
"""Load kubernetes configuration."""
if self.in_cluster:
config.load_incluster_config()
else:
config.load_kube_config(context=self.context)
def __init__(self,
namespace=None,
service_type=None,
gs_image=None,
etcd_image=None,
zookeeper_image=None,
gie_graph_manager_image=None,
coordinator_name=None,
coordinator_service_name=None,
etcd_cpu=None,
etcd_mem=None,
zookeeper_cpu=None,
zookeeper_mem=None,
gie_graph_manager_cpu=None,
gie_graph_manager_mem=None,
engine_cpu=None,
engine_mem=None,
vineyard_cpu=None,
vineyard_mem=None,
vineyard_shared_mem=None,
image_pull_policy=None,
image_pull_secrets=None,
volumes=None,
num_workers=None,
instance_id=None,
log_level=None,
timeout_seconds=None,
waiting_for_delete=None,
delete_namespace=None,
**kwargs):
try:
kube_config.load_incluster_config()
except: # noqa: E722
kube_config.load_kube_config()
self._api_client = kube_client.ApiClient()
self._core_api = kube_client.CoreV1Api(self._api_client)
self._app_api = kube_client.AppsV1Api(self._api_client)
self._instance_id = instance_id
# random for multiple k8s cluster in the same namespace
self._engine_name = self._engine_name_prefix + self._instance_id
self._etcd_name = self._etcd_name_prefix + self._instance_id
self._etcd_service_name = self._etcd_service_name_prefix + self._instance_id
self._gie_graph_manager_name = (self._gie_graph_manager_name_prefix +
self._instance_id)
self._gie_graph_manager_service_name = (
self._gie_graph_manager_service_name_prefix + self._instance_id)
self._vineyard_service_name = (self._vineyard_service_name_prefix +
self._instance_id)
self._namespace = namespace
self._service_type = service_type
self._num_workers = num_workers
self._coordinator_name = coordinator_name
self._coordinator_service_name = coordinator_service_name
self._resource_object = []
# engine container info
self._gs_image = gs_image
self._engine_cpu = engine_cpu
self._engine_mem = engine_mem
# vineyard container info
self._vineyard_cpu = vineyard_cpu
self._vineyard_mem = vineyard_mem
self._vineyard_shared_mem = vineyard_shared_mem
# etcd pod info
self._etcd_image = etcd_image
self._etcd_cpu = etcd_cpu
self._etcd_mem = etcd_mem
# zookeeper pod info
self._zookeeper_image = zookeeper_image
self._zookeeper_cpu = zookeeper_cpu
self._zookeeper_mem = zookeeper_mem
# interactive engine graph manager info
self._gie_graph_manager_image = gie_graph_manager_image
self._gie_graph_manager_cpu = gie_graph_manager_cpu
self._gie_graph_manager_mem = gie_graph_manager_mem
self._image_pull_policy = image_pull_policy
# image pull secrets
self._etcd_endpoint = None
if image_pull_secrets is not None:
self._image_pull_secrets = image_pull_secrets.split(",")
else:
self._image_pull_secrets = []
self._volumes = json.loads(volumes)
self._host0 = None
self._pod_name_list = None
self._pod_ip_list = None
self._pod_host_ip_list = None
self._analytical_engine_endpoint = None
self._vineyard_service_endpoint = None
self._closed = False
self._glog_level = parse_as_glog_level(log_level)
self._timeout_seconds = timeout_seconds
self._waiting_for_delete = waiting_for_delete
self._delete_namespace = delete_namespace
self._analytical_engine_process = None
# 8000 ~ 9000 is exposed
self._learning_engine_ports_usage = 8000
self._graphlearn_services = dict()
self._learning_instance_processes = {}
def online_install(ctx):
click.echo(
'Installing WALKOFF to Kubernetes cluster with Internet access.')
try:
config_dir = os.environ.get(
'KUBECONFIG',
os.path.join(os.path.expanduser("~"), ".kube", "config"))
config_dir = click.prompt("Enter location of kubernetes config",
default=config_dir)
contexts, current = config.list_kube_config_contexts(
config_file=config_dir)
contexts = [context["name"] for context in contexts]
current = current["name"]
context = click.prompt(
"Available contexts: {}\nEnter context to install WALKOFF to".
format(contexts),
default=current)
config.load_kube_config(config_file=config_dir, context=context)
k8s_api = k8s_client.CoreV1Api()
k8s_custom_api = k8s_client.CustomObjectsApi()
except IOError as e:
print("Could not open config: {}".format(e))
return
namespaces = k8s_api.list_namespace()
namespaces = [ns.metadata.name for ns in namespaces.items]
namespace = click.prompt(
"Available namespaces: {}\nEnter namespace to install WALKOFF in".
format(namespaces),
default="default")
if namespace not in namespaces:
if click.confirm("{} does not exist - do you want to create it now?"):
new_namespace = k8s_client.V1Namespace(
metadata={'name': namespace})
try:
k8s_api.create_namespace(new_namespace)
except k8s_client.rest.ApiException as e:
click.echo("Error creating namespace:\n{}".format(str(e)))
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
tiller_namespace = click.prompt(
'Enter the namespace your Tiller service resides in',
default='kube-system')
click.echo("Generating ZMQ certificates for WALKOFF.")
if subprocess.call(['python', 'scripts/generate_certificates.py']) != 0:
click.echo("Error generating ZMQ certificates.")
return
click.echo("Adding ZMQ certificates to Kubernetes secrets.")
kubectl_command([
'create', 'secret', 'generic', 'walkoff-zmq-private-keys',
'--from-file=server.key_secret=./.certificates/private_keys/server.key_secret',
'--from-file=client.key_secret=./.certificates/private_keys/client.key_secret'
], namespace)
kubectl_command([
'create', 'secret', 'generic', 'walkoff-zmq-public-keys',
'--from-file=server.key=./.certificates/public_keys/server.key',
'--from-file=client.key=./.certificates/public_keys/client.key'
], namespace)
existing_secrets = k8s_api.list_namespaced_secret(namespace)
redis_secret_name = None
redis_hostname = None
if click.confirm(
'Is there an existing Redis instance WALKOFF should use?'):
redis_hostname = click.prompt(
'Enter the Redis hostname (if it is not in the same Kubernetes namespace '
'as WALKOFF, enter a fully qualified domain name)')
if click.confirm(
"Is the Redis password already stored in a Kubernetes secret?"
):
redis_secret_name = click.prompt(
'Available secrets: {}\nEnter the name of the secret the Redis password '
'is stored in with a key of "redis-password" (leave blank for none): ',
default="")
if redis_secret_name not in existing_secrets:
redis_secret_name = None
click.echo(
'No secret with that name in this namespace. Creating a new secret to store password.'
)
if not redis_secret_name:
redis_secret_name = "walkoff-redis-secret"
new_pass = click.prompt('Enter a password for the Redis instance',
hide_input=True,
confirmation_prompt=True,
default='walkoff')
redis_secret_obj = k8s_client.V1Secret(
metadata={'name': redis_secret_name},
data={
'redis-password':
b64encode(new_pass.encode('utf-8')).decode('utf-8')
})
try:
k8s_api.create_namespaced_secret(namespace, redis_secret_obj)
except k8s_client.rest.ApiException as e:
click.echo("Error creating secret:\n{}".format(str(e)))
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
with open("k8s_manifests/setupfiles/redis-helm-values.yaml", 'r+') as f:
try:
y = yaml.load(f)
y['existingSecret'] = redis_secret_name
f.seek(0)
f.truncate()
yaml.dump(y, f, default_flow_style=False)
except yaml.YAMLError as e:
click.echo(
"Error reading k8s_manifests/setupfiles/redis-helm-values.yaml"
)
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
if not redis_hostname:
redis_hostname = 'walkoff-redis'
helm_command([
'install', 'stable/redis', '--name', redis_hostname, '--values',
'k8s_manifests/setupfiles/redis-helm-values.yaml', '--set',
'existingSecret={}'.format(redis_secret_name)
], tiller_namespace)
execution_secret_name = None
execution_db_hostname = None
if click.confirm(
'Do you have an existing PostgreSQL database to store WALKOFF execution data in?'
):
execution_db_hostname = click.prompt(
'Enter the database hostname (if it is not in the same Kubernetes '
'namespace as WALKOFF, enter a fully qualified domain name)')
execution_db_username = click.prompt(
'Enter a username that is able to create/read/write/update databases'
)
if click.confirm(
"Is the PostgreSQL password already stored in a Kubernetes secret?"
):
execution_secret_name = click.prompt(
'Available secrets: {}\nEnter the name of the secret the PostgreSQL '
'password is stored in with a key of "postgres-password" '
'(leave blank for none): ',
default="")
if execution_secret_name not in existing_secrets:
execution_secret_name = None
click.echo(
'No secret with that name in this namespace. Creating a new secret to store password.'
)
if not execution_secret_name:
execution_secret_name = "walkoff-postgres-execution-secret"
execution_db_username = click.prompt('Enter a username to create',
default='walkoff')
execution_db_password = click.prompt(
'Enter a password for the PostgreSQL instance',
hide_input=True,
confirmation_prompt=True,
default='walkoff')
execution_secret_obj = k8s_client.V1Secret(
metadata={'name': execution_secret_name},
data={
'postgres-password':
b64encode(
execution_db_password.encode('utf-8')).decode('utf-8')
})
try:
k8s_api.create_namespaced_secret(namespace, execution_secret_obj)
except k8s_client.rest.ApiException as e:
click.echo("Error creating secret:\n{}".format(str(e)))
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
with open("k8s_manifests/setupfiles/execution-postgres-helm-values.yaml",
'r+') as f:
try:
y = yaml.load(f)
y['postgresqlUsername'] = execution_db_username
y['postgresqlPassword'] = execution_db_password
f.seek(0)
f.truncate()
yaml.dump(y, f, default_flow_style=False)
except yaml.YAMLError as e:
click.echo(
"Error reading k8s_manifests/setupfiles/execution-postgres-helm-values.yaml"
)
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
if not execution_db_hostname:
helm_command([
'install', 'stable/postgresql', '--name', 'execution-db',
'--values',
'k8s_manifests/setupfiles/execution-postgres-helm-values.yaml'
], tiller_namespace)
execution_db_hostname = 'execution-db-postgresql'
walkoff_db_secret_name = None
walkoff_db_hostname = None
if click.confirm(
'Do you have an existing PostgreSQL database to store WALKOFF application data in? '
'(This can be the same or different as the previous)'):
walkoff_db_hostname = click.prompt(
'Enter the database hostname (if it is not in the same Kubernetes namespace '
'as WALKOFF, enter a fully qualified domain name)')
walkoff_db_username = click.prompt(
'Enter a username that is able to create/read/write/update databases'
)
if click.confirm(
"Is the PostgreSQL password already stored in a Kubernetes secret?"
):
walkoff_db_secret_name = click.prompt(
'Available secrets: {}\nEnter the name of the secret the PostgreSQL '
'password is stored in with a key of "postgres-password" '
'(leave blank for none): ',
default="")
if walkoff_db_secret_name not in existing_secrets:
walkoff_db_secret_name = None
click.echo(
'No secret with that name in this namespace. Creating a new secret to store password.'
)
if not walkoff_db_secret_name:
walkoff_db_secret_name = "walkoff-postgres-secret"
walkoff_db_username = click.prompt('Enter a username to create',
default='walkoff')
walkoff_db_password = click.prompt(
'Enter a password for the PostgreSQL instance',
hide_input=True,
confirmation_prompt=True,
default='walkoff')
walkoff_db_secret_obj = k8s_client.V1Secret(
metadata={'name': walkoff_db_secret_name},
data={
'postgres-password':
b64encode(walkoff_db_password.encode('utf-8')).decode('utf-8')
})
try:
k8s_api.create_namespaced_secret(namespace, walkoff_db_secret_obj)
except k8s_client.rest.ApiException as e:
click.echo("Error creating secret:\n{}".format(str(e)))
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
with open("k8s_manifests/setupfiles/walkoff-postgres-helm-values.yaml",
'r+') as f:
try:
y = yaml.load(f)
y['postgresqlUsername'] = walkoff_db_username
y['postgresqlPassword'] = walkoff_db_password
f.seek(0)
f.truncate()
yaml.dump(y, f, default_flow_style=False)
except yaml.YAMLError as e:
click.echo(
"Error reading k8s_manifests/setupfiles/walkoff-postgres-helm-values.yaml"
)
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
if not walkoff_db_hostname:
helm_command([
'install', 'stable/postgresql', '--name', 'walkoff-db', '--values',
'k8s_manifests/setupfiles/walkoff-postgres-helm-values.yaml'
], tiller_namespace)
walkoff_db_hostname = 'walkoff-db-postgresql'
walkoff_ca_key_pair = None
if click.confirm(
'Do you have an existing CA signing key pair stored in Kubernetes secrets?'
):
walkoff_ca_key_pair = click.prompt(
'Available secrets: {}\nEnter the name of the secret the key pair is stored in (leave blank for none): ',
default="")
if walkoff_ca_key_pair not in existing_secrets:
walkoff_ca_key_pair = None
click.echo(
'No secret with that name in this namespace. Creating a new secret to store keypair.'
)
if not walkoff_ca_key_pair:
crt = None
key = None
if click.confirm('Do you have existing CA signing key pair files?'):
while not crt:
crt = click.prompt('Enter the path to a cert (.crt) file: ')
try:
with open(crt, 'rb') as f:
crt = b64encode(f.read()).decode('ascii')
click.echo('Successfully loaded cert')
except IOError as e:
click.echo('Error reading {}: {}'.format(crt, e))
crt = None
while not key:
key = click.prompt(
'Enter the path to the matching private key (.key) file: ')
try:
with open(key, 'rb') as f:
key = b64encode(f.read()).decode('ascii')
click.echo('Successfully loaded key.')
except IOError as e:
click.echo('Error reading {}: {}'.format(key, e))
key = None
if not all((crt, key)):
private_key = rsa.generate_private_key(public_exponent=65537,
key_size=2048,
backend=default_backend())
public_key = private_key.public_key()
builder = x509.CertificateBuilder()
builder = builder.subject_name(
x509.Name(
[x509.NameAttribute(NameOID.COMMON_NAME, u'walkoff')]))
builder = builder.issuer_name(
x509.Name([
x509.NameAttribute(NameOID.COMMON_NAME, u'walkoff'),
]))
builder = builder.not_valid_before(datetime.datetime.today() -
datetime.timedelta(days=1))
builder = builder.not_valid_after(datetime.datetime.today() +
datetime.timedelta(days=3650))
builder = builder.serial_number(int(uuid.uuid4()))
builder = builder.public_key(public_key)
builder = builder.add_extension(
x509.SubjectKeyIdentifier.from_public_key(public_key),
critical=False)
builder = builder.add_extension(
x509.AuthorityKeyIdentifier.from_issuer_public_key(public_key),
critical=False)
builder = builder.add_extension(x509.BasicConstraints(
ca=True, path_length=None),
critical=True)
certificate = builder.sign(private_key=private_key,
algorithm=hashes.SHA256(),
backend=default_backend())
with open("ca.key", "wb") as f:
byte_cert = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption())
key = b64encode(byte_cert).decode('ascii')
f.write(byte_cert)
with open("ca.crt", "wb") as f:
byte_key = certificate.public_bytes(
encoding=serialization.Encoding.PEM, )
crt = b64encode(byte_key).decode('ascii')
f.write(byte_key)
tls_secret = k8s_client.V1Secret(
metadata={'name': 'walkoff-ca-key-pair'},
data={
'tls.crt': crt,
'tls.key': key
},
type='kubernetes.io/tls')
try:
k8s_api.create_namespaced_secret('default', tls_secret)
except k8s_client.rest.ApiException as e:
click.echo("Error creating secret:\n{}".format(str(e)))
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
walkoff_ca_key_pair = 'walkoff-ca-key-pair'
helm_command(
['install', 'stable/cert-manager', '--name', 'walkoff-cert-manager'],
tiller_namespace)
with open("k8s_manifests/setupfiles/cert-issuer.yaml", 'r+') as f:
try:
y = yaml.load(f)
y['spec']['ca']['secretName'] = walkoff_ca_key_pair
f.seek(0)
f.truncate()
yaml.dump(y, f, default_flow_style=False)
except yaml.YAMLError as e:
click.echo(
"Error reading k8s_manifests/setupfiles/cert-issuer.yaml")
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
kubectl_command(
['apply', '-f', 'k8s_manifests/setupfiles/cert-issuer.yaml'],
namespace)
kubectl_command(['apply', '-f', 'k8s_manifests/setupfiles/cert.yaml'],
namespace)
with open("k8s_manifests/setupfiles/walkoff-values.yaml", 'r+') as f:
try:
y = yaml.load(f)
y['namespace'] = namespace
y['resources']['redis']['service_name'] = redis_hostname
y['resources']['redis']['secret_name'] = redis_secret_name
y['resources']['execution_db'][
'service_name'] = execution_db_hostname
y['resources']['execution_db'][
'secret_name'] = execution_secret_name
y['resources']['execution_db']['username'] = execution_db_username
y['resources']['walkoff_db']['service_name'] = walkoff_db_hostname
y['resources']['walkoff_db'][
'secret_name'] = walkoff_db_secret_name
y['resources']['walkoff_db']['username'] = walkoff_db_username
f.seek(0)
f.truncate()
yaml.dump(y, f, default_flow_style=False)
except yaml.YAMLError as e:
click.echo(
"Error reading k8s_manifests/setupfiles/walkoff-values.yaml")
click.echo(
'You should use the uninstall command to rollback changes made by this installer.'
)
return
helm_command([
'install', 'k8s_manifests/helm_charts/walkoff', '--name',
'walkoff-deployment'
], tiller_namespace)
def get_all_namespaces(kubeconfig) -> list[str]:
config.load_kube_config(kubeconfig)
v1 = client.CoreV1Api()
response = v1.list_namespace()
all_namespaces = [item.metadata.name for item in response.items]
return all_namespaces
def uninstall(ctx):
"""
Removes resources and deployments created by this installer from a Kubernetes cluster.
"""
if click.confirm(
("Are you sure you wish to uninstall WALKOFF from your Kubernetes cluster? "
"(This will only uninstall components that WALKOFF created through this installer.)"
)):
k8s_api = None
try:
config_dir = os.environ.get(
'KUBECONFIG',
os.path.join(os.path.expanduser("~"), ".kube", "config"))
config_dir = click.prompt("Enter location of kubernetes config",
default=config_dir)
contexts, current = config.list_kube_config_contexts(
config_file=config_dir)
contexts = [context["name"] for context in contexts]
current = current["name"]
context = click.prompt(
"Available contexts: {}\nEnter context to uninstall WALKOFF from: "
.format(contexts),
default=current)
config.load_kube_config(config_file=config_dir, context=context)
k8s_api = k8s_client.CoreV1Api()
except IOError as e:
print("Could not open config: {}".format(e))
namespaces = k8s_api.list_namespace()
namespaces = [ns.metadata.name for ns in namespaces.items]
namespace = click.prompt(
"Available namespaces: {}\nEnter namespace to uninstall WALKOFF from"
.format(namespaces),
default="default")
tiller_namespace = click.prompt(
'Enter the namespace your Tiller service resides in',
default='kube-system')
kubectl_command(['delete', 'cert', 'walkoff-cert'],
namespace,
exit_on_err=False)
kubectl_command(['delete', 'issuer', 'walkoff-ca-issuer'],
namespace,
exit_on_err=False)
kubectl_command([
'delete', 'secrets', 'walkoff-redis-secret',
'walkoff-zmq-private-keys', 'walkoff-zmq-public-keys',
'walkoff-postgres-execution-secret', 'walkoff-postgres-secret',
'walkoff-ca-key-pair'
],
namespace,
exit_on_err=False)
kubectl_command([
'delete', 'pvc', 'data-execution-db-postgresql-0',
'redis-data-walkoff-redis-master-0', 'data-walkoff-db-postgresql-0'
],
namespace,
exit_on_err=False)
kubectl_command([
'delete', 'crd', 'certificates.certmanager.k8s.io',
'clusterissuers.certmanager.k8s.io', 'issuers.certmanager.k8s.io'
],
namespace,
exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-redis'],
tiller_namespace,
exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-db'],
tiller_namespace,
exit_on_err=False)
helm_command(['del', '--purge', 'execution-db'],
tiller_namespace,
exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-cert-manager'],
tiller_namespace,
exit_on_err=False)
helm_command(['del', '--purge', 'walkoff-deployment'],
tiller_namespace,
exit_on_err=False)
def __init__(self, config_file, context, namespace):
""" Constructor """
config.load_kube_config(config_file, context)
self.namespace = namespace
def load_config():
if frappe.get_conf().get("developer_mode"):
config.load_kube_config()
else:
config.load_incluster_config()
def handler(event, context):
cplJobId = event['CodePipeline.job']['id']
var = event['CodePipeline.job']['data']['actionConfiguration'][
'configuration']['UserParameters']
var = json.loads(var)
deployment_name = var["deployment_name"]
service_name = var["service_name"]
namespace = var["namespace"]
deployment_file = var["deployment_file"]
service_file = var = ["service_file"]
s3.download_file(bucket_name, deployment_file, '/tmp/deployment.yml')
s3.download_file(bucket_name, service_file, '/tmp/service.yml')
# Get Token
eks = EKSAuth(CLUSTER_NAME)
token = eks.get_token()
# Configure
config.load_kube_config()
configuration = client.Configuration()
configuration.api_key['authorization'] = token
configuration.api_key_prefix['authorization'] = 'Bearer'
# API
api = client.ApiClient(configuration)
v1 = client.CoreV1Api(api)
# Get all the pods
try:
with open(path.join(path.dirname(__file__),
"/tmp/deployment.yml")) as f:
dep = yaml.load(f)
k8s_beta = client.ExtensionsV1beta1Api()
try:
resp = k8s_beta.patch_namespaced_deployment(
name=deployment_name, body=dep, namespace=namespace)
print("Deployment created. status='%s'" % str(resp.status))
except:
resp = k8s_beta.create_namespaced_deployment(
body=dep, namespace=namespace)
print("Deployment created. status='%s'" % str(resp.status))
with open(path.join(path.dirname(__file__), "/tmp/service.yml")) as f:
dep = yaml.load(f)
try:
resp = v1.patch_namespaced_service(name=service_name,
body=dep,
namespace=namespace)
print("Service created. status='%s'" % str(resp.status))
except:
resp = v1.create_namespaced_service(name=service_name,
body=dep,
namespace=namespace)
print("Service created. status='%s'" % str(resp.status))
code_pipeline.put_job_success_result(jobId=cplJobId)
return 'Success'
except Exception as e:
code_pipeline.put_job_failure_result(jobId=cplJobId,
failureDetails={
'message': 'Job Failed',
'type': 'JobFailed'
})
print(e)
raise e
def deploy_manifest(yaml_file):
"Deploy the injector manifest"
k8s_config.load_kube_config()
k8s = k8s_client.ApiClient()
resp = k8s_utils.create_from_yaml(k8s, yaml_file)
log.info(resp)
def podStatus(name, namespace):
config.load_kube_config()
k8s_api = client.CoreV1Api()
V1pod = k8s_api.read_namespaced_pod_status(name=name, namespace=namespace)
return V1pod
def restart_server(helm_release_name):
res = True
timeout = 120
from kubernetes import client, config
client.rest.logger.setLevel(logging.WARNING)
# service_name = "%s.%s.svc.cluster.local" % (helm_release_name, namespace)
config.load_kube_config()
v1 = client.CoreV1Api()
pod_name = None
# config_map_names = v1.list_namespaced_config_map(namespace, pretty='true')
# body = {"replicas": 0}
pods = v1.list_namespaced_pod(namespace)
for i in pods.items:
if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1:
pod_name = i.metadata.name
break
# v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true')
# status_res = v1.read_namespaced_service_status(helm_release_name, namespace, pretty='true')
logging.getLogger().debug("Pod name: %s" % pod_name)
if pod_name is not None:
try:
v1.delete_namespaced_pod(pod_name, namespace)
except Exception as e:
logging.error(str(e))
logging.error("Exception when calling CoreV1Api->delete_namespaced_pod")
res = False
return res
logging.error("Sleep 10s after pod deleted")
time.sleep(10)
# check if restart successfully
pods = v1.list_namespaced_pod(namespace)
for i in pods.items:
pod_name_tmp = i.metadata.name
logging.error(pod_name_tmp)
if pod_name_tmp == pod_name:
continue
elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1:
continue
else:
status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true')
logging.error(status_res.status.phase)
start_time = time.time()
ready_break = False
while time.time() - start_time <= timeout:
logging.error(time.time())
status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true')
if status_res.status.phase == "Running":
logging.error("Already running")
ready_break = True
time.sleep(10)
break
else:
time.sleep(1)
if time.time() - start_time > timeout:
logging.error("Restart pod: %s timeout" % pod_name_tmp)
res = False
return res
if ready_break:
break
else:
raise Exception("Pod: %s not found" % pod_name)
follow = True
pretty = True
previous = True # bool | Return previous terminated container logs. Defaults to false. (optional)
since_seconds = 56 # int | A relative time in seconds before the current time from which to show logs. If this value precedes the time a pod was started, only logs since the pod start will be returned. If this value is in the future, no logs will be returned. Only one of sinceSeconds or sinceTime may be specified. (optional)
timestamps = True # bool | If true, add an RFC3339 or RFC3339Nano timestamp at the beginning of every line of log output. Defaults to false. (optional)
container = "milvus"
# start_time = time.time()
# while time.time() - start_time <= timeout:
# try:
# api_response = v1.read_namespaced_pod_log(pod_name_tmp, namespace, container=container, follow=follow,
# pretty=pretty, previous=previous, since_seconds=since_seconds,
# timestamps=timestamps)
# logging.error(api_response)
# return res
# except Exception as e:
# logging.error("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e)
# # waiting for server start
# time.sleep(5)
# # res = False
# # return res
# if time.time() - start_time > timeout:
# logging.error("Restart pod: %s timeout" % pod_name_tmp)
# res = False
return res
def launch_app(k8s_app_name: NAUTAAppNames = None,
no_launch: bool = False,
port: int = None,
app_name: str = None,
number_of_retries: int = 0,
url_end: str = "",
namespace: str = None):
try:
with spinner(text=Texts.LAUNCHING_APP_MSG) as proxy_spinner, \
K8sProxy(nauta_app_name=k8s_app_name, port=port, app_name=app_name,
number_of_retries=number_of_retries, namespace=namespace) as proxy:
url = FORWARDED_URL.format(proxy.tunnel_port, url_end)
# run socat if on Windows or Mac OS
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyBroadException
try:
socat.start(proxy.container_port)
except Exception:
err_message = Texts.LOCAL_DOCKER_TUNNEL_ERROR_MSG
logger.exception(err_message)
raise LaunchError(err_message)
if k8s_app_name == NAUTAAppNames.INGRESS:
config.load_kube_config()
user_token = configuration.Configuration().api_key.get(
'authorization')
prepared_user_token = user_token.replace('Bearer ', '')
url = f'{url}?token={prepared_user_token}'
if not no_launch:
if is_gui_browser_available():
wait_for_connection(url)
webbrowser.open_new(url)
proxy_spinner.hide()
else:
click.echo(Texts.NO_WEB_BROWSER_ERROR_MSG)
if port and port != proxy.tunnel_port:
click.echo(
Texts.CANNOT_USE_PORT.format(
required_port=port, random_port=proxy.tunnel_port))
proxy_spinner.hide()
click.echo(Texts.GO_TO_MSG.format(url=url))
click.echo(Texts.PROXY_CREATED_MSG)
wait_for_ctrl_c()
except K8sProxyCloseError:
err_message = Texts.PROXY_CLOSE_ERROR_MSG.format(app_name=k8s_app_name)
raise ProxyClosingError(err_message)
except LocalPortOccupiedError as exe:
err_message = Texts.PROXY_CREATED_EXTENDED_ERROR_MSG.format(
app_name=k8s_app_name, reason=exe.message)
raise LaunchError(err_message)
except K8sProxyOpenError:
error_msg = Texts.PROXY_CREATED_ERROR_MSG.format(app_name=k8s_app_name)
logger.exception(error_msg)
raise LaunchError(error_msg)
except LaunchError as e:
raise e
except Exception:
err_message = Texts.WEB_APP_LAUCH_FAIL_MSG
logger.exception(err_message)
raise LaunchError(err_message)
finally:
# noinspection PyBroadException
# terminate socat if on Windows or Mac OS
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyBroadException
try:
with spinner(text=Texts.WEB_APP_CLOSING_MSG):
socat.stop()
except Exception:
err_message = Texts.PROXY_CLOSE_ERROR_MSG.format(k8s_app_name)
raise ProxyClosingError(err_message)
def main():
global cpx_count
global output_yaml
parser = argparse.ArgumentParser(
description=
'Taking service yamls or name as input. And namespace in case service names is provided'
)
parser.add_argument(
'ServiceList',
action="store",
help=
'Please provide comma seperated list of service yamls or names to convert into SML'
)
parser.add_argument(
'Namespace',
action="store",
nargs='?',
help='Please provide namespace in which services are deployed')
svcs = parser.parse_args().ServiceList.split(',')
namespace = parser.parse_args().Namespace
converted_yamls_dict_list = []
if (".yaml" not in svcs[0]):
if not namespace:
logger.error(
"Please provide namespace as well in which services are deployed while starting the script"
)
sys.exit(0)
prompts = chain(
["Do you want to connect to a Remote Kubernetes Cluster? (Y/N): "],
repeat("Invaild input. Please respond with (Y/N): "))
remote_access = validate_input(prompts, "yesORno")
if remote_access.lower() == 'n':
prompts = chain([
'Do you want to use default kubeconfig file present at "/root/.kube/config"? (Y/N): '
], repeat("Invaild input. Please respond with (Y/N): "))
default_kube_config = validate_input(prompts, "yesORno")
if default_kube_config.lower() == "y":
kube_config_path = "/root/.kube/config"
else:
prompts = chain(
["Please provide path of kubeconfig file: "],
repeat(
"This path doesn't exist! Please provide a valid kubeconfig file path: "
))
kube_config_path = validate_input(prompts, "path")
config.load_kube_config(config_file=kube_config_path)
v1 = client.CoreV1Api()
elif remote_access.lower() == 'y':
prompts = chain([
"Do you want to use Bearer Token for connecting to a Remote Kubernetes Cluster? (Y/N): "
], repeat("Invaild input. Please respond with (Y/N): "))
use_bearer_token = validate_input(prompts, "yesORno")
if use_bearer_token.lower() == "y":
configuration = client.Configuration()
bearer_token = input(
"Please provide bearer token key of SA having permission to access given service: "
)
configuration.api_key = {
"authorization": "Bearer " + bearer_token
}
prompts = chain(
["Please provide API server <IP:PORT>: "],
repeat(
"Invaild input. Please provide valid IP and port in format <IP:PORT>: "
))
api_server = validate_input(prompts, "apiServerIPPort")
configuration.host = 'https://' + api_server
configuration.verify_ssl = False
v1 = client.CoreV1Api(client.ApiClient(configuration))
elif use_bearer_token.lower() == "n":
prompts = chain(
["Please provide path of kubeconfig file: "],
repeat(
"This path doesn't exist! Please provide a valid kubeconfig file path: "
))
kube_config_path = validate_input(prompts, "path")
config.load_kube_config(config_file=kube_config_path)
v1 = client.CoreV1Api()
app_frontend_svc_name = input(
"Please provide name of the service exposed to tier-1: ")
app_hostname = input("Please provide hostname for exposing \"" +
app_frontend_svc_name + "\" service: ")
for svc in svcs:
if (".yaml" in svc):
with open(svc, 'r') as stream:
try:
yaml_dictionaries = yaml.safe_load_all(stream)
for yaml_instance in yaml_dictionaries:
if yaml_instance:
converted_yamls_dict_list.extend(
convert_yaml_to_sml(yaml_instance,
app_frontend_svc_name,
app_hostname))
except Exception as e:
logger.error(
"Retrieving yaml from service yamls provided failed with error: {}"
.format(e))
sys.exit("Please ensure service yaml is in proper format")
else:
try:
service = v1.read_namespaced_service(svc, namespace)
except Exception as e:
logger.error(
"Retrieving service from KubeAPI server failed with error: {}"
.format(e))
sys.exit("Please ensure service name and namespace is correct")
service = service.__dict__
service = convert_kube_object_to_dict(service)
# service = OrderedDict([('apiVersion',service['apiVersion']),
# ('kind',service['kind']),
# ('metadata',service['metadata']),
# ('spec',service['spec']),
converted_yamls_dict_list.extend(
convert_yaml_to_sml(service, app_frontend_svc_name,
app_hostname))
'''
Adding CPX yamls
'''
rbac = manifestCreator.rbac()
rbac = rbac.createRbac()
converted_yamls_dict_list.extend(rbac)
for i in range(1, cpx_count + 1):
cpx_cic_yaml = manifestCreator.cpxCic({
"name":
cpx_ingress_name + str(i),
"ingressClass":
ingress_class + str(i)
})
cpx_cic_yaml = cpx_cic_yaml.create()
converted_yamls_dict_list.extend(cpx_cic_yaml)
'''
Writing dictionay of YAML into a .yaml file
'''
write_dictionaries_into_yaml(converted_yamls_dict_list, output_yaml)
def initialize_clients(kubeconfig_path, chunk_size):
global cli
global request_chunk_size
config.load_kube_config(kubeconfig_path)
cli = client.CoreV1Api()
request_chunk_size = str(chunk_size)
def run_test_within_pod(test_path: str, with_pv_at: Optional[str] = None):
"""
run selected test from within an openshift pod
:param test_path: relative path to the test
:param with_pv_at: path to PV within the pod
"""
config.load_kube_config()
configuration = client.Configuration()
assert configuration.api_key
api = client.CoreV1Api(client.ApiClient(configuration))
container = {
"image":
TEST_IMAGE_NAME,
"name":
POD_NAME,
"tty":
True, # corols
"command": [
"bash",
"-c",
"ls -lha "
"&& id "
"&& pytest-3 --collect-only"
f"&& pytest-3 -vv -l -p no:cacheprovider {test_path}",
],
"imagePullPolicy":
"Never",
}
spec = {"containers": [container], "restartPolicy": "Never"}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": POD_NAME
},
"spec": spec,
}
if with_pv_at:
cleaned_test_name = clean_string(test_path)
ts = get_timestamp_now()
volume_name = f"{cleaned_test_name}-{ts}-vol"[-63:]
claim_name = f"{cleaned_test_name}-{ts}-pvc"[-63:]
container["env"] = [{"name": "SANDCASTLE_PVC", "value": claim_name}]
pvc_dict = {
"kind": "PersistentVolumeClaim",
"spec": {
"accessModes": ["ReadWriteMany"],
"resources": {
"requests": {
"storage": "1Gi"
}
},
},
"apiVersion": "v1",
"metadata": {
"name": claim_name
},
}
api.create_namespaced_persistent_volume_claim(NAMESPACE, pvc_dict)
container["volumeMounts"] = [{
"mountPath": with_pv_at,
"name": volume_name
}]
spec["volumes"] = [{
"name": volume_name,
"persistentVolumeClaim": {
"claimName": claim_name
}
}]
try:
api.delete_namespaced_pod(POD_NAME, NAMESPACE, body=V1DeleteOptions())
except ApiException as ex:
if ex.status != 404:
raise
try:
api.create_namespaced_pod(body=pod_manifest, namespace=NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not start on time.")
info = api.read_namespaced_pod(POD_NAME, NAMESPACE)
if info.status.phase == "Running":
break
time.sleep(2.0)
counter -= 1
print(
api.read_namespaced_pod_log(name=POD_NAME,
namespace=NAMESPACE,
follow=True))
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not finish on time.")
info = api.read_namespaced_pod(POD_NAME, NAMESPACE)
if info.status.phase == "Succeeded":
break
if info.status.phase == "Failed":
raise RuntimeError("Test failed")
time.sleep(2.0)
counter -= 1
finally:
print(
api.read_namespaced_pod_log(name=POD_NAME,
namespace=NAMESPACE,
follow=True))
api.delete_namespaced_pod(POD_NAME, NAMESPACE, body=V1DeleteOptions())
if with_pv_at:
api.delete_namespaced_persistent_volume_claim(
claim_name, NAMESPACE, V1DeleteOptions())
