def _write_to_object(obj, res_map):
if obj.metadata is None:
obj.metadata = swagger_client.V1ObjectMeta()
if obj.metadata.annotations is None:
obj.metadata.annotations = {}
final = {"version": AXResources.current_version, "resources": res_map}
obj.metadata.annotations["ax_resources"] = json.dumps(final)
def generate_metadata(name=None, labels=None, annotations=None):
meta = swagger_client.V1ObjectMeta()
if name is not None:
meta.name = name
if labels is not None:
meta.labels = labels
if annotations is not None:
meta.annotations = annotations
return meta
def _scale_to(self, replicas):
logger.debug("Scaling deployment to {} for {} {}".format(replicas, self.application, self.name))
scale = swagger_client.V1beta1Scale()
scale.spec = swagger_client.V1beta1ScaleSpec()
scale.spec.replicas = replicas
scale.metadata = swagger_client.V1ObjectMeta()
scale.metadata.name = self.name
scale.metadata.namespace = self.application
self.client.apisappsv1beta1_api.replace_namespaced_scale_scale(scale, self.application, self.name)
def copy_imgpull(self, secret, to_ns):
"""
Copy secret to a new namespace
Args:
secret: V1Secret
to_ns: name of namespace
"""
new_secret = swagger_client.V1Secret()
new_secret.metadata = swagger_client.V1ObjectMeta()
new_secret.metadata.name = secret.metadata.name
new_secret.data = secret.data
new_secret.type = "kubernetes.io/dockerconfigjson"
self._create_in_provider(to_ns, new_secret)
def get_spec(self):
# generate the metadata
metadata = swagger_client.V1ObjectMeta()
metadata.name = self.name
metadata.annotations = {
"pod.beta.kubernetes.io/init-containers":
self._init_containers_spec()
}
for a in self.annotations:
metadata.annotations[a] = self.annotations[a]
metadata.labels = {}
for l in self.labels:
metadata.labels[l] = self.labels[l]
# generate the pod specification
pspec = swagger_client.V1PodSpec()
if self.hostname:
pspec.hostname = self.hostname
pspec.containers = []
if "wait" in self.cmap:
pspec.containers.append(self.cmap["wait"].generate_spec())
assert "main" in self.cmap, "Pod specification cannot be generated without a main container"
pspec.containers.append(self.cmap["main"].generate_spec())
if "dind" in self.cmap:
pspec.containers.append(self.cmap["dind"].generate_spec())
pspec.image_pull_secrets = self._build_image_pull_secrets()
pspec.volumes = self._volume_spec()
if self.restart_policy is not None:
pspec.restart_policy = self.restart_policy
cluster_name_id = os.getenv("AX_CLUSTER_NAME_ID", None)
assert cluster_name_id, "Cluster name id is None!"
cluster_config = AXClusterConfig(cluster_name_id=cluster_name_id)
if not cluster_config.get_cluster_provider().is_user_cluster():
pspec.node_selector = {"ax.tier": self._tier}
# finalize the pod template spec
spec = swagger_client.V1PodTemplateSpec()
spec.metadata = metadata
spec.spec = pspec
return spec
def _generate_spec(self, port_spec, selector, owner):
# generate port spec
k8s_ports = []
for port in port_spec or []:
p = swagger_client.V1ServicePort()
p.port = int(port["port"])
p.name = "port-{}".format(p.port)
p.target_port = int(port["target_port"])
if "node_port" in port:
p.node_port = port["node_port"]
k8s_ports.append(p)
# service spec
srv_spec = swagger_client.V1ServiceSpec()
srv_spec.ports = k8s_ports
srv_spec.selector = selector
srv_spec.type = "ClusterIP"
# generate meta
meta = swagger_client.V1ObjectMeta()
meta.name = self.name
meta.labels = {
"srv": self.name,
"tier": "deployment",
"role": "user"
}
meta.annotations = {}
self.ax_meta = {
"name": self.name,
"application": self.namespace,
"owner": owner,
"port_spec": port_spec,
"selector": selector
}
# finally the service
srv = swagger_client.V1Service()
srv.metadata = meta
srv.spec = srv_spec
AXResource.set_ax_meta(srv, self.ax_meta)
return srv
def create(self, service, ports, host_mapping=None, whitelist_cidrs=[]):
ing = self._get_from_provider()
if ing is None:
self._delete_in_provider()
ing = swagger_client.V1beta1Ingress()
ing.metadata = swagger_client.V1ObjectMeta()
ing.metadata.name = self.name
ing.metadata.annotations = {
"kubernetes.io/ingress.class":
self.ingress_class,
"ingress.kubernetes.io/whitelist-source-range":
",".join(whitelist_cidrs)
}
ing.spec = swagger_client.V1beta1IngressSpec()
ing.spec.rules = []
r = swagger_client.V1beta1IngressRule()
if host_mapping:
r.host = host_mapping
r.http = swagger_client.V1beta1HTTPIngressRuleValue()
r.http.paths = []
# only one rule with multiple paths for now
ing.spec.rules.append(r)
# build the metadata for the AXResource
self.ax_meta = {
"name": self.name,
"namespace": self.namespace,
"ingress_class": self.ingress_class,
"ingress_to": {
"service": service,
"ports": ports,
"whitelist": whitelist_cidrs
}
}
AXResource.set_ax_meta(ing, self.ax_meta)
self._rules_to_ingress(service, r.http.paths, ports)
with IngressRuleOperation(self):
self._create_in_provider(ing)
def _generate_deployment_spec_for_pod(self, pod_spec):
metadata = swagger_client.V1ObjectMeta()
metadata.name = self.name
dspec = swagger_client.V1beta1DeploymentSpec()
dspec.strategy = self._get_strategy()
if self.spec.template.min_ready_seconds:
dspec.min_ready_seconds = self.spec.template.min_ready_seconds
dspec.selector = swagger_client.V1LabelSelector()
dspec.selector.match_labels = {"deployment": self.name}
dspec.replicas = self.spec.template.scale.min
dspec.template = pod_spec
deployment_obj = swagger_client.V1beta1Deployment()
deployment_obj.metadata = metadata
deployment_obj.spec = dspec
return deployment_obj
def get_spec(self):
# generate the metadata
metadata = swagger_client.V1ObjectMeta()
metadata.name = self.name
metadata.annotations = {
"pod.beta.kubernetes.io/init-containers":
self._init_containers_spec()
}
for a in self.annotations:
metadata.annotations[a] = self.annotations[a]
metadata.labels = {}
for l in self.labels:
metadata.labels[l] = self.labels[l]
# generate the pod specification
pspec = swagger_client.V1PodSpec()
pspec.containers = []
if "wait" in self.cmap:
pspec.containers.append(self.cmap["wait"].generate_spec())
assert "main" in self.cmap, "Pod specification cannot be generated without a main container"
pspec.containers.append(self.cmap["main"].generate_spec())
if "dind" in self.cmap:
pspec.containers.append(self.cmap["dind"].generate_spec())
pspec.image_pull_secrets = self._build_image_pull_secrets()
pspec.volumes = self._volume_spec()
if self.restart_policy is not None:
pspec.restart_policy = self.restart_policy
pspec.node_selector = {"ax.tier": self._tier}
# finalize the pod template spec
spec = swagger_client.V1PodTemplateSpec()
spec.metadata = metadata
spec.spec = pspec
return spec
def insert_imgpull(self, name, namespace, token):
"""
apiVersion: v1
kind: Secret
metadata:
name: applatix-registry
data:
.dockerconfigjson: XXX
type: kubernetes.io/dockerconfigjson
"""
name = reformat_name(name)
secret = swagger_client.V1Secret()
secret.metadata = swagger_client.V1ObjectMeta()
secret.metadata.name = name
secret.data = {".dockerconfigjson": token}
secret.type = "kubernetes.io/dockerconfigjson"
# always delete a secret if it exists
self.delete_imgpull(name, namespace)
self._create_in_provider(namespace, secret)
def _create(self, appname, port_spec, layer7, type):
srv = swagger_client.V1Service()
srv.metadata = swagger_client.V1ObjectMeta()
srv.metadata.name = self.name
srv.metadata.labels = {
"app": appname,
"tier": "deployment",
"role": "user"
}
if layer7:
srv.metadata.annotations = {
"service.beta.kubernetes.io/aws-load-balancer-backend-protocol": "http"
}
spec = swagger_client.V1ServiceSpec()
spec.selector = {
'app': appname
}
spec.type = type
spec.ports = []
for p in port_spec:
port = swagger_client.V1ServicePort()
port.name = p["name"]
port.port = p["port"]
port.target_port = p["containerPort"]
spec.ports.append(port)
srv.spec = spec
@parse_kubernetes_exception
@retry(retry_on_exception=raise_apiexception_else_retry,
wait_exponential_multiplier=100,
stop_max_attempt_number=10)
def create_in_provider(service):
client = self.get_k8s_client()
client.api.create_namespaced_service(service, self.namespace)
create_in_provider(srv)
def create_generic(self, name, data, metadata=None, namespace="axsys"):
"""
Create/update a generic kubernetes secret from configuration
:param name: name of secret
:param data: key, value pairs
:param metadata: key, value pairs of metadata to store with secret
metadata is not loaded with secret into the container
:param namespace: The namespace to create secret in
NOTE: update only if secret already exists in the namespace.
"""
logger.debug("Create secret {}/{}".format(name, namespace))
obj = swagger_client.V1Secret()
obj.metadata = swagger_client.V1ObjectMeta()
obj.metadata.name = name
obj.metadata.annotations = {"user_metadata": json.dumps(metadata)}
obj.type = "Opaque"
enc_data = {}
for k, v in iteritems(data):
enc_data[k] = base64.b64encode(v)
obj.data = enc_data
self._create_in_provider(namespace, obj)
logger.debug("Create secret {}/{} complete".format(name, namespace))
def create(self, conf):
"""
Create a Kubernetes Job object
:param conf: conf data from DevOps
:return:
"""
logger.debug("Task create for {}".format(json.dumps(conf)))
self.service = Service()
self.service.parse(conf)
self.jobname = Task.generate_job_name(conf)
job = swagger_client.V1Job()
labels = {
"app":
self.jobname,
"service_instance_id":
self.service.service_context.service_instance_id,
"root_workflow_id":
self.service.service_context.root_workflow_id,
"leaf_full_path":
string_to_k8s_label(self.service.service_context.leaf_full_path
or "no_path"),
"tier":
"devops",
"role":
"user",
}
job_meta = swagger_client.V1ObjectMeta()
job_meta.name = self.jobname
job_meta.labels = labels
job.metadata = job_meta
job_spec = swagger_client.V1JobSpec()
job_spec.template = self._container_to_pod(labels)
job.spec = job_spec
return job
def create_ns_in_provider():
namespace = swagger_client.V1Namespace()
namespace.metadata = swagger_client.V1ObjectMeta()
namespace.metadata.name = self.name
self._client.api.create_namespace(namespace)
def create_webhook():
"""
Create a kubernetes service load balancer connecting external traffic to
axops, with a range of trusted ips
Data input format:
{
"port_spec": [
{
"name": "webhook",
"port": 8443,
"targetPort": 8087
}
],
"ip_ranges": ["0.0.0.0/0"]
}
:return:
{
"ingress": "xxxxxx.us-west-2.elb.amazonaws.com",
"detail": V1Service
}
"""
data = request.get_json()
port_spec = data.get("port_spec", None)
ip_ranges = data.get("ip_ranges", ["0.0.0.0/0"])
if not port_spec:
raise AXIllegalArgumentException("No port spec provided")
webhook_svc_name = "axops-webhook"
srv = swagger_client.V1Service()
srv.metadata = swagger_client.V1ObjectMeta()
srv.metadata.name = webhook_svc_name
srv.metadata.labels = {
"app": webhook_svc_name,
"tier": "platform",
"role": "axcritical"
}
spec = swagger_client.V1ServiceSpec()
spec.selector = {
'app': "axops-deployment"
}
spec.type = "LoadBalancer"
spec.ports = port_spec
spec.load_balancer_source_ranges = ip_ranges
srv.spec = spec
# Don't have to retry here as creating webhook is a manual process
# and it is fair to throw proper error message to user and have them
# retry manually
need_update = False
try:
kubectl.api.create_namespaced_service(body=srv, namespace="axsys")
except ApiException as ae:
if ae.status == 409:
need_update = True
elif ae.status == 422:
raise AXIllegalArgumentException("Unable to create webhook due to invalid argument", detail=str(ae))
else:
raise AXPlatformException("Unable to create webhook due to Kubernetes internal error", detail=str(ae))
except Exception as e:
raise AXPlatformException("Unable to create webhook", detail=str(e))
if need_update:
update_body = {
"spec": {
"ports": port_spec,
"load_balancer_source_ranges": ip_ranges
}
}
try:
kubectl.api.patch_namespaced_service(body=update_body, namespace="axsys", name=webhook_svc_name)
except Exception as e:
raise AXPlatformException("Unable to update webhook", detail=str(e))
trail = 0
rst = {
"port_spec": port_spec,
"ip_ranges": ip_ranges
}
while trail < 60:
time.sleep(3)
try:
svc = kubectl.api.read_namespaced_service_status(namespace="axsys", name=webhook_svc_name)
if svc.status.load_balancer and svc.status.load_balancer.ingress:
rst["hostname"] = svc.status.load_balancer.ingress[0].hostname
return jsonify(rst)
except ApiException:
pass
trail += 1
try:
kubectl.api.delete_namespaced_service(namespace="axsys", name=webhook_svc_name)
except ApiException as ae:
if ae.status != 404:
raise ae
raise AXPlatformException("Webhook creation timeout")
def create_ns_in_provider():
namespace = swagger_client.V1Namespace()
namespace.metadata = swagger_client.V1ObjectMeta()
namespace.metadata.name = self.name
namespace.metadata.labels = {"creator": "argo-application", "app": self.name}
self._client.api.create_namespace(namespace)
def create(self, node_selector=None):
from ax.kubernetes.swagger_client import ApiClient
converter = ApiClient()
client = KubernetesApiClient(use_proxy=True)
with open("/ax/config/service/nginx-ingress-template.yml.in") as f:
deployment_spec = yaml.load(f)
spec = converter._ApiClient__deserialize(deployment_spec,
'V1beta1Deployment')
spec.metadata.name = self.name
spec.metadata.labels = {
"app": self.name,
"tier": "platform",
"role": "axcritical"
}
spec.spec.selector.match_labels = spec.metadata.labels
spec.spec.template.metadata.labels = spec.metadata.labels
spec.spec.template.spec.containers[0].args.append(
"--ingress-class={}".format(self.name))
spec.spec.template.spec.containers[0].args.append(
"--nginx-configmap=$(POD_NAMESPACE)/{}".format(self.cmap_name))
spec.spec.template.spec.node_selector = node_selector
# define the config map
cmap = swagger_client.V1ConfigMap()
cmap.metadata = swagger_client.V1ObjectMeta()
cmap.metadata.name = self.cmap_name
cmap.data = {
"server-name-hash-bucket-size": "512",
"server-name-hash-max-size": "512"
}
@retry_unless()
def create_config_map():
try:
client.api.replace_namespaced_config_map(
cmap, self.namespace, self.cmap_name)
except swagger_client.rest.ApiException as ee:
if ee.status == 404:
client.api.create_namespaced_config_map(
cmap, self.namespace)
else:
raise ee
@retry_unless()
def create_in_provider():
logger.debug("Creating deployment in provider")
try:
client.extensionsvbeta.replace_namespaced_deployment(
spec, self.namespace, self.name)
except swagger_client.rest.ApiException as ee:
if ee.status == 404:
client.extensionsvbeta.create_namespaced_deployment(
spec, self.namespace)
else:
raise ee
s = ServiceEndpoint(self.name, self.namespace)
ports = [{
"name": "http",
"port": 80,
"containerPort": 80
}, {
"name": "https",
"port": 443,
"containerPort": 443
}]
try:
s.create(self.name, ports, layer7=True)
except Exception as e:
logger.debug(
"Got exception while trying to create ServiceEndpoint {}".
format(e))
raise e
create_config_map()
create_in_provider()
