def status(self, with_loadbalancer_info=False):
# Locking so that a create is not meddling with status
s = self._get_from_provider()
if s is None:
raise AXNotFoundException("Could not find Route {}.{}".format(
self.name, self.namespace))
ep = self._get_ep_from_provider()
if ep is None:
raise AXNotFoundException(
"Could not find Route Endpoint for {}.{}".format(
self.name, self.namespace))
field_map = {
"name": "metadata.name",
"application": "metadata.namespace",
"ip": "spec.cluster_ip"
}
if with_loadbalancer_info:
field_map["loadbalancer"] = "status.load_balancer.ingress.hostname"
ret = KubeObject.swagger_obj_extract(s, field_map)
field_map = {"ips": "subsets.addresses.ip"}
ret["endpoints"] = KubeObject.swagger_obj_extract(ep, field_map)
return ret
def _create_deployment_resources(self):
res = AXResources()
for route in self.spec.template.internal_routes:
# ignore empty port spec
if len(route.ports) == 0:
logger.debug(
"Skipping internal route {} as port spec is empty".format(
route.name))
continue
ir = InternalRoute(route.name, self.application)
ir.create(route.to_dict()["ports"],
selector={"deployment": self.name},
owner=self.name)
res.insert(ir)
logger.debug("Created route {}".format(ir))
for route in self.spec.template.external_routes:
dns_name = route.dns_name()
if dns_name.endswith("."):
dns_name = dns_name[:-1]
r = ExternalRoute(dns_name, self.application,
{"deployment": self.name}, route.target_port,
route.ip_white_list, route.visibility)
try:
elb_addr = visibility_to_elb_addr(route.visibility)
elb_name = visibility_to_elb_name(route.visibility)
except AXNotFoundException:
if route.visibility == ExternalRouteVisibility.VISIBILITY_WORLD:
raise AXNotFoundException(
"Could not find the public ELB. Please report this error to Applatix Support at [email protected]"
)
else:
assert route.visibility == ExternalRouteVisibility.VISIBILITY_ORGANIZATION, "Only world and organization are currently supported as visibility attributes"
raise AXNotFoundException(
"Please create a private ELB using the template named 'ax_private_elb_creator_workflow' before using 'visibility=organization'"
)
name = r.create(elb_addr, elb_name=elb_name)
res.insert(r)
logger.debug("Created external route {} for {}/{}/{}".format(
name, self.application, self.name, dns_name))
main_container = self.spec.template.get_main_container()
for key_name, vol in iteritems(main_container.inputs.volumes):
assert "resource_id" in vol.details, "Volume resource_id absent in volume details"
name = vol.details.get("axrn", None)
resource_id = vol.details.get("resource_id", None)
assert name is not None and resource_id is not None, "axrn and resource_id are required details for volume {}".format(
key_name)
nv_res = AXNamedVolumeResource(name, resource_id)
nv_res.create()
res.insert(nv_res)
logger.debug(
"Using named volume resource {} in application {}".format(
name, self.application))
return res
def login(self, registry, username, password):
"""
Returns a base64 encoded token of username and password
only if login is successful else it raises exceptions
"""
try:
self._conn.login(username,
password=password,
registry=registry,
reauth=True)
except APIError as e:
code = e.response.status_code
if code == 401:
# on login failure it raises a docker.errors.APIError:
# 401 Client Error: Unauthorized
raise AXUnauthorizedException(e.explanation)
elif code == 404:
raise AXNotFoundException(e.explanation)
elif code == 500:
if "x509: certificate signed by unknown authority" in e.response.text:
raise AXIllegalArgumentException(
"Certificate signed by unknown authority for {}".
format(registry))
else:
raise e
else:
raise e
token = base64.b64encode("{}:{}".format(username, password))
return token
def get_record(self, name, type):
for record in self.list_records():
if record['Name'] == name and record['Type'] == type:
return record
raise AXNotFoundException(
"Could not find record {} of type {} in hosted zone {}".format(
name, type, self.name))
def describe_webhook():
@retry(wait_exponential_multiplier=1000,
stop_max_attempt_number=3)
def _get_webhook_from_kube_with_retry():
try:
return kubectl.api.read_namespaced_service_status(namespace="axsys", name="axops-webhook")
except ApiException as ae:
if ae.status == 404:
return None
else:
raise
webhook = _get_webhook_from_kube_with_retry()
if not webhook or not webhook.spec:
raise AXNotFoundException("No webhook found")
rst = {
"port_spec": [],
"ip_ranges": webhook.spec.load_balancer_source_ranges,
"hostname": webhook.status.load_balancer.ingress[0].hostname
}
for p in webhook.spec.ports:
rst["port_spec"].append({
"name": p.name,
"port": p.port,
"targetPort": int(p.target_port)
})
return jsonify(rst)
def status(self):
"""
Get the status of the deployment.
Returns: Returns the entire V1Deployment as a dict.
If deployment is not found then this will raise an AXNotFoundException (404)
"""
# STEP 1: Get status of deployment
stat = self._deployment_status()
if stat is None:
raise AXNotFoundException("Deployment {} not found in application {}".format(self.name, self.application))
dep_field_map = {
"name": "metadata.name",
"generation": "metadata.annotations.ax_generation",
"desired_replicas": "status.replicas",
"available_replicas": "status.available_replicas",
"unavailable_replicas": "status.unavailable_replicas"
}
ret = KubeObject.swagger_obj_extract(stat, dep_field_map, serializable=True)
# STEP 2: Get the pods for the deployment and events associated
podlist = self._deployment_pods().items
dep_events = self._app_obj.events(name=self.name)
event_field_map = {
"message": "message",
"reason": "reason",
"source": "source.component",
"host": "source.host",
"firstTS": "first_timestamp",
"lastTS": "last_timestamp",
"count": "count",
"container": "involved_object.field_path",
"type": "type"
}
ret["events"] = []
for event in dep_events:
ret["events"].append(KubeObject.swagger_obj_extract(event, event_field_map, serializable=True))
ret["pods"] = []
for pod in podlist or []:
# fill pod status and containers
pstatus = Pod.massage_pod_status(pod)
# fill events for pod
pstatus["events"] = []
events = self._app_obj.events(name=pod.metadata.name)
for event in events:
pstatus["events"].append(KubeObject.swagger_obj_extract(event, event_field_map, serializable=True))
# fill pod failure information for pod based on events
pstatus["failure"] = Deployment._pod_failed_info(pstatus)
ret["pods"].append(pstatus)
# STEP 3: From the deployment spec get the resources created by deployment
# TODO: Add this when services are created by deployment
return ret
def swagger_exception_handler(*args, **kwargs):
try:
return func(*args, **kwargs)
except ApiException as e:
if e.status == 404:
raise AXNotFoundException(e.reason, detail=e.body)
elif e.status == 409:
raise AXConflictException(e.reason, detail=e.body)
raise AXKubeApiException(e.reason, detail=e.body)
def delete_registry(self, server):
s = SecretsManager()
dns_name = urlparse("https://{}".format(server)).netloc
if s.get_imgpull(dns_name, "axuser"):
s.delete_imgpull(dns_name, "axuser")
else:
raise AXNotFoundException(
"Registry server {} not registered with platform".format(
dns_name))
def get_labels(self):
"""
Get a dict of labels used for this deployment
"""
state = self._deployment_status()
if state is None:
raise AXNotFoundException("Did not find deployment {} in application {}".format(self.name, self.application))
return KubeObject.swagger_obj_extract(state, {"labels": "spec.selector.match_labels"})['labels']
def get_hosted_zone_id(self):
if not self._id:
zone = self.client.get_hosted_zone(self.name)
if not zone:
raise AXNotFoundException("Hosted zone {} not found".format(
self.name))
self._id = zone.get_hosted_zone_id()
return self._id
def get_elb_addr(self, elb_name):
logger.debug("Getting address {}".format(elb_name))
with self._lock:
params = {
"elb_name": elb_name
}
elb_entries = self._db.retry_request('get', self._table, params=params, max_retry=5, retry_on_exception=self._db.get_retry_on_exception, value_only=True)
for elb in elb_entries or []:
return elb["elb_addr"]
raise AXNotFoundException("Did not find an elb address for elb {}".format(elb_name))
def add_elb_addr(self, elb_name, elb_addr):
logger.debug("Adding address {} to elb {}".format(elb_addr, elb_name))
with self._lock:
params = {
"elb_name": elb_name
}
elb_entries = self._db.retry_request('get', self._table, params=params, max_retry=5, retry_on_exception=self._db.get_retry_on_exception, value_only=True)
if len(elb_entries) < 1:
raise AXNotFoundException("Did not find an entry for {} in node port table".format(elb_name))
for elb in elb_entries:
elb["elb_addr"] = elb_addr
self._db.retry_request('put', self._table, data=elb, max_retry=5, retry_on_exception=self._db.create_retry_on_exception, value_only=True)
def create_alias_record(self, name, elb_addr, elb_name=None):
"""
Create an alias record. This is idempotent as it uses UPSERT (create or update if exists)
Args:
name: The name of the alias. E.g. For zone test.acme.com if you want to create a alias alias1.test.acme.com
then name is alias1
elb_addr: The ELB address to point to
elb_hostedzoneid: the hosted zone ID of the ELB
Returns:
"""
name = name.lower()
subdomain_name_check(name)
@retry_exponential(noretry=[404])
def get_load_balancer_info(elb_client, lb_name, elb_addr):
for elb in elb_client.describe_load_balancers(
LoadBalancerNames=[lb_name])['LoadBalancerDescriptions']:
if elb['DNSName'] == elb_addr:
return elb['CanonicalHostedZoneNameID']
return None
region = AXClusterConfig().get_region()
elb_client = boto3.Session().client("elb", region_name=region)
if elb_name is None:
# try to infer from elb_addr
# http://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_CreateLoadBalancer.html
# From the link above "This name must be unique within your set of load balancers for the region, must have
# a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and cannot begin or end with a hyphen."
load_balancer_name = elb_addr[:32]
else:
load_balancer_name = elb_name
elb_hostedzoneid = get_load_balancer_info(elb_client,
load_balancer_name, elb_addr)
if elb_hostedzoneid is None:
raise AXNotFoundException(
"Could not find the ELB for the elb_addr {} in AWS".format(
elb_addr))
self.get_hosted_zone_id()
record = {
"Name": "{}.{}".format(name, self.name),
"Type": "A",
"AliasTarget": {
'DNSName': elb_addr,
'HostedZoneId': elb_hostedzoneid,
'EvaluateTargetHealth': False
}
}
self.client.change_record(self._id, "UPSERT", record)
def exception_handler(*args, **kwargs):
try:
return func(*args, **kwargs)
except botocore.exceptions.ClientError as e:
code = e.response["ResponseMetadata"]["HTTPStatusCode"]
msg = e.message
if code == 409:
raise AXConflictException(msg)
elif code == 404 or code == 400:
# 400 also seems to be used for not found by AWS
raise AXNotFoundException(msg)
else:
raise AXPlatformException(msg)
def delete(self, aws_resources_only=False):
(application, deployment) = self._get_app_and_dep()
if application is None or deployment is None:
raise AXNotFoundException("Could not find application and deployment for deleting managed elb {}".format(self.name))
if not aws_resources_only:
# Start by deleting NodeRoute Objects first
logger.debug("Deleting node routes for the managed elb {}".format(self.name))
nr_name = self.name
noderoute = NodeRoute(nr_name, application)
with ManagedElbNodeOperation(noderoute):
if noderoute.exists():
noderoute.delete()
# Release all the nodeport after the NodeRoutes are gone
logger.debug("Releasing nodeports for the managed elb {}".format(self.name))
self._npm.release_all(self.name)
# Now unterraform to get rid of the ELB
init_command = 'terraform init -backend=true -backend-config="bucket={}" -backend-config="key={}managed_elbs/{}.tfstate" -backend-config="region={}" -reconfigure'.format(
self.bucket, self.terraform_dir, self.name, self.region
)
terraform_command = "terraform destroy --force -var 'asg_name={}' -var 'cluster_id={}' -var 'cidrs=[]' -var 'elb_prefix={}' -var 'ports=[]' -var 'protocols=[]' -var 'region={}' -var 'application={}' -var 'deployment={}'".format(
self._get_asg_name(application), self.name_id, self.name, self.region, application, deployment
)
tempdir = tempfile.mkdtemp()
self.s3.download_file("{}managed_elbs/{}.tf".format(self.terraform_dir, self.name), "{}/elb.tf".format(tempdir))
logger.debug("Terraform init\n{}".format(subprocess.check_output(init_command, shell=True, cwd=tempdir)))
logger.debug("Terraform destroy\n{}".format(subprocess.check_output(terraform_command, shell=True, cwd=tempdir)))
shutil.rmtree(tempdir, ignore_errors=True)
# delete the terraform state and config file
self.s3.delete_object("{}managed_elbs/{}.tf".format(self.terraform_dir, self.name))
self.s3.delete_object("{}managed_elbs/{}.tfstate".format(self.terraform_dir, self.name))
def create(self, application, deployment, ports_spec, internal=True, labels=None):
self._creation_prechecks(application, deployment)
# Create the port spec for the NodeRoute and get node ports for each of the listener ports and generate
# the listener spec and the health check spec
listener_spec = ""
health_check_spec = None
i = 0
srv_pspec = []
listen_ports = []
protocols = []
for port in ports_spec:
# Build the port spec for NodeRoute
curr_port = {}
curr_port["name"] = "port-{}".format(i)
curr_port["port"] = port["listen_port"]
curr_port["target_port"] = port["container_port"]
curr_port["node_port"] = self._npm.get(self.name, port["listen_port"])
srv_pspec.append(curr_port)
# Build array of listen ports and protocol for security rules
listen_ports.append(port["listen_port"])
protocols.append(port["protocol"])
# Build listener spec for ELB
ssl_cert = ""
if "https" in port["protocol"] or "ssl" in port["protocol"]:
ssl_cert = "ssl_certificate_id = \"{}\"".format(port["certificate"])
listener_spec += LISTENER_TEMPLATE.format(curr_port["node_port"], self.protocol_map[port["protocol"]], port["listen_port"], port["protocol"], ssl_cert)
# Build health check spec for ELB
if health_check_spec is None:
health_check_spec = HEALTHCHECK_TEMPLATE.format(curr_port["node_port"])
i += 1
# create the nodeport k8s service
# Note: We generate a random string [a-z] rather than using the elb name (even though elb name needs to be unique
# w.r.t other elb names, because we need to generate a kubernetes service object. It is possible that the
# user has generated an internal route with that name and we do not want to clobber the internal route
nr_name = self.name
noderoute = NodeRoute(nr_name, application)
# Get a lock for operations of exist() delete() and create() as the InternalRouteOperation will not be
# atomic across all these operations.
with ManagedElbNodeOperation(noderoute):
if noderoute.exists():
noderoute.delete()
noderoute.create(srv_pspec, selector=labels)
# modify the elb template using the pass port specification
my_elb_template = AWS_ELB_TEMPLATE.replace("# AXMODIFY-LISTENERS", listener_spec)
my_elb_template = my_elb_template.replace("# AXMODIFY-HEALTHCHECK", health_check_spec)
# Create cloud specific resources using terraform
init_command = 'terraform init -backend=true -backend-config="bucket={}" -backend-config="key={}managed_elbs/{}.tfstate" -backend-config="region={}" -reconfigure'.format(
self.bucket, self.terraform_dir, self.name, self.region
)
terraform_command = "terraform apply -var 'asg_name={}' -var 'cluster_id={}' -var 'cidrs={}' -var 'elb_prefix={}' -var 'ports={}' -var 'protocols={}' -var 'region={}' -var 'application={}' -var 'deployment={}' -var 'elb_internal={}'".format(
self._get_asg_name(application), self.name_id, json.dumps(self.trusted_cidrs), self.name, json.dumps(listen_ports), json.dumps(protocols), self.region, application, deployment, (str(internal)).lower()
)
logger.debug("Terraform spec\n{}".format(my_elb_template))
logger.debug("Terraform init command is: {}".format(init_command))
logger.debug("Terraform command is: {}".format(terraform_command))
tempdir = tempfile.mkdtemp()
with open("{}/elb.tf".format(tempdir), "w") as f:
f.write(my_elb_template)
# also write the file to S3
self.s3.put_object("{}managed_elbs/{}.tf".format(self.terraform_dir, self.name), my_elb_template)
logger.debug("Terraform init\n{}".format(subprocess.check_output(init_command, shell=True, cwd=tempdir)))
logger.debug("Terraform apply\n{}".format(subprocess.check_output(terraform_command, shell=True, cwd=tempdir)))
shutil.rmtree(tempdir, ignore_errors=True)
elb = self._get_load_balancer_info()
if elb is None:
raise AXNotFoundException(
"Created ELB {} was not found. That is odd. Please try creating again".format(self.name))
dnsname = elb["DNSName"]
self._npm.add_elb_addr(self.name, dnsname)
return dnsname