def create_service_account(namespace: str) -> object:
enforce_authorization(namespace)
f = open("templates/keycloak/client.json", "r")
j = json.loads(f.read())
cid = "sa-%s-%s" % (namespace, get_random_string(10))
j['clientId'] = cid
j['protocolMappers'][0]['config']['claim.value'] = namespace
keycloak_admin = admin_api()
try:
response = keycloak_admin.create_client(j)
cuuid = keycloak_admin.get_client_id(cid)
r = keycloak_admin.generate_client_secrets(cuuid)
return ({'client_id': cid, 'client_secret': r['value']}, 201)
except KeycloakGetError as err:
if err.response_code == 409:
abort(
make_response(
jsonify(
error=
"Service Account for this namespace is already created."
), 400))
else:
log.error(err)
abort(
make_response(jsonify(error="Failed to add service account"),
400))
def update_service_account_credentials(namespace: str,
client_id: str) -> object:
enforce_authorization(namespace)
cid = "sa-%s-" % namespace
if not client_id.startswith(cid):
abort(make_response(jsonify(error="Invalid client ID"), 400))
keycloak_admin = admin_api()
try:
cuuid = keycloak_admin.get_client_id(client_id)
r = keycloak_admin.generate_client_secrets(cuuid)
return ({'client_id': client_id, 'client_secret': r['value']}, 201)
except KeycloakGetError as err:
if err.response_code == 409:
abort(
make_response(
jsonify(
error=
"Service Account for this namespace is already created."
), 400))
else:
log.error(err)
abort(
make_response(jsonify(error="Failed to add service account"),
400))
def update_namespace(namespace: str) -> object:
log = app.logger
enforce_authorization(namespace)
params = request.get_json(force=True)
if not namespace_valid(namespace):
log.error("Namespace validation failed %s", namespace)
abort(
make_response(
jsonify(
error=
"Namespace name validation failed. Reference regular expression '%s'."
% namespace_validation_rule), 400))
try:
svc = NamespaceService()
ns_group = svc.get_namespace(namespace)
svc.update_ns_attributes(ns_group, params)
except KeycloakGetError as err:
log.error("Failed to update namespace %s", namespace)
log.error(err)
abort(make_response(jsonify(error="Failed to update namespace"), 400))
return make_response(jsonify())
def update_membership(namespace: str) -> object:
# Sync the membership list provided with the group membership
# in Keycloak
#
enforce_authorization(namespace)
desired_membership_list = json.loads(request.get_data())
ucounts_added, ucounts_removed, ucounts_missing = membership_sync(
namespace, 'viewer', desired_membership_list)
acounts_added, acounts_removed, acounts_missing = membership_sync(
namespace, 'admin', desired_membership_list)
return make_response(
jsonify(added=ucounts_added + acounts_added,
removed=ucounts_removed + acounts_removed,
missing=ucounts_missing + acounts_missing))
def delete_namespace(namespace: str) -> object:
log = app.logger
enforce_authorization(namespace)
enforce_role_authorization('aps.ns:manage')
keycloak_admin = admin_api()
try:
for role_name in ['viewer', 'admin']:
group = keycloak_admin.get_group_by_path(
"%s/%s" % (get_base_group_path(role_name), namespace),
search_in_subgroups=True)
if group is not None:
keycloak_admin.delete_group(group['id'])
except KeycloakGetError as err:
log.error(err)
abort(make_response(jsonify(error="Failed to delete namespace"), 400))
return ('', 204)
def list_service_accounts(namespace: str) -> object:
enforce_authorization(namespace)
keycloak_admin = admin_api()
try:
params_path = {"realm-name": keycloak_admin.realm_name}
data_raw = keycloak_admin.raw_get(
URL_ADMIN_CLIENTS.format(**params_path),
clientId='sa-%s-' % namespace,
search=True)
response = raise_error_from_response(data_raw, KeycloakGetError)
result = []
for r in response:
result.append(r['clientId'])
return (json.dumps(result), 200)
except KeycloakGetError as err:
log.error(err)
abort(
make_response(jsonify(error="Failed to read service accounts"),
400))
def delete_service_account(namespace: str, client_id: str) -> object:
log = app.logger
enforce_authorization(namespace)
if client_id_valid(namespace, client_id) == False:
abort(make_response(jsonify(error="Invalid client ID"), 400))
keycloak_admin = admin_api()
try:
cuuid = keycloak_admin.get_client_id(client_id)
if cuuid is None:
abort(
make_response(jsonify(error="Service Account does not exist"),
400))
else:
keycloak_admin.delete_client(cuuid)
return ({}, 204)
except KeycloakGetError as err:
log.error(err)
abort(
make_response(jsonify(error="Failed to delete service account"),
400))
def get_statuses(namespace: str) -> object:
enforce_authorization(namespace)
log = app.logger
log.info("Get status for %s" % namespace)
services = get_services_by_ns(namespace)
routes = get_routes_by_ns(namespace)
response = []
for service in services:
url = build_url(service)
status = "UP"
reason = ""
actual_host = None
host = None
for route in routes:
if route['service']['id'] == service['id'] and 'hosts' in route:
actual_host = route['hosts'][0]
host = clean_host(actual_host)
try:
addr = socket.gethostbyname(service['host'])
log.info("Address = %s" % addr)
except:
status = "DOWN"
reason = "DNS"
if status == "UP":
try:
headers = {}
if host is None or service['host'].endswith('.svc'):
r = requests.get(url, headers=headers, timeout=3.0)
status_code = r.status_code
else:
u = urlparse(url)
headers['Host'] = host
log.info("GET %-30s %s" % ("%s://%s" %
(u.scheme, u.netloc), headers))
urllib3.disable_warnings()
if u.scheme == "https":
pool = urllib3.HTTPSConnectionPool(
"%s" % (u.netloc),
assert_hostname=host,
server_hostname=host,
cert_reqs='CERT_NONE',
ca_certs=certifi.where())
else:
pool = urllib3.HTTPConnectionPool("%s" % (u.netloc))
req = pool.urlopen("GET",
u.path,
headers={"Host": host},
assert_same_host=False,
timeout=1.0,
retries=False)
status_code = req.status
log.info("Result received!! %d" % status_code)
if status_code < 400:
status = "UP"
reason = "%d Response" % status_code
elif status_code == 401 or status_code == 403:
status = "UP"
reason = "AUTH %d" % status_code
else:
status = "DOWN"
reason = "%d Response" % status_code
except requests.exceptions.Timeout as ex:
status = "DOWN"
reason = "TIMEOUT"
except urllib3.exceptions.ConnectTimeoutError as ex:
status = "DOWN"
reason = "TIMEOUT"
except requests.exceptions.ConnectionError as ex:
log.error("ConnError %s" % ex)
status = "DOWN"
reason = "CONNECTION"
except requests.exceptions.SSLError as ex:
status = "DOWN"
reason = "SSL"
except urllib3.exceptions.NewConnectionError as ex:
log.error("NewConnError %s" % ex)
status = "DOWN"
reason = "CON_ERR"
except urllib3.exceptions.SSLError as ex:
log.error(ex)
status = "DOWN"
reason = "SSL_URLLIB3"
except Exception as ex:
log.error(ex)
traceback.print_exc(file=sys.stdout)
status = "DOWN"
reason = "UNKNOWN"
log.info("GET %-30s %s" % (url, reason))
response.append({
"name": service['name'],
"upstream": url,
"status": status,
"reason": reason,
"host": host,
"env_host": actual_host
})
return make_response(jsonify(response))
def delete_config(namespace: str, qualifier = "") -> object:
enforce_authorization(namespace)
event_id = str(uuid.uuid4())
record_gateway_event(event_id, 'delete', 'received', namespace)
log = app.logger
outFolder = namespace
tempFolder = "%s/%s/%s" % ('/tmp', uuid.uuid4(), outFolder)
os.makedirs (tempFolder, exist_ok=False)
with open("%s/%s" % (tempFolder, 'empty.yaml'), 'w') as file:
file.write("")
selectTag = "ns.%s" % namespace
log.debug("ST = %s" % selectTag)
if qualifier is not None and qualifier != "":
log.debug("What is qual? %s" % qualifier)
selectTag = "ns.%s.%s" % (namespace, qualifier)
# Call the 'deck' command
cmd = "sync"
log.info("[%s] %s action using %s" % (namespace, cmd, selectTag))
args = [
"deck", cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", selectTag, "--state", tempFolder
]
log.debug("[%s] Running %s" % (namespace, args))
deck_run = Popen(args, stdout=PIPE, stderr=STDOUT)
out, err = deck_run.communicate()
if deck_run.returncode != 0:
cleanup (tempFolder)
log.warn("%s - %s" % (namespace, out.decode('utf-8')))
abort_early(event_id, 'delete', namespace, jsonify(error="Sync Failed.", results=mask(out.decode('utf-8'))) )
elif cmd == "sync":
try:
route_count = prepare_apply_routes (namespace, selectTag, is_host_transform_enabled(), tempFolder)
log.debug("%s - Prepared %d routes" % (namespace, route_count))
if route_count > 0:
apply_routes (tempFolder)
log.debug("%s - Applied %d routes" % (namespace, route_count))
route_count = prepare_delete_routes (namespace, selectTag, tempFolder)
log.debug("%s - Prepared %d deletions" % (namespace, route_count))
if route_count > 0:
delete_routes (tempFolder)
# create Network Security Policies (nsp) for any upstream that
# has the format: <name>.<ocp_ns>.svc
log.debug("%s - Update NSPs" % (namespace))
ocp_ns_list = get_ocp_service_namespaces (tempFolder)
for ocp_ns in ocp_ns_list:
if check_nsp (namespace, ocp_ns) is False:
apply_nsp (namespace, ocp_ns, tempFolder)
# ok all looks good, so update a secret containing the original submitted request
log.debug("%s - Update Original Config" % (namespace))
write_submitted_config ("", tempFolder)
prep_and_apply_secret (namespace, selectTag, tempFolder)
log.debug("%s - Updated Original Config" % (namespace))
except HTTPException as ex:
traceback.print_exc()
log.error("Error updating custom routes, nsps and secrets. %s" % ex)
abort_early(event_id, 'delete', namespace, jsonify(error="Partially failed.") )
except:
traceback.print_exc()
log.error("Error updating custom routes, nsps and secrets. %s" % sys.exc_info()[0])
abort_early(event_id, 'delete', namespace, jsonify(error="Partially failed.") )
cleanup (tempFolder)
log.debug("[%s] The exit code was: %d" % (namespace, deck_run.returncode))
message = "Sync successful."
if cmd == 'diff':
message = "Dry-run. No changes applied."
record_gateway_event(event_id, 'delete', 'completed', namespace)
return make_response('', http.HTTPStatus.NO_CONTENT)
def write_config(namespace: str) -> object:
"""
(Over)write
:return: JSON of success message or error message
"""
enforce_authorization(namespace)
event_id = str(uuid.uuid4())
record_gateway_event(event_id, 'publish', 'received', namespace)
log = app.logger
outFolder = namespace
# Build a list of existing hosts that are outside this namespace
# They become reserved and any conflict will return an error
reserved_hosts = []
all_routes = get_routes()
tag_match = "ns.%s" % namespace
for route in all_routes:
if tag_match not in route['tags'] and 'hosts' in route:
for host in route['hosts']:
reserved_hosts.append(transform_host(host))
reserved_hosts = list(set(reserved_hosts))
ns_svc = NamespaceService()
ns_attributes = ns_svc.get_namespace_attributes (namespace)
dfile = None
if 'configFile' in request.files:
log.debug("[%s] %s", namespace, request.files['configFile'])
dfile = request.files['configFile']
dry_run = request.values['dryRun']
elif request.content_type.startswith("application/json"):
dfile = request.json['configFile']
dry_run = request.json['dryRun']
else:
log.error("Missing input")
log.error("%s", request.get_data())
log.error(request.form)
log.error(request.content_type)
log.error(request.headers)
abort_early(event_id, 'publish', namespace, jsonify(error="Missing input"))
tempFolder = "%s/%s/%s" % ('/tmp', uuid.uuid4(), outFolder)
os.makedirs (tempFolder, exist_ok=False)
# dfile.save("%s/%s" % (tempFolder, 'config.yaml'))
# log.debug("Saved to %s" % tempFolder)
yaml_documents_iter = yaml.load_all(dfile, Loader=yaml.FullLoader)
yaml_documents = []
for doc in yaml_documents_iter:
yaml_documents.append(doc)
selectTag = "ns.%s" % namespace
ns_qualifier = None
orig_config = prep_submitted_config (yaml_documents)
update_routes_flag = False
if len(yaml_documents) == 0:
update_routes_flag = True
for index, gw_config in enumerate(yaml_documents):
log.info("[%s] Parsing file %s" % (namespace, index))
if gw_config is None:
continue
#######################
# Enrichments
#######################
# Transformation route hosts if in non-prod environment (HOST_TRANSFORM_ENABLED)
host_transformation (namespace, gw_config)
# If there is a tag with a pipeline qualifier (i.e./ ns.<namespace>.dev)
# then add to tags automatically the tag: ns.<namespace>
tags_transformation (namespace, gw_config)
#
# Enrich the rate-limiting plugin with the appropriate Redis details
plugins_transformations (namespace, gw_config)
with open("%s/%s" % (tempFolder, 'config-%02d.yaml' % index), 'w') as file:
yaml.dump(gw_config, file)
#######################
# Validations
#######################
# Validate that the every object is tagged with the namespace
try:
validate_tags (gw_config, selectTag)
except Exception as ex:
traceback.print_exc()
log.error("%s - %s" % (namespace, " Tag Validation Errors: %s" % ex))
abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ex))
# Validate that hosts are valid
try:
validate_hosts (gw_config, reserved_hosts, ns_attributes)
except Exception as ex:
traceback.print_exc()
log.error("%s - %s" % (namespace, " Host Validation Errors: %s" % ex))
abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ex))
# Validate upstream URLs are valid
try:
protected_kube_namespaces = json.loads(app.config['protectedKubeNamespaces'])
validate_upstream (gw_config, ns_attributes, protected_kube_namespaces)
except Exception as ex:
traceback.print_exc()
log.error("%s - %s" % (namespace, " Upstream Validation Errors: %s" % ex))
abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ex))
# Validation #3
# Validate that certain plugins are configured (such as the gwa_gov_endpoint) at the right level
# Validate based on DNS 952
nsq = traverse_get_ns_qualifier (gw_config, selectTag)
if nsq is not None:
if ns_qualifier is not None and nsq != ns_qualifier:
abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ("Conflicting ns qualifiers (%s != %s)" % (ns_qualifier, nsq))))
ns_qualifier = nsq
log.info("[%s] CHANGING ns_qualifier %s" % (namespace, ns_qualifier))
if update_routes_check(gw_config):
update_routes_flag = True
if ns_qualifier is not None:
selectTag = ns_qualifier
# Call the 'deck' command
cmd = "sync"
if dry_run == 'true' or dry_run is True:
cmd = "diff"
log.info("[%s] %s action using %s" % (namespace, cmd, selectTag))
args = [
"deck", cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", selectTag, "--state", tempFolder
]
log.debug("[%s] Running %s" % (namespace, args))
deck_run = Popen(args, stdout=PIPE, stderr=STDOUT)
out, err = deck_run.communicate()
if deck_run.returncode != 0:
cleanup (tempFolder)
log.warn("[%s] - %s" % (namespace, out.decode('utf-8')))
abort_early(event_id, 'publish', namespace, jsonify(error="Sync Failed.", results=mask(out.decode('utf-8'))))
elif cmd == "sync":
try:
if update_routes_flag:
route_count = prepare_apply_routes (namespace, selectTag, is_host_transform_enabled(), tempFolder)
log.debug("[%s] - Prepared %d routes" % (namespace, route_count))
if route_count > 0:
apply_routes (tempFolder)
log.debug("[%s] - Applied %d routes" % (namespace, route_count))
route_count = prepare_delete_routes (namespace, selectTag, tempFolder)
log.debug("[%s] - Prepared %d deletions" % (namespace, route_count))
if route_count > 0:
delete_routes (tempFolder)
# create Network Security Policies (nsp) for any upstream that
# has the format: <name>.<ocp_ns>.svc
if should_we_apply_nsp_policies():
log.debug("[%s] - Update NSPs" % (namespace))
ocp_ns_list = get_ocp_service_namespaces (tempFolder)
for ocp_ns in ocp_ns_list:
if check_nsp (namespace, ocp_ns) is False:
apply_nsp (namespace, ocp_ns, tempFolder)
# ok all looks good, so update a secret containing the original submitted request
log.debug("[%s] - Update Original Config" % (namespace))
write_submitted_config (orig_config, tempFolder)
prep_and_apply_secret (namespace, selectTag, tempFolder)
log.debug("[%s] - Updated Original Config" % (namespace))
except HTTPException as ex:
traceback.print_exc()
log.error("[%s] Error updating custom routes, nsps and secrets. %s" % (namespace, ex))
abort_early(event_id, 'publish', namespace, jsonify(error="Partially failed."))
except:
traceback.print_exc()
log.error("[%s] Error updating custom routes, nsps and secrets. %s" % (namespace, sys.exc_info()[0]))
abort_early(event_id, 'publish', namespace, jsonify(error="Partially failed."))
cleanup (tempFolder)
log.debug("[%s] The exit code was: %d" % (namespace, deck_run.returncode))
message = "Sync successful."
if cmd == 'diff':
message = "Dry-run. No changes applied."
record_gateway_event(event_id, 'publish', 'completed', namespace)
return make_response(jsonify(message=message, results=mask(out.decode('utf-8'))))