def get_api_auth_names(session, domain):
"""
gets the api and auth hostnames from the domain
Args:
session(Session): boto3 session object
domain(str): name of domain. Ex: integration.boss
Returns:
tuple(str) of api hostname and auth hostname
"""
hosted_zone = aws.get_hosted_zone(session)
if domain in hosts.BASE_DOMAIN_CERTS.keys():
if hosted_zone != hosts.PROD_DOMAIN:
print(
"Incorrect AWS credentials being provided for the production domain, please check them."
)
sys.exit(1)
api = "api.{}.".format(hosts.BASE_DOMAIN_CERTS[domain])
auth = "auth.{}.".format(hosts.BASE_DOMAIN_CERTS[domain])
else:
print(
"Maintenance can only be performed in production account."
) # The reason for this is cloudfront sessions would need to be created for dev systems.
sys.exit(1)
# if hosted_zone != hosts.DEV_DOMAIN:
# print("Possibly wrong credentials being used, domain, {}, is not supposed to be used in this aws account")
# sys.exit(1)
# api = "api-{}.{}.".format(domain.split('.')[0], hosts.DEV_DOMAIN)
# auth = "auth-{}.{}.".format(domain.split('.')[0], hosts.DEV_DOMAIN)
return (api, auth)
def get_api_auth_names(session, domain):
"""
gets the api and auth hostnames from the domain
Args:
session(Session): boto3 session object
domain(str): name of domain. Ex: integration.boss
Returns:
tuple(str) of api hostname and auth hostname
"""
hosted_zone = aws.get_hosted_zone(session)
if domain in hosts.BASE_DOMAIN_CERTS.keys():
if hosted_zone != hosts.PROD_DOMAIN:
print("Incorrect AWS credentials being provided for the production domain, please check them.")
sys.exit(1)
api = "api.{}.".format(hosts.BASE_DOMAIN_CERTS[domain])
auth = "auth.{}.".format(hosts.BASE_DOMAIN_CERTS[domain])
else:
print("Maintenance can only be performed in production account.") # The reason for this is cloudfront sessions would need to be created for dev systems.
sys.exit(1)
# if hosted_zone != hosts.DEV_DOMAIN:
# print("Possibly wrong credentials being used, domain, {}, is not supposed to be used in this aws account")
# sys.exit(1)
# api = "api-{}.{}.".format(domain.split('.')[0], hosts.DEV_DOMAIN)
# auth = "auth-{}.{}.".format(domain.split('.')[0], hosts.DEV_DOMAIN)
return (api, auth)
def migrations_on(session, domain):
"""
changes Route53 entry for the api in domain to use cloudfront for the s3 maintenance bucket.
Args:
session(Session): boto3 session object
domain(str): name of domain. Ex: integration.boss
Returns:
Nothing
"""
hosted_zone = aws.get_hosted_zone(session)
(api, auth) = get_api_auth_names(session, domain)
api_cloud_front = aws.cloudfront_public_lookup(session, api[:-1])
print("Setting Route53 for: ")
print("{}: {}".format(api, api_cloud_front))
aws.set_domain_to_dns_name(session, api, api_cloud_front, hosted_zone)
warnings(api)
def migrations_on(session, domain):
"""
changes Route53 entry for the api in domain to use cloudfront for the s3 maintenance bucket.
Args:
session(Session): boto3 session object
domain(str): name of domain. Ex: integration.boss
Returns:
Nothing
"""
hosted_zone = aws.get_hosted_zone(session)
(api, auth) = get_api_auth_names(session, domain)
api_cloud_front = aws.cloudfront_public_lookup(session, api[:-1])
print("Setting Route53 for: ")
print("{}: {}".format(api, api_cloud_front))
aws.set_domain_to_dns_name(session, api, api_cloud_front, hosted_zone)
warnings(api)
def migrations_off(session, domain):
"""
changes Route53 entries for the domain to use the api elastic load balancers
Args:
session(Session): boto3 session object
domain(str): name of domain. Ex: integration.boss
Returns:
Nothing
"""
names = AWSNames(domain)
hosted_zone = aws.get_hosted_zone(session)
(api, auth) = get_api_auth_names(session, domain)
api_elb = aws.elb_public_lookup(session, "elb." + domain)
print("Setting Route53 for: ")
print("{}: {}".format(api, api_elb))
aws.set_domain_to_dns_name(session, api, api_elb, hosted_zone)
warnings(api)
def migrations_off(session, domain):
"""
changes Route53 entries for the domain to use the api elastic load balancers
Args:
session(Session): boto3 session object
domain(str): name of domain. Ex: integration.boss
Returns:
Nothing
"""
names = AWSNames(domain)
hosted_zone = aws.get_hosted_zone(session)
(api, auth) = get_api_auth_names(session, domain)
api_elb = aws.elb_public_lookup(session, "elb." + domain)
print("Setting Route53 for: ")
print("{}: {}".format(api, api_elb))
aws.set_domain_to_dns_name(session, api, api_elb, hosted_zone)
warnings(api)
def post_init(session, domain):
# Keypair is needed by ExternalCalls
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Configure external DNS
# DP ???: Can this be moved into the CloudFormation template?
dns = names.public_dns("api")
dns_elb = aws.elb_public_lookup(session, names.endpoint_elb)
aws.set_domain_to_dns_name(session, dns, dns_elb, aws.get_hosted_zone(session))
# Write data into Vault
# DP TODO: Move into the pre-launch Vault writes, so it is available when the
# machines initially start
with call.vault() as vault:
uri = "https://{}".format(dns)
#vault.update(const.VAULT_ENDPOINT_AUTH, public_uri = uri)
creds = vault.read("secret/auth")
bossadmin = vault.read("secret/auth/realm")
auth_uri = vault.read("secret/endpoint/auth")['url']
# Verify Keycloak is accessible
print("Checking for Keycloak availability")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
# Add the API servers to the list of OIDC valid redirects
with call.tunnel(names.auth, 8080) as auth_port:
print("Update KeyCloak Client Info")
auth_url = "http://localhost:{}".format(auth_port)
with KeyCloakClient(auth_url, **creds) as kc:
# DP TODO: make add_redirect_uri able to work multiple times without issue
kc.add_redirect_uri("BOSS","endpoint", uri + "/*")
# Get the boss admin's bearer token
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
params = {
'grant_type': 'password',
'client_id': bossadmin['client_id'],
'username': bossadmin['username'],
'password': bossadmin['password'],
}
auth_uri += '/protocol/openid-connect/token'
req = Request(auth_uri,
headers = headers,
data = urlencode(params).encode('utf-8'))
resp = json.loads(urlopen(req).read().decode('utf-8'))
# Make an API call that will log the boss admin into the endpoint
call.check_url(uri + '/ping', 60)
headers = {
'Authorization': 'Bearer {}'.format(resp['access_token']),
}
api_uri = uri + '/latest/collection'
req = Request(api_uri, headers = headers)
resp = json.loads(urlopen(req).read().decode('utf-8'))
print("Collections: {}".format(resp))
# Tell Scalyr to get CloudWatch metrics for these instances.
instances = [names.endpoint]
scalyr.add_instances_to_scalyr(
session, const.REGION, instances)
if __name__ == '__main__':
os.chdir(os.path.abspath(os.path.dirname(__file__)))
# DP NOTE: This should possibly be in user-scratch as it is specific to our deployment
# DP ???: Still used?
parser = argparse.ArgumentParser(
description="Request SSL domain certificates for theboss.io subdomains",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='create domain cert')
parser.add_argument(
"domain_name",
help=
"Domain to create the SSL certificate for (Ex: api.integration.theboss.io)"
)
args = parser.parse_args()
if args.aws_credentials is None:
parser.print_usage()
print(
"Error: AWS credentials not provided and AWS_CREDENTIALS is not defined"
)
sys.exit(1)
session = aws.create_session(args.aws_credentials)
results = aws.request_cert(session, args.domain_name,
aws.get_hosted_zone(session))
print(results)
def post_init(session, domain, startup_wait=False):
# Keypair is needed by ExternalCalls
global keypair
if keypair is None:
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Figure out the external domain name of the auth server(s), matching the SSL cert
auth_domain = names.public_dns("auth")
# OIDC Discovery URL
auth_discovery_url = "https://{}/auth/realms/BOSS".format(auth_domain)
# Configure external DNS
auth_elb = aws.elb_public_lookup(session, names.auth)
aws.set_domain_to_dns_name(session, auth_domain, auth_elb, aws.get_hosted_zone(session))
# Generate initial user accounts
username = "******"
password = utils.generate_password()
realm_username = "******"
realm_password = utils.generate_password()
# Initialize Vault
print("Waiting for Vault...")
call.check_vault(const.TIMEOUT_VAULT) # Expecting this to also check Consul
with call.vault() as vault:
print("Initializing Vault...")
try:
vault.initialize()
except Exception as ex:
print(ex)
print("Could not initialize Vault")
print("Call: {}".format(utils.get_command("post-init")))
print("Before launching other stacks")
return
#Check and see if these secrets already exist before we overwrite them with new ones.
# Write data into Vault
if not vault.read(const.VAULT_AUTH):
print("Writing {}".format(const.VAULT_AUTH))
vault.write(const.VAULT_AUTH, password = password, username = username, client_id = "admin-cli")
if not vault.read(const.VAULT_REALM):
print("Writing {}".format(const.VAULT_REALM))
vault.write(const.VAULT_REALM, username = realm_username, password = realm_password, client_id = "endpoint")
if not vault.read(const.VAULT_KEYCLOAK):
print("Updating {}".format(const.VAULT_KEYCLOAK))
vault.update(const.VAULT_KEYCLOAK, password = password, username = username, client_id = "admin-cli", realm = "master")
if not vault.read(const.VAULT_ENDPOINT_AUTH):
# DP TODO: Move this update call into the api config
print("Updating {}".format(const.VAULT_ENDPOINT_AUTH))
vault.update(const.VAULT_ENDPOINT_AUTH, url = auth_discovery_url, client_id = "endpoint")
if not vault.read(const.VAULT_PROOFREAD_AUTH):
# DP TODO: Move this update call into the proofreader config
print("Updating {}".format(const.VAULT_PROOFREAD_AUTH))
vault.update(const.VAULT_PROOFREAD_AUTH, url = auth_discovery_url, client_id = "endpoint")
# Configure Keycloak
print("Waiting for Keycloak to bootstrap")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
#######
## DP TODO: Need to find a check so that the master user is only added once to keycloak
## Also need to guard the writes to vault with the admin password
#######
with call.ssh(names.auth) as ssh:
print("Creating initial Keycloak admin user")
ssh("/srv/keycloak/bin/add-user.sh -r master -u {} -p {}".format(username, password))
print("Restarting Keycloak")
ssh("sudo service keycloak stop")
time.sleep(2)
ssh("sudo killall java") # the daemon command used by the keycloak service doesn't play well with standalone.sh
# make sure the process is actually killed
time.sleep(3)
ssh("sudo service keycloak start")
print("Waiting for Keycloak to restart")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
with call.tunnel(names.auth, 8080) as port:
URL = "http://localhost:{}".format(port) # TODO move out of tunnel and use public address
with KeyCloakClient(URL, username, password) as kc:
print("Opening realm file at '{}'".format(const.KEYCLOAK_REALM))
with open(const.KEYCLOAK_REALM, "r") as fh:
realm = json.load(fh)
try:
realm["users"][0]["username"] = realm_username
realm["users"][0]["credentials"][0]["value"] = realm_password
except:
print("Could not set realm admin's username or password, not creating user")
if "users" in realm:
del realm["users"]
print("Uploading BOSS.realm configuration")
kc.create_realm(realm)
# Tell Scalyr to get CloudWatch metrics for these instances.
instances = [ names.vault ]
scalyr.add_instances_to_scalyr(session, const.REGION, instances)
import sys
import os
import alter_path
from lib import aws
if __name__ == '__main__':
os.chdir(os.path.abspath(os.path.dirname(__file__)))
parser = argparse.ArgumentParser(description="Request SSL domain certificates for theboss.io subdomains",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='create domain cert')
parser.add_argument("--aws-credentials", "-a",
metavar="<file>",
default=os.environ.get("AWS_CREDENTIALS"),
type=argparse.FileType('r'),
help="File with credentials to use when connecting to AWS (default: AWS_CREDENTIALS)")
parser.add_argument("domain_name", help="Domain to create the SSL certificate for (Ex: api.integration.theboss.io)")
args = parser.parse_args()
if args.aws_credentials is None:
parser.print_usage()
print("Error: AWS credentials not provided and AWS_CREDENTIALS is not defined")
sys.exit(1)
session = aws.create_session(args.aws_credentials)
results = aws.request_cert(session, args.domain_name, aws.get_hosted_zone(session))
print(results)