def create(bosslet_config):
"""Configure Vault, create the configuration, and launch it"""
db_config = const.ENDPOINT_DB_CONFIG.copy()
db_config['password'] = utils.generate_password()
with bosslet_config.call.vault() as vault:
vault.write(const.VAULT_ENDPOINT, secret_key=str(uuid.uuid4()))
vault.write(const.VAULT_ENDPOINT_DB, **db_config)
vault.write(const.VAULT_ENDPOINT_THROTTLE,
config=json.dumps(const.THROTTLE))
dns = bosslet_config.names.public_dns("api")
uri = "https://{}".format(dns)
vault.update(const.VAULT_ENDPOINT_AUTH, public_uri=uri)
config = create_config(bosslet_config, db_config)
pre_init(bosslet_config)
try:
config.create()
except:
print("Error detected, revoking secrets")
try:
with bosslet_config.call.vault() as vault:
vault.delete(const.VAULT_ENDPOINT)
vault.delete(const.VAULT_ENDPOINT_DB)
#vault.delete(const.VAULT_ENDPOINT_AUTH) # Deleting this will bork the whole stack
except:
print("Error revoking Django credentials")
raise
# Outside the try/except so it can be run again if there is an error
post_init(bosslet_config)
def create(session, domain):
"""Configure Vault, create the configuration, and launch it"""
keypair = aws.keypair_lookup(session)
names = AWSNames(domain)
call = ExternalCalls(session, keypair, domain)
db = {
"name": "microns_proofreader",
"user": "******",
"password": utils.generate_password(),
"port": "3306"
}
# Configure Vault and create the user data config that proofreader-web will
# use for connecting to Vault and the DB instance
# DP TODO: Remove token and use AWS-EC2 authentication with Vault
with call.vault() as vault:
proofreader_token = vault.provision("proofreader")
user_data = UserData()
user_data["vault"]["token"] = proofreader_token
user_data["system"]["fqdn"] = names.proofreader
user_data["system"]["type"] = "proofreader-web"
user_data["aws"]["db"] = names.proofreader_db
user_data["auth"]["OIDC_VERIFY_SSL"] = 'True'
user_data = str(user_data)
vault.write(const.VAULT_PROOFREAD, secret_key=str(uuid.uuid4()))
vault.write(const.VAULT_PROOFREAD_DB, **db)
config = create_config(session, domain, keypair, user_data, db)
try:
success = config.create(session)
except:
print("Error detected, revoking secrets"
) # Do we want to revoke if an exception from post_init?
with call.vault() as vault:
try:
vault.delete(const.VAULT_PROOFREAD)
vault.delete(const.VAULT_PROOFREAD_DB)
except:
print("Error revoking Django credentials")
try:
vault.revoke(proofreader_token)
except:
print("Error revoking Proofreader Server Vault access token")
raise
if not success:
raise Exception("Create Failed")
else:
post_init(session, domain)
def create(session, domain):
"""Configure Vault, create the configuration, and launch it"""
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
db_config = const.ENDPOINT_DB_CONFIG.copy()
db_config['password'] = utils.generate_password()
with call.vault() as vault:
vault.write(const.VAULT_ENDPOINT, secret_key = str(uuid.uuid4()))
vault.write(const.VAULT_ENDPOINT_DB, **db_config)
dns = names.public_dns("api")
uri = "https://{}".format(dns)
vault.update(const.VAULT_ENDPOINT_AUTH, public_uri = uri)
config = create_config(session, domain, keypair, db_config)
try:
success = config.create(session)
except:
print("Error detected, revoking secrets")
try:
with call.vault() as vault:
vault.delete(const.VAULT_ENDPOINT)
vault.delete(const.VAULT_ENDPOINT_DB)
#vault.delete(const.VAULT_ENDPOINT_AUTH) # Deleting this will bork the whole stack
except:
print("Error revoking Django credentials")
raise
if not success:
raise Exception("Create Failed")
else:
# Outside the try/except so it can be run again if there is an error
post_init(session, domain)
def post_init(bosslet_config):
session = bosslet_config.session
call = bosslet_config.call
names = bosslet_config.names
# OIDC Discovery URL
auth_domain = names.public_dns("auth")
auth_discovery_url = "https://{}/auth/realms/BOSS".format(auth_domain)
# Generate initial user accounts
username = "******"
password = utils.generate_password()
realm_username = "******"
realm_password = utils.generate_password()
# Initialize Vault
print("Waiting for Vault...")
call.check_vault(const.TIMEOUT_VAULT)
with call.vault() as vault:
print("Initializing Vault...")
try:
vault.initialize(bosslet_config.ACCOUNT_ID)
except Exception as ex:
raise BossManageError("Problem initializing Vault: {}".format(
str(ex)))
#Check and see if these secrets already exist before we overwrite them with new ones.
# Write data into Vault
# DP TODO: update checks to also verify the passwords / pull the passwords from Vault
# so we don't use a different password in Keycloak then what is stored in Vault
if not vault.read(const.VAULT_AUTH):
print("Writing {}".format(const.VAULT_AUTH))
vault.write(const.VAULT_AUTH,
password=password,
username=username,
client_id="admin-cli")
if not vault.read(const.VAULT_REALM):
print("Writing {}".format(const.VAULT_REALM))
vault.write(const.VAULT_REALM,
username=realm_username,
password=realm_password,
client_id="endpoint")
if not vault.read(const.VAULT_KEYCLOAK):
print("Updating {}".format(const.VAULT_KEYCLOAK))
vault.update(const.VAULT_KEYCLOAK,
password=password,
username=username,
client_id="admin-cli",
realm="master")
if not vault.read(const.VAULT_KEYCLOAK_DB):
print("Writing {}".format(const.VAULT_KEYCLOAK_DB))
# Values are hardcodded both here and in the add_rds
# as the values are also hardcodded in the Keycloak config
#
# These values are for use by the backup / restore process
vault.write(const.VAULT_KEYCLOAK_DB,
name="keycloak",
user="******",
password="******")
if not vault.read(const.VAULT_ENDPOINT_AUTH):
# DP TODO: Move this update call into the api config
print("Updating {}".format(const.VAULT_ENDPOINT_AUTH))
vault.update(const.VAULT_ENDPOINT_AUTH,
url=auth_discovery_url,
client_id="endpoint")
# Configure Keycloak
print("Waiting for Keycloak to bootstrap")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
#######
## DP TODO: Need to find a check so that the master user is only added once to keycloak
## Also need to guard the writes to vault with the admin password
#######
with call.ssh(names.auth.dns) as ssh:
print("Creating initial Keycloak admin user")
ssh("/srv/keycloak/bin/add-user.sh -r master -u {} -p {}".format(
username, password))
print("Restarting Keycloak")
ssh("sudo service keycloak stop")
time.sleep(2)
ssh(
"sudo killall java"
) # the daemon command used by the keycloak service doesn't play well with standalone.sh
# make sure the process is actually killed
time.sleep(3)
ssh("sudo service keycloak start")
print("Waiting for Keycloak to restart")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
with call.tunnel(names.auth.dns, 8080) as port:
URL = "http://localhost:{}".format(
port) # TODO move out of tunnel and use public address
with KeyCloakClient(URL, username, password) as kc:
print("Opening realm file at '{}'".format(const.KEYCLOAK_REALM))
with open(const.KEYCLOAK_REALM, "r") as fh:
realm = json.load(fh)
try:
realm["users"][0]["username"] = realm_username
realm["users"][0]["credentials"][0]["value"] = realm_password
except:
print(
"Could not set realm admin's username or password, not creating user"
)
if "users" in realm:
del realm["users"]
print("Uploading BOSS.realm configuration")
kc.create_realm(realm)
cursor = db.conn.execute("""
INSERT INTO Users
(username, email, hash_scheme, passhash)
VALUES
(?, ?, ?, ?)
""", (username, email, "sha256_crypt", passlib.hash.sha256_crypt.encrypt(password))
)
db.conn.commit()
# TODO potential race condition if tornado is multithreaded
token = create_token(cursor.lastrowid)
self.write(token)
global FAIL_PASSHASH
# Create a hash to verify against if non existant user
FAIL_PASSHASH = passlib.hash.sha256_crypt.encrypt(generate_password(64))
class Login(tornado.web.RequestHandler):
SUPPORTED_METHODS = ('POST')
def post(self):
global FAIL_PASSHASH
username = self.get_argument("username", default=None, strip=False)
password = self.get_argument("password", default=None, strip=False)
cursor = db.conn.execute("""
SELECT
id,
hash_scheme,
passhash
FROM
def post_init(session, domain, startup_wait=False):
# Keypair is needed by ExternalCalls
global keypair
if keypair is None:
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Figure out the external domain name of the auth server(s), matching the SSL cert
auth_domain = names.public_dns("auth")
# OIDC Discovery URL
auth_discovery_url = "https://{}/auth/realms/BOSS".format(auth_domain)
# Configure external DNS
auth_elb = aws.elb_public_lookup(session, names.auth)
aws.set_domain_to_dns_name(session, auth_domain, auth_elb, aws.get_hosted_zone(session))
# Generate initial user accounts
username = "******"
password = utils.generate_password()
realm_username = "******"
realm_password = utils.generate_password()
# Initialize Vault
print("Waiting for Vault...")
call.check_vault(const.TIMEOUT_VAULT) # Expecting this to also check Consul
with call.vault() as vault:
print("Initializing Vault...")
try:
vault.initialize()
except Exception as ex:
print(ex)
print("Could not initialize Vault")
print("Call: {}".format(utils.get_command("post-init")))
print("Before launching other stacks")
return
#Check and see if these secrets already exist before we overwrite them with new ones.
# Write data into Vault
if not vault.read(const.VAULT_AUTH):
print("Writing {}".format(const.VAULT_AUTH))
vault.write(const.VAULT_AUTH, password = password, username = username, client_id = "admin-cli")
if not vault.read(const.VAULT_REALM):
print("Writing {}".format(const.VAULT_REALM))
vault.write(const.VAULT_REALM, username = realm_username, password = realm_password, client_id = "endpoint")
if not vault.read(const.VAULT_KEYCLOAK):
print("Updating {}".format(const.VAULT_KEYCLOAK))
vault.update(const.VAULT_KEYCLOAK, password = password, username = username, client_id = "admin-cli", realm = "master")
if not vault.read(const.VAULT_ENDPOINT_AUTH):
# DP TODO: Move this update call into the api config
print("Updating {}".format(const.VAULT_ENDPOINT_AUTH))
vault.update(const.VAULT_ENDPOINT_AUTH, url = auth_discovery_url, client_id = "endpoint")
if not vault.read(const.VAULT_PROOFREAD_AUTH):
# DP TODO: Move this update call into the proofreader config
print("Updating {}".format(const.VAULT_PROOFREAD_AUTH))
vault.update(const.VAULT_PROOFREAD_AUTH, url = auth_discovery_url, client_id = "endpoint")
# Configure Keycloak
print("Waiting for Keycloak to bootstrap")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
#######
## DP TODO: Need to find a check so that the master user is only added once to keycloak
## Also need to guard the writes to vault with the admin password
#######
with call.ssh(names.auth) as ssh:
print("Creating initial Keycloak admin user")
ssh("/srv/keycloak/bin/add-user.sh -r master -u {} -p {}".format(username, password))
print("Restarting Keycloak")
ssh("sudo service keycloak stop")
time.sleep(2)
ssh("sudo killall java") # the daemon command used by the keycloak service doesn't play well with standalone.sh
# make sure the process is actually killed
time.sleep(3)
ssh("sudo service keycloak start")
print("Waiting for Keycloak to restart")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
with call.tunnel(names.auth, 8080) as port:
URL = "http://localhost:{}".format(port) # TODO move out of tunnel and use public address
with KeyCloakClient(URL, username, password) as kc:
print("Opening realm file at '{}'".format(const.KEYCLOAK_REALM))
with open(const.KEYCLOAK_REALM, "r") as fh:
realm = json.load(fh)
try:
realm["users"][0]["username"] = realm_username
realm["users"][0]["credentials"][0]["value"] = realm_password
except:
print("Could not set realm admin's username or password, not creating user")
if "users" in realm:
del realm["users"]
print("Uploading BOSS.realm configuration")
kc.create_realm(realm)
# Tell Scalyr to get CloudWatch metrics for these instances.
instances = [ names.vault ]
scalyr.add_instances_to_scalyr(session, const.REGION, instances)
def post_init(bosslet_config):
session = bosslet_config.session
call = bosslet_config.call
names = bosslet_config.names
# OIDC Discovery URL
auth_domain = names.public_dns("auth")
auth_discovery_url = "https://{}/auth/realms/BOSS".format(auth_domain)
# Generate initial user accounts
username = "******"
password = utils.generate_password()
realm_username = "******"
realm_password = utils.generate_password()
# Initialize Vault
print("Waiting for Vault...")
call.check_vault(const.TIMEOUT_VAULT)
with call.vault() as vault:
print("Initializing Vault...")
try:
vault.initialize(bosslet_config.ACCOUNT_ID)
except Exception as ex:
raise BossManageError("Problem initializing Vault: {}".format(
str(ex)))
#Check and see if these secrets already exist before we overwrite them with new ones.
# Write data into Vault
auth_data = vault.read(const.VAULT_AUTH)
if not auth_data or 'password' not in auth_data:
print("Writing {}".format(const.VAULT_AUTH))
vault.write(const.VAULT_AUTH,
password=password,
username=username,
client_id="admin-cli")
else:
password = auth_data['password']
keycloak_data = vault.read(const.VAULT_KEYCLOAK)
if not keycloak_data or 'password' not in keycloak_data:
print("Updating {}".format(const.VAULT_KEYCLOAK))
vault.update(const.VAULT_KEYCLOAK,
password=password,
username=username,
client_id="admin-cli",
realm="master")
else:
if password != keycloak_data['password']:
# If there is no password match, then re-write the password for the admin-cli so that they match.
password = keycloak_data['password']
print("Re-writing {}".format(const.VAULT_KEYCLOAK))
vault.write(const.VAULT_KEYCLOAK,
password=password,
username=username,
client_id="admin-cli")
realm_data = vault.read(const.VAULT_REALM)
if not realm_data or 'password' not in realm_data:
print("Writing {}".format(const.VAULT_REALM))
vault.write(const.VAULT_REALM,
username=realm_username,
password=realm_password,
client_id="endpoint")
else:
realm_password = realm_data['password']
if not vault.read(const.VAULT_KEYCLOAK_DB):
print("Writing {}".format(const.VAULT_KEYCLOAK_DB))
# Values are hardcodded both here and in the add_rds
# as the values are also hardcodded in the Keycloak config
#
# These values are for use by the backup / restore process
vault.write(const.VAULT_KEYCLOAK_DB,
name="keycloak",
user="******",
password="******")
if not vault.read(const.VAULT_ENDPOINT_AUTH):
# DP TODO: Move this update call into the api config
print("Updating {}".format(const.VAULT_ENDPOINT_AUTH))
vault.update(const.VAULT_ENDPOINT_AUTH,
url=auth_discovery_url,
client_id="endpoint")
# Configure Keycloak
print("Waiting for Keycloak to bootstrap")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
print("Finished waiting for Keycloak to bootstrap")
#######
## DP TODO: Need to find a check so that the master user is only added once to keycloak
## Also need to guard the writes to vault with the admin password
#######
with call.ssh(names.auth.dns) as ssh:
print("Creating initial Keycloak admin user")
# This fails if the user already exists, but execution will continue.
ssh("/srv/keycloak/bin/add-user-keycloak.sh -r master -u {} -p {}".
format(username, password))
time.sleep(10)
ssh("/srv/keycloak/bin/jboss-cli.sh --connect reload")
time.sleep(3)
print("Waiting for Keycloak to restart")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
with call.tunnel(names.auth.dns, 8080) as port:
URL = "http://localhost:{}".format(
port) # TODO move out of tunnel and use public address
with KeyCloakClient(URL, username, password) as kc:
print("Opening realm file at '{}'".format(const.KEYCLOAK_REALM))
with open(const.KEYCLOAK_REALM, "r") as fh:
realm = json.load(fh)
try:
realm["users"][0]["username"] = realm_username
realm["users"][0]["credentials"][0]["value"] = realm_password
except:
print(
"Could not set realm admin's username or password, not creating user"
)
if "users" in realm:
del realm["users"]
# This will fail if the realm already exists, but that's ok.
print("Uploading BOSS.realm configuration")
try:
kc.create_realm(realm)
except Exception as ex:
print(
'Failed to upload Boss.realm config. Does it already exist? Error message: {}'
.format(ex))