def update(session, domain):
keypair = aws.keypair_lookup(session)
names = AWSNames(domain)
config = create_config(session, domain, keypair)
success = config.update(session)
return success
def update(session, domain):
keypair = aws.keypair_lookup(session)
names = AWSNames(domain)
config = create_config(session, domain, keypair)
success = config.update(session)
return success
def create(session, domain):
"""Configure Vault, create the configuration, and launch it"""
keypair = aws.keypair_lookup(session)
names = AWSNames(domain)
call = ExternalCalls(session, keypair, domain)
db = {
"name": "microns_proofreader",
"user": "******",
"password": utils.generate_password(),
"port": "3306"
}
# Configure Vault and create the user data config that proofreader-web will
# use for connecting to Vault and the DB instance
# DP TODO: Remove token and use AWS-EC2 authentication with Vault
with call.vault() as vault:
proofreader_token = vault.provision("proofreader")
user_data = UserData()
user_data["vault"]["token"] = proofreader_token
user_data["system"]["fqdn"] = names.proofreader
user_data["system"]["type"] = "proofreader-web"
user_data["aws"]["db"] = names.proofreader_db
user_data["auth"]["OIDC_VERIFY_SSL"] = 'True'
user_data = str(user_data)
vault.write(const.VAULT_PROOFREAD, secret_key=str(uuid.uuid4()))
vault.write(const.VAULT_PROOFREAD_DB, **db)
config = create_config(session, domain, keypair, user_data, db)
try:
success = config.create(session)
except:
print("Error detected, revoking secrets"
) # Do we want to revoke if an exception from post_init?
with call.vault() as vault:
try:
vault.delete(const.VAULT_PROOFREAD)
vault.delete(const.VAULT_PROOFREAD_DB)
except:
print("Error revoking Django credentials")
try:
vault.revoke(proofreader_token)
except:
print("Error revoking Proofreader Server Vault access token")
raise
if not success:
raise Exception("Create Failed")
else:
post_init(session, domain)
def create(session, domain):
"""Create the configuration, and launch it"""
names = AWSNames(domain)
user_data = UserData()
user_data["system"]["fqdn"] = names.cache_manager
user_data["system"]["type"] = "cachemanager"
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["s3-flush-queue"] = aws.sqs_lookup_url(
session, names.s3flush_queue)
user_data["aws"]["s3-flush-deadletter-queue"] = aws.sqs_lookup_url(
session, names.deadletter_queue)
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
#user_data["aws"]["sns-write-locked"] = str(Ref('WriteLock'))
mailing_list_arn = aws.sns_topic_lookup(session,
const.PRODUCTION_MAILING_LIST)
if mailing_list_arn is None:
msg = "MailingList {} needs to be created before running config".format(
const.PRODUCTION_MAILING_LIST)
raise Exception(msg)
user_data["aws"]["sns-write-locked"] = mailing_list_arn
user_data["lambda"]["flush_function"] = names.multi_lambda
user_data["lambda"]["page_in_function"] = names.multi_lambda
keypair = aws.keypair_lookup(session)
try:
pre_init(session, domain)
config = create_config(session, domain, keypair, user_data)
success = config.create(session)
if not success:
raise Exception("Create Failed")
else:
post_init(session, domain)
except:
# DP NOTE: This will catch errors from pre_init, create, and post_init
print("Error detected")
raise
def generate(session, domain):
"""Create the configuration and save it to disk"""
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
with call.vault() as vault:
db_config = vault.read(const.VAULT_ENDPOINT_DB)
if db_config is None:
db_config = const.ENDPOINT_DB_CONFIG.copy()
config = create_config(session, domain, keypair, db_config)
config.generate()
def update(session, domain):
# Only in the production scenario will data be preserved over the update
if os.environ["SCENARIO"] not in ("production", "ha-development",):
print("Can only update the production and ha-development scenario")
return None
consul_update_timeout = 5 # minutes
consul_size = int(get_scenario(const.CONSUL_CLUSTER_SIZE))
min_time = consul_update_timeout * consul_size
max_time = min_time + 5 # add some time to allow the CF update to happen
print("Update command will take {} - {} minutes to finish".format(min_time, max_time))
print("Stack will be available during that time")
resp = input("Update? [N/y] ")
if len(resp) == 0 or resp[0] not in ('y', 'Y'):
print("Canceled")
return
config = create_config(session, domain)
success = config.update(session)
if success:
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Unseal Vault first, so the rest of the system can continue working
print("Waiting for Vault...")
if not call.check_vault(90, exception=False):
print("Could not contact Vault, check networking and run the following command")
print("python3 bastion.py bastion.521.boss vault.521.boss vault-unseal")
return
with call.vault() as vault:
vault.unseal()
print("Stack should be ready for use")
print("Starting to cycle consul cluster instances")
# DP NOTE: Cycling the instances is done manually (outside of CF)
# so that Vault can be unsealed first, else the whole stacks
# would not be usable until all consul instance were restarted
with ThreadPoolExecutor(max_workers=3) as tpe:
# Need time for the ASG to detect the terminated instance,
# launch the new instance, and have the instance cluster
tpe.submit(aws.asg_restart,
session,
names.consul,
consul_update_timeout * 60)
return success
def update(session, domain):
keypair = aws.keypair_lookup(session)
config = create_config(session, domain, keypair)
success = config.update(session)
resp = input('Rebuild multilambda: [Y/n]:')
if len(resp) == 0 or (len(resp) > 0 and resp[0] in ('Y', 'y')):
pre_init(session, domain)
bucket = aws.get_lambda_s3_bucket(session)
update_lambda_code(session, domain, bucket)
post_init(session, domain)
return success
def create(session, domain):
"""Create the configuration, and launch it"""
names = AWSNames(domain)
user_data = UserData()
user_data["system"]["fqdn"] = names.cache_manager
user_data["system"]["type"] = "cachemanager"
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["s3-flush-queue"] = aws.sqs_lookup_url(session, names.s3flush_queue)
user_data["aws"]["s3-flush-deadletter-queue"] = aws.sqs_lookup_url(session, names.deadletter_queue)
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
#user_data["aws"]["sns-write-locked"] = str(Ref('WriteLock'))
mailing_list_arn = aws.sns_topic_lookup(session, const.PRODUCTION_MAILING_LIST)
if mailing_list_arn is None:
msg = "MailingList {} needs to be created before running config".format(const.PRODUCTION_MAILING_LIST)
raise Exception(msg)
user_data["aws"]["sns-write-locked"] = mailing_list_arn
user_data["lambda"]["flush_function"] = names.multi_lambda
user_data["lambda"]["page_in_function"] = names.multi_lambda
keypair = aws.keypair_lookup(session)
try:
pre_init(session, domain)
config = create_config(session, domain, keypair, user_data)
success = config.create(session)
if not success:
raise Exception("Create Failed")
else:
post_init(session, domain)
except:
# DP NOTE: This will catch errors from pre_init, create, and post_init
print("Error detected")
raise
def create(session, domain):
"""Create the configuration, and launch it"""
keypair = aws.keypair_lookup(session)
try:
pre_init(session, domain)
config = create_config(session, domain)
success = config.create(session)
if not success:
raise Exception("Create Failed")
else:
post_init(session, domain)
except:
# DP NOTE: This will catch errors from pre_init, create, and post_init
print("Error detected")
raise
def create(session, domain):
"""Create the configuration, and launch it"""
keypair = aws.keypair_lookup(session)
try:
pre_init(session, domain)
config = create_config(session, domain)
success = config.create(session)
if not success:
raise Exception("Create Failed")
else:
post_init(session, domain)
except:
# DP NOTE: This will catch errors from pre_init, create, and post_init
print("Error detected")
raise
def create(session, domain):
"""Configure Vault, create the configuration, and launch it"""
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
db_config = const.ENDPOINT_DB_CONFIG.copy()
db_config['password'] = utils.generate_password()
with call.vault() as vault:
vault.write(const.VAULT_ENDPOINT, secret_key = str(uuid.uuid4()))
vault.write(const.VAULT_ENDPOINT_DB, **db_config)
dns = names.public_dns("api")
uri = "https://{}".format(dns)
vault.update(const.VAULT_ENDPOINT_AUTH, public_uri = uri)
config = create_config(session, domain, keypair, db_config)
try:
success = config.create(session)
except:
print("Error detected, revoking secrets")
try:
with call.vault() as vault:
vault.delete(const.VAULT_ENDPOINT)
vault.delete(const.VAULT_ENDPOINT_DB)
#vault.delete(const.VAULT_ENDPOINT_AUTH) # Deleting this will bork the whole stack
except:
print("Error revoking Django credentials")
raise
if not success:
raise Exception("Create Failed")
else:
# Outside the try/except so it can be run again if there is an error
post_init(session, domain)
def post_init(session, domain):
# Keypair is needed by ExternalCalls
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Configure external DNS
# DP ???: Can this be moved into the CloudFormation template?
dns = names.public_dns("api")
dns_elb = aws.elb_public_lookup(session, names.endpoint_elb)
aws.set_domain_to_dns_name(session, dns, dns_elb, aws.get_hosted_zone(session))
# Write data into Vault
# DP TODO: Move into the pre-launch Vault writes, so it is available when the
# machines initially start
with call.vault() as vault:
uri = "https://{}".format(dns)
#vault.update(const.VAULT_ENDPOINT_AUTH, public_uri = uri)
creds = vault.read("secret/auth")
bossadmin = vault.read("secret/auth/realm")
auth_uri = vault.read("secret/endpoint/auth")['url']
# Verify Keycloak is accessible
print("Checking for Keycloak availability")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
# Add the API servers to the list of OIDC valid redirects
with call.tunnel(names.auth, 8080) as auth_port:
print("Update KeyCloak Client Info")
auth_url = "http://localhost:{}".format(auth_port)
with KeyCloakClient(auth_url, **creds) as kc:
# DP TODO: make add_redirect_uri able to work multiple times without issue
kc.add_redirect_uri("BOSS","endpoint", uri + "/*")
# Get the boss admin's bearer token
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
params = {
'grant_type': 'password',
'client_id': bossadmin['client_id'],
'username': bossadmin['username'],
'password': bossadmin['password'],
}
auth_uri += '/protocol/openid-connect/token'
req = Request(auth_uri,
headers = headers,
data = urlencode(params).encode('utf-8'))
resp = json.loads(urlopen(req).read().decode('utf-8'))
# Make an API call that will log the boss admin into the endpoint
call.check_url(uri + '/ping', 60)
headers = {
'Authorization': 'Bearer {}'.format(resp['access_token']),
}
api_uri = uri + '/latest/collection'
req = Request(api_uri, headers = headers)
resp = json.loads(urlopen(req).read().decode('utf-8'))
print("Collections: {}".format(resp))
# Tell Scalyr to get CloudWatch metrics for these instances.
instances = [names.endpoint]
scalyr.add_instances_to_scalyr(
session, const.REGION, instances)
def create_config(session, domain):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('core', domain, const.REGION)
names = AWSNames(domain)
global keypair
keypair = aws.keypair_lookup(session)
config.add_vpc()
# Create the internal and external subnets
config.add_subnet('InternalSubnet', names.subnet('internal'))
config.add_subnet('ExternalSubnet', names.subnet('external'))
internal_subnets, external_subnets = config.add_all_azs(session)
# it seems that both Lambdas and ASGs needs lambda_compatible_only subnets.
internal_subnets_lambda, external_subnets_lambda = config.add_all_azs(session, lambda_compatible_only=True)
config.add_ec2_instance("Bastion",
names.bastion,
aws.ami_lookup(session, const.BASTION_AMI),
keypair,
subnet = Ref("ExternalSubnet"),
public_ip = True,
user_data = const.BASTION_USER_DATA,
security_groups = [Ref("InternalSecurityGroup"), Ref("BastionSecurityGroup")],
depends_on = "AttachInternetGateway")
user_data = UserData()
user_data["system"]["fqdn"] = names.consul
user_data["system"]["type"] = "consul"
user_data["consul"]["cluster"] = str(get_scenario(const.CONSUL_CLUSTER_SIZE))
config.add_autoscale_group("Consul",
names.consul,
aws.ami_lookup(session, "consul.boss"),
keypair,
subnets = internal_subnets_lambda,
security_groups = [Ref("InternalSecurityGroup")],
user_data = str(user_data),
min = const.CONSUL_CLUSTER_SIZE,
max = const.CONSUL_CLUSTER_SIZE,
notifications = Ref("DNSSNS"),
role = aws.instance_profile_arn_lookup(session, 'consul'),
support_update = False, # Update will restart the instances manually
depends_on = ["DNSLambda", "DNSSNS", "DNSLambdaExecute"])
user_data = UserData()
user_data["system"]["fqdn"] = names.vault
user_data["system"]["type"] = "vault"
config.add_autoscale_group("Vault",
names.vault,
aws.ami_lookup(session, "vault.boss"),
keypair,
subnets = internal_subnets_lambda,
security_groups = [Ref("InternalSecurityGroup")],
user_data = str(user_data),
min = const.VAULT_CLUSTER_SIZE,
max = const.VAULT_CLUSTER_SIZE,
notifications = Ref("DNSSNS"),
depends_on = ["Consul", "DNSLambda", "DNSSNS", "DNSLambdaExecute"])
user_data = UserData()
user_data["system"]["fqdn"] = names.auth
user_data["system"]["type"] = "auth"
deps = ["AuthSecurityGroup",
"AttachInternetGateway",
"DNSLambda",
"DNSSNS",
"DNSLambdaExecute"]
SCENARIO = os.environ["SCENARIO"]
USE_DB = SCENARIO in ("production", "ha-development",)
# Problem: If development scenario uses a local DB. If the auth server crashes
# and is auto restarted by the autoscale group then the new auth server
# will not have any of the previous configuration, because the old DB
# was lost. Using an RDS for development fixes this at the cost of having
# the core config taking longer to launch.
if USE_DB:
deps.append("AuthDB")
user_data["aws"]["db"] = "keycloak" # flag for init script for which config to use
cert = aws.cert_arn_lookup(session, names.public_dns('auth'))
create_asg_elb(config,
"Auth",
names.auth,
aws.ami_lookup(session, "auth.boss"),
keypair,
str(user_data),
const.AUTH_CLUSTER_SIZE,
internal_subnets_lambda,
external_subnets_lambda,
[("443", "8080", "HTTPS", cert)],
"HTTP:8080/index.html",
sgs = [Ref("AuthSecurityGroup")],
type_=const.AUTH_TYPE,
depends_on=deps)
if USE_DB:
config.add_rds_db("AuthDB",
names.auth_db,
"3306",
"keycloak",
"keycloak",
"keycloak",
internal_subnets,
type_ = "db.t2.micro",
security_groups = [Ref("InternalSecurityGroup")])
config.add_lambda("DNSLambda",
names.dns,
aws.role_arn_lookup(session, 'UpdateRoute53'),
const.DNS_LAMBDA,
handler="index.handler",
timeout=10,
depends_on="DNSZone")
config.add_lambda_permission("DNSLambdaExecute", Ref("DNSLambda"))
config.add_sns_topic("DNSSNS",
names.dns,
names.dns,
[("lambda", Arn("DNSLambda"))])
config.add_security_group("InternalSecurityGroup",
names.internal,
[("-1", "-1", "-1", "10.0.0.0/8")])
# Allow SSH access to bastion from anywhere
config.add_security_group("BastionSecurityGroup",
names.ssh,
[("tcp", "22", "22", const.INCOMING_SUBNET)])
config.add_security_group("AuthSecurityGroup",
#names.https, DP XXX: hack until we can get production updated correctly
names.auth,
[("tcp", "443", "443", "0.0.0.0/0")])
# Create the internal route table to route traffic to the NAT Bastion
all_internal_subnets = internal_subnets.copy()
all_internal_subnets.append(Ref("InternalSubnet"))
config.add_route_table("InternalRouteTable",
names.internal,
subnets = all_internal_subnets)
config.add_route_table_route("InternalNatRoute",
Ref("InternalRouteTable"),
nat = Ref("NAT"),
depends_on = "NAT")
# Create the internet gateway and internet router
all_external_subnets = external_subnets.copy()
all_external_subnets.append(Ref("ExternalSubnet"))
config.add_route_table("InternetRouteTable",
names.internet,
subnets = all_external_subnets)
config.add_route_table_route("InternetRoute",
Ref("InternetRouteTable"),
gateway = Ref("InternetGateway"),
depends_on = "AttachInternetGateway")
config.add_internet_gateway("InternetGateway", names.internet)
config.add_endpoint("S3Endpoint", "s3", [Ref("InternalRouteTable"), Ref('InternetRouteTable')])
config.add_endpoint("DynamoDBEndpoint", "dynamodb", [Ref("InternalRouteTable"), Ref('InternetRouteTable')])
config.add_nat("NAT", Ref("ExternalSubnet"), depends_on="AttachInternetGateway")
return config
def generate(session, domain):
"""Create the configuration and save it to disk"""
keypair = aws.keypair_lookup(session)
config = create_config(session, domain, keypair)
config.generate()
def generate(session, domain):
"""Create the configuration and save it to disk"""
keypair = aws.keypair_lookup(session)
config = create_config(session, domain, keypair)
config.generate()
def post_init(session, domain):
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Get Keycloak admin account credentials
with call.vault() as vault:
creds = vault.read("secret/auth")
# Get Keycloak public address
auth_url = "https://{}/".format(names.public_dns("auth"))
# Write data into Vault
dns = aws.instance_public_lookup(session, names.proofreader)
uri = "http://{}".format(dns)
vault.update(const.VAULT_PROOFREAD_AUTH, public_uri=uri)
# Verify Keycloak is accessible
print("Checking for Keycloak availability")
call.check_url(auth_url + "auth/", const.TIMEOUT_KEYCLOAK)
with KeyCloakClient(auth_url, **creds) as kc:
print("Configuring KeyCloak")
kc.append_list_properties("BOSS", "endpoint", {
"redirectUris": uri + "/*",
"webOrigins": uri
})
print("Generating keycloak.json")
client_install = kc.get_client_installation_url("BOSS", "endpoint")
# Verify Django install doesn't have any issues
print("Checking Django status")
call.check_django("proofreader-web",
"/srv/www/app/proofreader_apis/manage.py")
print("Initializing Django")
with call.ssh(names.proofreader) as ssh:
def django(cmd):
ret = ssh(
"sudo python3 /srv/www/app/proofreader_apis/manage.py " +
cmd)
if ret != 0:
print("Django command '{}' did not sucessfully execute".
format(cmd))
django("makemigrations"
) # will hang if it cannot contact the auth server
django("makemigrations common")
django("makemigrations bossoidc")
django("migrate")
django("collectstatic --no-input")
ssh("sudo service uwsgi-emperor reload")
ssh("sudo service nginx restart")
# NOTE: This will put a valid bearer token in the bash history until the history is cleared
ssh("sudo wget --header=\"{}\" --no-check-certificate {} -O /srv/www/html/keycloak.json"
.format(client_install["headers"], client_install["url"]))
# clear the history
ssh("history -c")
def create_config(session, domain):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('activities', domain, const.REGION)
names = AWSNames(domain)
global keypair
keypair = aws.keypair_lookup(session)
vpc_id = config.find_vpc(session)
sgs = aws.sg_lookup_all(session, vpc_id)
internal_subnets, _ = config.find_all_availability_zones(session)
internal_subnets_lambda, _ = config.find_all_availability_zones(
session, lambda_compatible_only=True)
topic_arn = aws.sns_topic_lookup(session, "ProductionMicronsMailingList")
event_data = {
"lambda-name": "delete_lambda",
"db": names.endpoint_db,
"meta-db": names.meta,
"s3-index-table": names.s3_index,
"id-index-table": names.id_index,
"id-count-table": names.id_count_index,
"cuboid_bucket": names.cuboid_bucket,
"delete_bucket": names.delete_bucket,
"topic-arn": topic_arn,
"query-deletes-sfn-name": names.query_deletes,
"delete-sfn-name": names.delete_cuboid,
"delete-exp-sfn-name": names.delete_experiment,
"delete-coord-frame-sfn-name": names.delete_coord_frame,
"delete-coll-sfn-name": names.delete_collection
}
role_arn = aws.role_arn_lookup(session, "events_for_delete_lambda")
multi_lambda = names.multi_lambda
lambda_arn = aws.lambda_arn_lookup(session, multi_lambda)
target_list = [{
"Arn": lambda_arn,
"Id": multi_lambda,
"Input": json.dumps(event_data)
}]
schedule_expression = "cron(1 6-11/1 ? * TUE-FRI *)"
#schedule_expression = "cron(0/2 * * * ? *)" # testing fire every two minutes
config.add_event_rule("DeleteEventRule",
names.delete_event_rule,
role_arn=role_arn,
schedule_expression=schedule_expression,
target_list=target_list,
description=None)
# Events have to be given permission to run lambda.
config.add_lambda_permission('DeleteRulePerm',
multi_lambda,
principal='events.amazonaws.com',
source=Arn('DeleteEventRule'))
user_data = UserData()
user_data["system"]["fqdn"] = names.activities
user_data["system"]["type"] = "activities"
user_data["aws"]["db"] = names.endpoint_db
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["meta-db"] = names.meta
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["tile_bucket"] = names.tile_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["tile-index-table"] = names.tile_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
config.add_autoscale_group("Activities",
names.activities,
aws.ami_lookup(session, 'activities.boss'),
keypair,
subnets=internal_subnets_lambda,
type_=const.ACTIVITIES_TYPE,
security_groups=[sgs[names.internal]],
user_data=str(user_data),
role=aws.instance_profile_arn_lookup(
session, "activities"),
min=1,
max=1)
config.add_lambda("IngestLambda",
names.ingest_lambda,
aws.role_arn_lookup(session, 'IngestQueueUpload'),
const.INGEST_LAMBDA,
handler="index.handler",
timeout=60 * 5)
config.add_lambda_permission("IngestLambdaExecute", Ref("IngestLambda"))
return config
def post_init(session, domain, startup_wait=False):
# Keypair is needed by ExternalCalls
global keypair
if keypair is None:
keypair = aws.keypair_lookup(session)
call = ExternalCalls(session, keypair, domain)
names = AWSNames(domain)
# Figure out the external domain name of the auth server(s), matching the SSL cert
auth_domain = names.public_dns("auth")
# OIDC Discovery URL
auth_discovery_url = "https://{}/auth/realms/BOSS".format(auth_domain)
# Configure external DNS
auth_elb = aws.elb_public_lookup(session, names.auth)
aws.set_domain_to_dns_name(session, auth_domain, auth_elb, aws.get_hosted_zone(session))
# Generate initial user accounts
username = "******"
password = utils.generate_password()
realm_username = "******"
realm_password = utils.generate_password()
# Initialize Vault
print("Waiting for Vault...")
call.check_vault(const.TIMEOUT_VAULT) # Expecting this to also check Consul
with call.vault() as vault:
print("Initializing Vault...")
try:
vault.initialize()
except Exception as ex:
print(ex)
print("Could not initialize Vault")
print("Call: {}".format(utils.get_command("post-init")))
print("Before launching other stacks")
return
#Check and see if these secrets already exist before we overwrite them with new ones.
# Write data into Vault
if not vault.read(const.VAULT_AUTH):
print("Writing {}".format(const.VAULT_AUTH))
vault.write(const.VAULT_AUTH, password = password, username = username, client_id = "admin-cli")
if not vault.read(const.VAULT_REALM):
print("Writing {}".format(const.VAULT_REALM))
vault.write(const.VAULT_REALM, username = realm_username, password = realm_password, client_id = "endpoint")
if not vault.read(const.VAULT_KEYCLOAK):
print("Updating {}".format(const.VAULT_KEYCLOAK))
vault.update(const.VAULT_KEYCLOAK, password = password, username = username, client_id = "admin-cli", realm = "master")
if not vault.read(const.VAULT_ENDPOINT_AUTH):
# DP TODO: Move this update call into the api config
print("Updating {}".format(const.VAULT_ENDPOINT_AUTH))
vault.update(const.VAULT_ENDPOINT_AUTH, url = auth_discovery_url, client_id = "endpoint")
if not vault.read(const.VAULT_PROOFREAD_AUTH):
# DP TODO: Move this update call into the proofreader config
print("Updating {}".format(const.VAULT_PROOFREAD_AUTH))
vault.update(const.VAULT_PROOFREAD_AUTH, url = auth_discovery_url, client_id = "endpoint")
# Configure Keycloak
print("Waiting for Keycloak to bootstrap")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
#######
## DP TODO: Need to find a check so that the master user is only added once to keycloak
## Also need to guard the writes to vault with the admin password
#######
with call.ssh(names.auth) as ssh:
print("Creating initial Keycloak admin user")
ssh("/srv/keycloak/bin/add-user.sh -r master -u {} -p {}".format(username, password))
print("Restarting Keycloak")
ssh("sudo service keycloak stop")
time.sleep(2)
ssh("sudo killall java") # the daemon command used by the keycloak service doesn't play well with standalone.sh
# make sure the process is actually killed
time.sleep(3)
ssh("sudo service keycloak start")
print("Waiting for Keycloak to restart")
call.check_keycloak(const.TIMEOUT_KEYCLOAK)
with call.tunnel(names.auth, 8080) as port:
URL = "http://localhost:{}".format(port) # TODO move out of tunnel and use public address
with KeyCloakClient(URL, username, password) as kc:
print("Opening realm file at '{}'".format(const.KEYCLOAK_REALM))
with open(const.KEYCLOAK_REALM, "r") as fh:
realm = json.load(fh)
try:
realm["users"][0]["username"] = realm_username
realm["users"][0]["credentials"][0]["value"] = realm_password
except:
print("Could not set realm admin's username or password, not creating user")
if "users" in realm:
del realm["users"]
print("Uploading BOSS.realm configuration")
kc.create_realm(realm)
# Tell Scalyr to get CloudWatch metrics for these instances.
instances = [ names.vault ]
scalyr.add_instances_to_scalyr(session, const.REGION, instances)
def update(session, domain):
keypair = aws.keypair_lookup(session)
names = AWSNames(domain)
call = ExternalCalls(session, keypair, domain)
with call.vault() as vault:
db_config = vault.read(const.VAULT_ENDPOINT_DB)
'''
try:
import MySQLdb as mysql
except:
print("Cannot save data before migrating schema, exiting...")
return
print("Saving time step data")
print("Tunneling")
with call.tunnel(names.endpoint_db, db_config['port'], type_='rds') as local_port:
print("Connecting to MySQL")
db = mysql.connect(host = '127.0.0.1',
port = local_port,
user = db_config['user'],
passwd = db_config['password'],
db = db_config['name'])
cur = db.cursor()
try:
sql = "DROP TABLE temp_time_step"
cur.execute(sql)
except Exception as e:
#print(e)
pass # Table doesn't exist
print("Saving Data")
sql = """CREATE TABLE temp_time_step(time_step_unit VARCHAR(100), exp_id INT(11), coord_frame_id INT(11), time_step INT(11))
SELECT coordinate_frame.time_step_unit, experiment.id as exp_id, coord_frame_id, time_step
FROM experiment, coordinate_frame
WHERE coordinate_frame.id = experiment.coord_frame_id """
cur.execute(sql)
sql = "SELECT * FROM temp_time_step"
cur.execute(sql)
rows = cur.fetchall()
print("Saved {} rows of data".format(len(rows)))
#for r in rows:
# print(r)
cur.close()
db.close()
'''
config = create_config(session, domain, keypair, db_config)
success = config.update(session)
'''
print("Restoring time step data")
print("Tunneling")
with call.tunnel(names.endpoint_db, db_config['port'], type_='rds') as local_port:
print("Connecting to MySQL")
db = mysql.connect(host = '127.0.0.1',
port = local_port,
user = db_config['user'],
passwd = db_config['password'],
db = db_config['name'])
cur = db.cursor()
if success:
sql = """UPDATE experiment, temp_time_step
SET experiment.time_step_unit = temp_time_step.time_step_unit,
experiment.time_step = temp_time_step.time_step
WHERE experiment.id = temp_time_step.exp_id AND
experiment.coord_frame_id = temp_time_step.coord_frame_id"""
cur.execute(sql)
db.commit()
sql = "SELECT time_step_unit, id, coord_frame_id, time_step FROM experiment"
cur.execute(sql)
rows = cur.fetchall()
print("Migrated {} rows of data".format(len(rows)))
#for r in rows:
# print(r)
else:
if success is None:
print("Update canceled, not migrating data")
else:
print("Error during update, not migrating data")
print("Deleting temp table")
sql = "DROP TABLE temp_time_step"
cur.execute(sql)
cur.close()
db.close()
'''
return success
def create_config(session, domain):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('activities', domain, const.REGION)
names = AWSNames(domain)
global keypair
keypair = aws.keypair_lookup(session)
vpc_id = config.find_vpc(session)
sgs = aws.sg_lookup_all(session, vpc_id)
internal_subnets, _ = config.find_all_availability_zones(session)
internal_subnets_lambda, _ = config.find_all_availability_zones(session, lambda_compatible_only=True)
topic_arn = aws.sns_topic_lookup(session, "ProductionMicronsMailingList")
event_data = {
"lambda-name": "delete_lambda",
"db": names.endpoint_db,
"meta-db": names.meta,
"s3-index-table": names.s3_index,
"id-index-table": names.id_index,
"id-count-table": names.id_count_index,
"cuboid_bucket": names.cuboid_bucket,
"delete_bucket": names.delete_bucket,
"topic-arn": topic_arn,
"query-deletes-sfn-name": names.query_deletes,
"delete-sfn-name": names.delete_cuboid,
"delete-exp-sfn-name": names.delete_experiment,
"delete-coord-frame-sfn-name": names.delete_coord_frame,
"delete-coll-sfn-name": names.delete_collection
}
role_arn = aws.role_arn_lookup(session, "events_for_delete_lambda")
multi_lambda = names.multi_lambda
lambda_arn = aws.lambda_arn_lookup(session, multi_lambda)
target_list = [{
"Arn": lambda_arn,
"Id": multi_lambda,
"Input": json.dumps(event_data)
}]
schedule_expression = "cron(1 6-11/1 ? * TUE-FRI *)"
#schedule_expression = "cron(0/2 * * * ? *)" # testing fire every two minutes
config.add_event_rule("DeleteEventRule", names.delete_event_rule, role_arn=role_arn,
schedule_expression=schedule_expression, target_list=target_list, description=None)
# Events have to be given permission to run lambda.
config.add_lambda_permission('DeleteRulePerm', multi_lambda, principal='events.amazonaws.com',
source=Arn('DeleteEventRule'))
user_data = UserData()
user_data["system"]["fqdn"] = names.activities
user_data["system"]["type"] = "activities"
user_data["aws"]["db"] = names.endpoint_db
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["meta-db"] = names.meta
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["tile_bucket"] = names.tile_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["tile-index-table"] = names.tile_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
user_data["aws"]["max_task_id_suffix"] = str(const.MAX_TASK_ID_SUFFIX)
config.add_autoscale_group("Activities",
names.activities,
aws.ami_lookup(session, 'activities.boss'),
keypair,
subnets=internal_subnets_lambda,
type_=const.ACTIVITIES_TYPE,
security_groups=[sgs[names.internal]],
user_data=str(user_data),
role=aws.instance_profile_arn_lookup(session, "activities"),
min=1,
max=1)
config.add_lambda("IngestLambda",
names.ingest_lambda,
aws.role_arn_lookup(session, 'IngestQueueUpload'),
const.INGEST_LAMBDA,
handler="index.handler",
timeout=60 * 5,
memory=3008)
config.add_lambda_permission("IngestLambdaExecute", Ref("IngestLambda"))
# Downsample / Resolution Hierarchy support
lambda_role = aws.role_arn_lookup(session, "lambda_resolution_hierarchy")
config.add_lambda("DownsampleVolumeLambda",
names.downsample_volume_lambda,
lambda_role,
s3=(aws.get_lambda_s3_bucket(session),
"multilambda.{}.zip".format(domain),
"downsample_volume.handler"),
timeout=120,
memory=1024,
runtime='python3.6',
dlq = Ref('DownsampleDLQ'))
config.add_sns_topic("DownsampleDLQ",
names.downsample_dlq,
names.downsample_dlq,
[('lambda', Arn('DownsampleDLQLambda'))])
config.add_lambda('DownsampleDLQLambda',
names.downsample_dlq,
lambda_role,
const.DOWNSAMPLE_DLQ_LAMBDA,
handler='index.handler',
timeout=10)
config.add_lambda_permission('DownsampleDLQLambdaExecute',
Ref('DownsampleDLQLambda'))
return config
