def do_setup_eb_update():
print("\n", DO_SETUP_EB_UPDATE_OPEN)
files = sorted(
[f for f in os.listdir(STAGED_FILES) if f.lower().endswith(".zip")])
if not files:
print("Could not find any zip files in " + STAGED_FILES)
EXIT(1)
print("Enter the version of the codebase do you want to use:")
for i, file_name in enumerate(files):
print("[%s]: %s" % (i + 1, file_name))
print("(press CTL-C to cancel)\n")
try:
index = int(input("$ "))
except Exception:
log.error("Could not parse input.")
index = None # ide warnings
EXIT(1)
if index < 1 or index > len(files):
log.error("%s was not a valid option." % index)
EXIT(1)
# handle 1-indexing
file_name = files[index - 1]
# log.info("Processing %s..." % file_name)
time_ext = current_time_string().replace(" ", "_").replace(":", "_")
output_file_name = file_name[:-4] + "_processed_" + time_ext + ".zip"
do_zip_reduction(file_name, STAGED_FILES, output_file_name)
log.info("Done processing %s." % file_name)
log.info("The new file %s has been placed in %s" %
(output_file_name, STAGED_FILES))
print(
"You can now provide Elastic Beanstalk with %s to run an automated deployment of the new code."
% output_file_name)
EXIT(0)
def create_new_rds_instance(eb_environment_name):
db_instance_identifier = construct_db_name(eb_environment_name)
# identify whether there is already a database with this name, we don't want to
try:
_ = get_db_info(eb_environment_name)
log.error("There is already a database named %s" % eb_environment_name)
EXIT()
except DBInstanceNotFound:
pass
database_server_type = get_server_configuration_file(
eb_environment_name)['DB_SERVER_TYPE']
engine = get_most_recent_postgres_engine()
credentials = generate_valid_postgres_credentials()
log.info(
"writing database credentials to disk, database address will be added later."
)
write_rds_credentials(eb_environment_name, credentials, True)
# There is some weirdness involving security groups. It looks like there is this concept of
# non-vpc security groups, I am fairly certain that this interacts with cross-vpc, IAM based
# database access.
create_rds_security_groups(db_instance_identifier)
db_sec_grp_id = get_rds_security_groups(
db_instance_identifier)['database_sec_grp']['GroupId']
log.info("Creating RDS Postgres database named %s" %
db_instance_identifier)
rds_client = create_rds_client()
rds_instance = rds_client.create_db_instance(
# server details
DBInstanceIdentifier=db_instance_identifier,
DBInstanceClass="db." + database_server_type,
MultiAZ=False,
PubliclyAccessible=False,
Port=POSTGRES_PORT,
# attach the security group that will allow access
VpcSecurityGroupIds=[db_sec_grp_id],
#TODO: is this even relevant?
# providing the subnet is critical, not providing this value causes the db to be non-vpc
# DBSubnetGroupName='string',
# db storage
StorageType='gp2', # valid options are standard, gp2, io1
# Iops=1000, # multiple between 3 and 10 times the storage; only for use with io1.
# AllocatedStorage has weird constraints:
# General Purpose (SSD) storage (gp2): Must be an integer from 5 to 6144.
# Provisioned IOPS storage (io1): Must be an integer from 100 to 6144.
# Magnetic storage (standard): Must be an integer from 5 to 3072.
AllocatedStorage=50, # in gigabytes
# StorageEncrypted=True | False, # buh? drive encryption I think.
# KmsKeyId='string',
# TdeCredentialArn='string', # probably not something we will implement
# TdeCredentialPassword='******', # probably not something we will implement
# Security
MasterUsername=credentials['RDS_USERNAME'],
MasterUserPassword=credentials['RDS_PASSWORD'],
DBName=credentials['RDS_DB_NAME'],
EnableIAMDatabaseAuthentication=False,
Engine=engine['Engine'], # will be "postgres"
EngineVersion=engine[
'EngineVersion'], # most recent postgres version in this region.
PreferredMaintenanceWindow=MAINTAINANCE_WINDOW,
PreferredBackupWindow=BACKUP_WINDOW,
AutoMinorVersionUpgrade=True, # auto-upgrades are fantastic
BackupRetentionPeriod=BACKUP_RETENTION_PERIOD_DAYS,
Tags=[
{
'Key': 'BEIWE-NAME',
'Value': 'Beiwe postgres database for %s' % eb_environment_name
},
],
# Enhanced monitoring, leave disabled
# MonitoringInterval=5, # in seconds, Valid Values: 0, 1, 5, 10, 15, 30, 60
# MonitoringRoleArn='string', # required for monitoring interval other than 0
# near as I can tell this is the "insert postgres paratmeters here" section.
# DBParameterGroupName='string',
# AvailabilityZone='string', # leave as default (random)
# DBSecurityGroups=['strings'], # non-vpc rds instance settings
# LicenseModel='string',
# CharacterSetName='string',
# OptionGroupName='string', # don't think this is required.
# Domain='string', # has the phrase "active directory" in the description
# DomainIAMRoleName='string',
# CopyTagsToSnapshot=True | False,
# Timezone='string', # only used by MSSQL
# DBClusterIdentifier='string', #
# EnablePerformanceInsights=True, # Aurora specific
# PerformanceInsightsKMSKeyId='string' # Aurora specific
# PromotionTier = 123, # Aurora specific
)
while True:
try:
db = get_db_info(eb_environment_name)
except DBInstanceNotFound:
log.error(
"couldn't find database %s, hopefully this is a momentary glitch. Retrying."
)
sleep(5)
continue
log.info(
'%s: RDS instance status is %s, waiting until status is "Ready"' %
(current_time_string(), db['DBInstanceStatus']))
# RDS spinup goes creating > backing up > available.
if db['DBInstanceStatus'] in ["creating", 'backing-up']:
sleep(5)
elif db['DBInstanceStatus'] == "available":
log.info("Database status is no longer 'creating', it is '%s'" %
db['DBInstanceStatus'])
break
else:
raise Exception('encountered unknown database state "%s"' %
db['DBInstanceStatus'])
return db
def create_eb_environment(eb_environment_name, without_db=False):
# Don't actually use the without_db flag in production, it is for debugging
app = get_or_create_eb_application()
# if not without_db:
# try:
# _ = get_db_info(eb_environment_name)
# except DBInstanceNotFound:
# log.error("could not find a database named '%s,' you must create a database first."
# % construct_db_name(eb_environment_name))
# EXIT()
option_settings = construct_eb_environment_variables(eb_environment_name)
log.info(
"creating a new Elastic Beanstalk environment named %s... this will take a while."
% eb_environment_name)
eb_client = create_eb_client()
env = eb_client.create_environment(
ApplicationName=BEIWE_APPLICATION_NAME,
EnvironmentName=eb_environment_name,
Description='elastic beanstalk beiwe cluster',
PlatformArn=get_python36_platform_arn(),
OptionSettings=option_settings,
# VersionLabel='string', # TODO: this will probably be required later?
# a different form of configuration management
# OptionsToRemove=[
# {'ResourceName': 'string',
# 'Namespace': 'string',
# 'OptionName': 'string'}]
# Tags=[{'Key': 'string',
# 'Value': 'string'}],
# CNAMEPrefix='string', # not required
# Tier={'Name': 'string',
# 'Type': 'string',
# 'Version': 'string'},
# GroupName='string', # for use in other methods of eb configuration
# TemplateName='string', # nope
# SolutionStackName='string', # more about templates
)
env_id = env['EnvironmentId']
good_eb_environment_states = ["Launching", "Updating"]
bad_eb_environment_states = ["Terminating", "Terminated"]
while True:
envs = retry(eb_client.describe_environments,
EnvironmentIds=[env_id])['Environments']
log.info(
'%s: Elastic Beanstalk status is "%s", waiting until status is "Ready"'
% (current_time_string(), env['Status']))
if len(envs) != 1:
raise Exception(
"describe_environments is broken, %s environments returned" %
len(envs))
env = envs[0]
if env['Status'] in bad_eb_environment_states:
msg = "environment deployment failed:\n%s" % format(env)
log.error(
msg
) # python logging is weird and this fails to print if python exits too quickly.
raise EnvironmentDeploymentFailure(msg)
if env['Status'] in good_eb_environment_states:
sleep(5)
continue
if env['Status'] == "Ready":
log.info("environment %s, is ready to have Beiwe deployed to it." %
eb_environment_name)
break
encrypt_eb_s3_bucket()
allow_eb_environment_database_access(eb_environment_name)
allow_443_traffic_to_load_balancer(eb_environment_name)
return env