def do_setup_eb_update(): print("\n", DO_SETUP_EB_UPDATE_OPEN) files = sorted( [f for f in os.listdir(STAGED_FILES) if f.lower().endswith(".zip")]) if not files: print("Could not find any zip files in " + STAGED_FILES) EXIT(1) print("Enter the version of the codebase do you want to use:") for i, file_name in enumerate(files): print("[%s]: %s" % (i + 1, file_name)) print("(press CTL-C to cancel)\n") try: index = int(input("$ ")) except Exception: log.error("Could not parse input.") index = None # ide warnings EXIT(1) if index < 1 or index > len(files): log.error("%s was not a valid option." % index) EXIT(1) # handle 1-indexing file_name = files[index - 1] # log.info("Processing %s..." % file_name) time_ext = current_time_string().replace(" ", "_").replace(":", "_") output_file_name = file_name[:-4] + "_processed_" + time_ext + ".zip" do_zip_reduction(file_name, STAGED_FILES, output_file_name) log.info("Done processing %s." % file_name) log.info("The new file %s has been placed in %s" % (output_file_name, STAGED_FILES)) print( "You can now provide Elastic Beanstalk with %s to run an automated deployment of the new code." % output_file_name) EXIT(0)
def create_new_rds_instance(eb_environment_name): db_instance_identifier = construct_db_name(eb_environment_name) # identify whether there is already a database with this name, we don't want to try: _ = get_db_info(eb_environment_name) log.error("There is already a database named %s" % eb_environment_name) EXIT() except DBInstanceNotFound: pass database_server_type = get_server_configuration_file( eb_environment_name)['DB_SERVER_TYPE'] engine = get_most_recent_postgres_engine() credentials = generate_valid_postgres_credentials() log.info( "writing database credentials to disk, database address will be added later." ) write_rds_credentials(eb_environment_name, credentials, True) # There is some weirdness involving security groups. It looks like there is this concept of # non-vpc security groups, I am fairly certain that this interacts with cross-vpc, IAM based # database access. create_rds_security_groups(db_instance_identifier) db_sec_grp_id = get_rds_security_groups( db_instance_identifier)['database_sec_grp']['GroupId'] log.info("Creating RDS Postgres database named %s" % db_instance_identifier) rds_client = create_rds_client() rds_instance = rds_client.create_db_instance( # server details DBInstanceIdentifier=db_instance_identifier, DBInstanceClass="db." + database_server_type, MultiAZ=False, PubliclyAccessible=False, Port=POSTGRES_PORT, # attach the security group that will allow access VpcSecurityGroupIds=[db_sec_grp_id], #TODO: is this even relevant? # providing the subnet is critical, not providing this value causes the db to be non-vpc # DBSubnetGroupName='string', # db storage StorageType='gp2', # valid options are standard, gp2, io1 # Iops=1000, # multiple between 3 and 10 times the storage; only for use with io1. # AllocatedStorage has weird constraints: # General Purpose (SSD) storage (gp2): Must be an integer from 5 to 6144. # Provisioned IOPS storage (io1): Must be an integer from 100 to 6144. # Magnetic storage (standard): Must be an integer from 5 to 3072. AllocatedStorage=50, # in gigabytes # StorageEncrypted=True | False, # buh? drive encryption I think. # KmsKeyId='string', # TdeCredentialArn='string', # probably not something we will implement # TdeCredentialPassword='******', # probably not something we will implement # Security MasterUsername=credentials['RDS_USERNAME'], MasterUserPassword=credentials['RDS_PASSWORD'], DBName=credentials['RDS_DB_NAME'], EnableIAMDatabaseAuthentication=False, Engine=engine['Engine'], # will be "postgres" EngineVersion=engine[ 'EngineVersion'], # most recent postgres version in this region. PreferredMaintenanceWindow=MAINTAINANCE_WINDOW, PreferredBackupWindow=BACKUP_WINDOW, AutoMinorVersionUpgrade=True, # auto-upgrades are fantastic BackupRetentionPeriod=BACKUP_RETENTION_PERIOD_DAYS, Tags=[ { 'Key': 'BEIWE-NAME', 'Value': 'Beiwe postgres database for %s' % eb_environment_name }, ], # Enhanced monitoring, leave disabled # MonitoringInterval=5, # in seconds, Valid Values: 0, 1, 5, 10, 15, 30, 60 # MonitoringRoleArn='string', # required for monitoring interval other than 0 # near as I can tell this is the "insert postgres paratmeters here" section. # DBParameterGroupName='string', # AvailabilityZone='string', # leave as default (random) # DBSecurityGroups=['strings'], # non-vpc rds instance settings # LicenseModel='string', # CharacterSetName='string', # OptionGroupName='string', # don't think this is required. # Domain='string', # has the phrase "active directory" in the description # DomainIAMRoleName='string', # CopyTagsToSnapshot=True | False, # Timezone='string', # only used by MSSQL # DBClusterIdentifier='string', # # EnablePerformanceInsights=True, # Aurora specific # PerformanceInsightsKMSKeyId='string' # Aurora specific # PromotionTier = 123, # Aurora specific ) while True: try: db = get_db_info(eb_environment_name) except DBInstanceNotFound: log.error( "couldn't find database %s, hopefully this is a momentary glitch. Retrying." ) sleep(5) continue log.info( '%s: RDS instance status is %s, waiting until status is "Ready"' % (current_time_string(), db['DBInstanceStatus'])) # RDS spinup goes creating > backing up > available. if db['DBInstanceStatus'] in ["creating", 'backing-up']: sleep(5) elif db['DBInstanceStatus'] == "available": log.info("Database status is no longer 'creating', it is '%s'" % db['DBInstanceStatus']) break else: raise Exception('encountered unknown database state "%s"' % db['DBInstanceStatus']) return db
def create_eb_environment(eb_environment_name, without_db=False): # Don't actually use the without_db flag in production, it is for debugging app = get_or_create_eb_application() # if not without_db: # try: # _ = get_db_info(eb_environment_name) # except DBInstanceNotFound: # log.error("could not find a database named '%s,' you must create a database first." # % construct_db_name(eb_environment_name)) # EXIT() option_settings = construct_eb_environment_variables(eb_environment_name) log.info( "creating a new Elastic Beanstalk environment named %s... this will take a while." % eb_environment_name) eb_client = create_eb_client() env = eb_client.create_environment( ApplicationName=BEIWE_APPLICATION_NAME, EnvironmentName=eb_environment_name, Description='elastic beanstalk beiwe cluster', PlatformArn=get_python36_platform_arn(), OptionSettings=option_settings, # VersionLabel='string', # TODO: this will probably be required later? # a different form of configuration management # OptionsToRemove=[ # {'ResourceName': 'string', # 'Namespace': 'string', # 'OptionName': 'string'}] # Tags=[{'Key': 'string', # 'Value': 'string'}], # CNAMEPrefix='string', # not required # Tier={'Name': 'string', # 'Type': 'string', # 'Version': 'string'}, # GroupName='string', # for use in other methods of eb configuration # TemplateName='string', # nope # SolutionStackName='string', # more about templates ) env_id = env['EnvironmentId'] good_eb_environment_states = ["Launching", "Updating"] bad_eb_environment_states = ["Terminating", "Terminated"] while True: envs = retry(eb_client.describe_environments, EnvironmentIds=[env_id])['Environments'] log.info( '%s: Elastic Beanstalk status is "%s", waiting until status is "Ready"' % (current_time_string(), env['Status'])) if len(envs) != 1: raise Exception( "describe_environments is broken, %s environments returned" % len(envs)) env = envs[0] if env['Status'] in bad_eb_environment_states: msg = "environment deployment failed:\n%s" % format(env) log.error( msg ) # python logging is weird and this fails to print if python exits too quickly. raise EnvironmentDeploymentFailure(msg) if env['Status'] in good_eb_environment_states: sleep(5) continue if env['Status'] == "Ready": log.info("environment %s, is ready to have Beiwe deployed to it." % eb_environment_name) break encrypt_eb_s3_bucket() allow_eb_environment_database_access(eb_environment_name) allow_443_traffic_to_load_balancer(eb_environment_name) return env