def do_create_manager(): name = prompt_for_extant_eb_environment_name() do_fail_if_environment_does_not_exist(name) create_processing_server_configuration_file(name) try: settings = get_server_configuration_file(name) except Exception as e: log.error("could not read settings file") log.error(e) EXIT(1) log.info("creating manager server for %s..." % name) try: instance = create_processing_control_server( name, settings["MANAGER_SERVER_INSTANCE_TYPE"]) except Exception as e: log.error(e) EXIT(1) public_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][ 'Association']['PublicIp'] configure_fabric(name, public_ip) push_files() apt_installs(manager=True) setup_rabbitmq() load_git_repo() setup_python() push_beiwe_configuration(name) push_manager_private_ip_and_password(name) setup_manager_cron()
def do_create_manager(): name = prompt_for_extant_eb_environment_name() do_fail_if_environment_does_not_exist(name) create_processing_server_configuration_file(name) try: settings = get_server_configuration_file(name) except Exception as e: log.error("could not read settings file") log.error(e) EXIT(1) log.info("creating manager server for %s..." % name) try: instance = create_processing_control_server( name, settings["MANAGER_SERVER_INSTANCE_TYPE"]) except Exception as e: log.error(e) EXIT(1) public_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][ 'Association']['PublicIp'] log.info("Finished creating manager server for %s..." % name) # TODO: fabric up the rabbitmq and cron task, ensure other servers can connect, watch data process configure_fabric(name, public_ip) push_files() apt_installs(manager=True) load_git_repo() setup_python() push_beiwe_configuration(name) push_manager_private_ip(name) # CC add script to create rabbitmq user setup_celery_manager() setup_manager_cron()
def do_fail_if_bad_environment_name(name): if not (4 <= len(name) < 40): log.error("That name is either too long or too short.") EXIT(1) if not re.match("^[a-zA-Z0-9-]+$", name) or name.endswith("-"): log.error("that is not a valid Elastic Beanstalk environment name.") EXIT(1)
def create_finalized_configuration(eb_environment_name): # requires an rds server has been created for the environment. # FLASK_SECRET_KEY # S3_BUCKET finalized_cred_path = get_finalized_credentials_file_path( eb_environment_name) if os.path.exists(finalized_cred_path): log.error("Encountered a finalized configuration file at %s." % finalized_cred_path) log.error( "This file contains autogenerated parameters which must be identical between " "data processing servers and the Elastic Beanstalk frontend servers. This file " "should not exist at this time, so the deployment process has been aborted." ) EXIT(1) config = validate_beiwe_environment_config(eb_environment_name) config.update(get_full_db_credentials(eb_environment_name)) config['FLASK_SECRET_KEY'] = random_alphanumeric_string(80) config["S3_BUCKET"] = create_data_bucket(eb_environment_name) config.update(create_server_access_credentials(config["S3_BUCKET"])) with open(finalized_cred_path, 'w') as f: json.dump(config, f, indent=1) return config
def do_help_setup_new_environment(): print(HELP_SETUP_NEW_ENVIRONMENT) name = prompt_for_new_eb_environment_name() do_fail_if_bad_environment_name(name) do_fail_if_environment_exists(name) beiwe_environment_fp = get_beiwe_python_environment_variables_file_path( name) processing_server_settings_fp = get_server_configuration_file_path(name) extant_files = os.listdir(DEPLOYMENT_SPECIFIC_CONFIG_FOLDER) for fp in (beiwe_environment_fp, processing_server_settings_fp): if os.path.basename(fp) in extant_files: log.error("is already a file at %s" % relpath(beiwe_environment_fp)) EXIT(1) with open(beiwe_environment_fp, 'w') as f: json.dump(reference_environment_configuration_file(), f, indent=1) with open(processing_server_settings_fp, 'w') as f: json.dump(reference_data_processing_server_configuration(), f, indent=1) print("Environment specific files have been created at %s and %s." % ( relpath(beiwe_environment_fp), relpath(processing_server_settings_fp), )) # Note: we actually cannot generate RDS credentials until we have a server, this is because # the hostname cannot exist until the server exists. print( """After filling in the required contents of these newly created files you will be able to run the -create-environment command. Note that several more credentials files will be generated as part of that process. """)
def do_create_worker(): name = prompt_for_extant_eb_environment_name() do_fail_if_environment_does_not_exist(name) manager_instance = get_manager_instance_by_eb_environment_name(name) if manager_instance is None: log.error( "There is no manager server for the %s cluster, cannot deploy a worker until there is." % name) EXIT(1) if manager_instance['State']['Name'] != 'running': log.error( "There is a manager server for the %s cluster, but it is not in the running state (%s)." % (name, manager_instance['State']['Name'])) EXIT(1) manager_public_ip = get_manager_public_ip(name) manager_private_ip = get_manager_private_ip(name) try: settings = get_server_configuration_file(name) except Exception as e: log.error("could not read settings file") log.error(e) EXIT(1) log.info("creating worker server for %s..." % name) try: instance = create_processing_server( name, settings["MANAGER_SERVER_INSTANCE_TYPE"]) except Exception as e: log.error(e) EXIT(1) instance_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][ 'Association']['PublicIp'] # TODO: fabric up the worker with the celery/supervisord and ensure it can connect to manager. configure_fabric(name, instance_ip) push_files() apt_installs() load_git_repo() setup_python() push_beiwe_configuration(name) push_manager_private_ip(name) setup_celery_worker() setup_worker_cron()
def prompt_for_extant_eb_environment_name(): print(EXTANT_ENVIRONMENT_PROMPT) name = input() environment_exists = check_if_eb_environment_exists(name) if not environment_exists: log.error("There is no environment with the name %s" % name) EXIT(1) validate_beiwe_environment_config(name) return name
def cli_args_validation(): # Use '"count"' as the type, don't try and be fancy, argparse is a pain. parser.add_argument( '-create-environment', action="count", help="creates new environment with the provided environment name", ) parser.add_argument( '-create-manager', action="count", help="creates a data processing manager for the provided environment", ) parser.add_argument( '-create-worker', action="count", help="creates a data processing worker for the provided environment", ) parser.add_argument( "-help-setup-new-environment", action="count", help= "assists in creation of configuration files for a beiwe environment deployment", ) parser.add_argument( "-fix-health-checks-blocking-deployment", action="count", help= "sometimes deployment operations fail stating that health checks do not have sufficient permissions, run this command to fix that.", ) parser.add_argument( "-dev", action="count", help= "Worker and Manager deploy operations will swap the server over to the development branch instead of master.", ) parser.add_argument( "-prod", action="count", help= "Worker and Manager deploy operations will swap the server over to the production branch instead of master.", ) parser.add_argument( "-purge-instance-profiles", action="count", help=PURGE_COMMAND_BLURB, ) # Note: this arguments variable is not iterable. # access entities as arguments.long_name_of_argument, like arguments.update_manager arguments = parser.parse_args() # print help message if no arguments were supplied if len(sys.argv) == 1: parser.print_help() EXIT() return arguments
def do_create_worker(): name = prompt_for_extant_eb_environment_name() do_fail_if_environment_does_not_exist(name) manager_instance = get_manager_instance_by_eb_environment_name(name) if manager_instance is None: log.error( "There is no manager server for the %s cluster, cannot deploy a worker until there is." % name) EXIT(1) try: settings = get_server_configuration_file(name) except Exception as e: log.error("could not read settings file") log.error(e) settings = None # ide warnings... EXIT(1) log.info("creating worker server for %s..." % name) try: instance = create_processing_server( name, settings[WORKER_SERVER_INSTANCE_TYPE]) except Exception as e: log.error(e) instance = None # ide warnings... EXIT(1) instance_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][ 'Association']['PublicIp'] configure_fabric(name, instance_ip) create_swap() push_home_directory_files() apt_installs() load_git_repo() setup_python() push_beiwe_configuration(name) push_manager_private_ip_and_password(name) setup_worker_cron() setup_celery_worker() # run setup worker last. log.warning( "Server is almost up. Waiting 20 seconds to avoid a race condition..." ) sleep(20) run("supervisord")
def do_setup_eb_update(): print("\n", DO_SETUP_EB_UPDATE_OPEN) files = sorted( [f for f in os.listdir(STAGED_FILES) if f.lower().endswith(".zip")]) if not files: print("Could not find any zip files in " + STAGED_FILES) EXIT(1) print("Enter the version of the codebase do you want to use:") for i, file_name in enumerate(files): print("[%s]: %s" % (i + 1, file_name)) print("(press CTL-C to cancel)\n") try: index = int(input("$ ")) except Exception: log.error("Could not parse input.") index = None # ide warnings EXIT(1) if index < 1 or index > len(files): log.error("%s was not a valid option." % index) EXIT(1) # handle 1-indexing file_name = files[index - 1] # log.info("Processing %s..." % file_name) time_ext = current_time_string().replace(" ", "_").replace(":", "_") output_file_name = file_name[:-4] + "_processed_" + time_ext + ".zip" do_zip_reduction(file_name, STAGED_FILES, output_file_name) log.info("Done processing %s." % file_name) log.info("The new file %s has been placed in %s" % (output_file_name, STAGED_FILES)) print( "You can now provide Elastic Beanstalk with %s to run an automated deployment of the new code." % output_file_name) EXIT(0)
def cli_args_validation(): # Warning: any change to format here requires you re-check all parameter validation parser = argparse.ArgumentParser( description="interactive set of commands for deploying a Beiwe Cluster" ) # Use '"count"' as the type, don't try and be fancy, argparse is a pain. parser.add_argument( '-create-environment', action="count", help="creates new environment with the provided environment name", ) parser.add_argument( '-create-manager', action="count", help="creates a data processing manager for the provided environment", ) parser.add_argument( '-create-worker', action="count", help="creates a data processing worker for the provided environment", ) parser.add_argument( "-help-setup-new-environment", action="count", help= "assists in creation of configuration files for a beiwe environment deployment", ) parser.add_argument( "-fix-health-checks-blocking-deployment", action="count", help= "sometimes deployment operations fail stating that health checks do not have sufficient permissions, run this command to fix that.", ) parser.add_argument( "-purge-instance-profiles", action="count", help=PURGE_COMMAND_BLURB, ) # Note: this arguments variable is not iterable. # access entities as arguments.long_name_of_argument, like arguments.update_manager arguments = parser.parse_args() # print help message if no arguments were supplied if len(sys.argv) == 1: parser.print_help() EXIT() return arguments
def cli_args_validation(): # Use '"count"' as the type, don't try and be fancy, argparse is a pain. parser.add_argument('-create-environment', action="count", help=CREATE_ENVIRONMENT_HELP) parser.add_argument('-create-manager', action="count", help=CREATE_MANAGER_HELP) parser.add_argument('-create-worker', action="count", help=CREATE_WORKER_HELP) parser.add_argument("-help-setup-new-environment", action="count", help=HELP_SETUP_NEW_ENVIRONMENT_HELP) parser.add_argument("-fix-health-checks-blocking-deployment", action="count", help=FIX_HEALTH_CHECKS_BLOCKING_DEPLOYMENT_HELP) parser.add_argument("-dev", action="count", help=DEV_HELP) parser.add_argument("-prod", action="count", help=PROD_HELP) parser.add_argument("-purge-instance-profiles", action="count", help=PURGE_INSTANCE_PROFILES_HELP) parser.add_argument("-terminate-processing-servers", action="count", help=TERMINATE_PROCESSING_SERVERS_HELP) parser.add_argument('-get-manager-ip', action="count", help=GET_MANAGER_IP_ADDRESS_HELP) parser.add_argument('-get-worker-ips', action="count", help=GET_WORKER_IP_ADDRESS_HELP) # Note: this arguments variable is not iterable. # access entities as arguments.long_name_of_argument, like arguments.update_manager arguments = parser.parse_args() # print help message if no arguments were supplied if len(sys.argv) == 1: parser.print_help() EXIT() return arguments
def validate_beiwe_environment_config(eb_environment_name): # DOMAIN_NAME # SENTRY_ANDROID_DSN # SENTRY_DATA_PROCESSING_DSN # SENTRY_ELASTIC_BEANSTALK_DSN # SENTRY_JAVASCRIPT_DSN # SYSADMIN_EMAILS errors = [] try: aws_credentials = get_aws_credentials() global_config = get_global_config() beiwe_variables = get_beiwe_environment_variables(eb_environment_name) except Exception as e: log.error( "encountered an error while trying to read configuration files.") log.error(e) EXIT(1) beiwe_variables_name = os.path.basename( get_beiwe_python_environment_variables_file_path(eb_environment_name)) reference_environment_configuration_keys = reference_environment_configuration_file( ).keys() # Validate the data sysadmin_email = global_config.get('SYSTEM_ADMINISTRATOR_EMAIL', "") if not sysadmin_email: errors.append( '(Global Configuration) System administrator email cannot be empty.' ) else: if not re.match('^[\S]+@[\S]+\.[\S]+$', sysadmin_email): errors.append( '(Global Configuration) Invalid email address: {}'.format( sysadmin_email)) # check sentry urls sentry_dsns = { "SENTRY_ELASTIC_BEANSTALK_DSN": beiwe_variables.get('SENTRY_ELASTIC_BEANSTALK_DSN', ''), "SENTRY_DATA_PROCESSING_DSN": beiwe_variables.get('SENTRY_DATA_PROCESSING_DSN', ''), "SENTRY_ANDROID_DSN": beiwe_variables.get('SENTRY_ANDROID_DSN', ''), "SENTRY_JAVASCRIPT_DSN": beiwe_variables.get('SENTRY_JAVASCRIPT_DSN', ''), } for name, dsn in sentry_dsns.iteritems(): if ensure_nonempty_string(dsn, name, errors, beiwe_variables_name): if not DSN_REGEX.match(dsn): errors.append('({}) Invalid DSN: {}'.format( beiwe_variables_name, dsn)) # if name == "SENTRY_JAVASCRIPT_DSN": # if not PUBLIC_DSN_REGEX.match(dsn): # errors.append('({}) Invalid DSN: {}'.format(beiwe_variables_name, dsn)) # elif not PRIVATE_DSN_REGEX.match(dsn): # errors.append('({}) Invalid DSN: {}'.format(beiwe_variables_name, dsn)) domain_name = beiwe_variables.get('DOMAIN', None) ensure_nonempty_string(domain_name, 'Domain name', errors, beiwe_variables_name) for key in reference_environment_configuration_keys: if key not in beiwe_variables: errors.append("{} is missing.".format(key)) for key in beiwe_variables: if key not in reference_environment_configuration_keys: errors.append("{} is present but was not expected.".format(key)) # Raise any errors if errors: for e in errors: log.error(e) sleep( 0.1 ) # python logging has some issues if you exit too fast... isn't it supposed to be synchronous? EXIT(1) # forcibly exit, do not continue to run any code. # Check for presence of the server settings file: if not file_exists( get_server_configuration_file_path(eb_environment_name)): log.error("No server settings file exists at %s." % get_server_configuration_file_path(eb_environment_name)) EXIT(1) # Put the data into one dict to be returned return { 'DOMAIN_NAME': domain_name, 'SYSADMIN_EMAILS': sysadmin_email, 'SENTRY_ELASTIC_BEANSTALK_DSN': sentry_dsns['SENTRY_ELASTIC_BEANSTALK_DSN'], 'SENTRY_DATA_PROCESSING_DSN': sentry_dsns['SENTRY_DATA_PROCESSING_DSN'], 'SENTRY_ANDROID_DSN': sentry_dsns['SENTRY_ANDROID_DSN'], 'SENTRY_JAVASCRIPT_DSN': sentry_dsns['SENTRY_JAVASCRIPT_DSN'] }
def do_fail_if_environment_exists(name): environment_exists = check_if_eb_environment_exists(name) if environment_exists: log.error("There is already an environment named '%s'" % name.lower()) EXIT(1)
if len(sys.argv) == 1: parser.print_help() EXIT() return arguments #################################################################################################### ##################################### Argument Parsing ############################################# #################################################################################################### if __name__ == "__main__": # validate the global configuration file if not all( (are_aws_credentials_present(), is_global_configuration_valid())): EXIT(1) # get CLI arguments, see function for details arguments = cli_args_validation() if arguments.prod: log.warning("RUNNING IN PROD MODE") PROD_MODE.set(True) if arguments.dev: if PROD_MODE: log.error("You cannot provide -prod and -dev at the same time.") EXIT(1) DEV_MODE.set(True) log.warning("RUNNING IN DEV MODE")
def create_new_rds_instance(eb_environment_name): db_instance_identifier = construct_db_name(eb_environment_name) # identify whether there is already a database with this name, we don't want to try: _ = get_db_info(eb_environment_name) log.error("There is already a database named %s" % eb_environment_name) EXIT() except DBInstanceNotFound: pass database_server_type = get_server_configuration_file( eb_environment_name)['DB_SERVER_TYPE'] engine = get_most_recent_postgres_engine() credentials = generate_valid_postgres_credentials() log.info( "writing database credentials to disk, database address will be added later." ) write_rds_credentials(eb_environment_name, credentials, True) # There is some weirdness involving security groups. It looks like there is this concept of # non-vpc security groups, I am fairly certain that this interacts with cross-vpc, IAM based # database access. create_rds_security_groups(db_instance_identifier) db_sec_grp_id = get_rds_security_groups( db_instance_identifier)['database_sec_grp']['GroupId'] log.info("Creating RDS Postgres database named %s" % db_instance_identifier) rds_client = create_rds_client() rds_instance = rds_client.create_db_instance( # server details DBInstanceIdentifier=db_instance_identifier, DBInstanceClass="db." + database_server_type, MultiAZ=False, PubliclyAccessible=False, Port=POSTGRES_PORT, # attach the security group that will allow access VpcSecurityGroupIds=[db_sec_grp_id], #TODO: is this even relevant? # providing the subnet is critical, not providing this value causes the db to be non-vpc # DBSubnetGroupName='string', # db storage StorageType='gp2', # valid options are standard, gp2, io1 # Iops=1000, # multiple between 3 and 10 times the storage; only for use with io1. # AllocatedStorage has weird constraints: # General Purpose (SSD) storage (gp2): Must be an integer from 5 to 6144. # Provisioned IOPS storage (io1): Must be an integer from 100 to 6144. # Magnetic storage (standard): Must be an integer from 5 to 3072. AllocatedStorage=50, # in gigabytes # StorageEncrypted=True | False, # buh? drive encryption I think. # KmsKeyId='string', # TdeCredentialArn='string', # probably not something we will implement # TdeCredentialPassword='******', # probably not something we will implement # Security MasterUsername=credentials['RDS_USERNAME'], MasterUserPassword=credentials['RDS_PASSWORD'], DBName=credentials['RDS_DB_NAME'], EnableIAMDatabaseAuthentication=False, Engine=engine['Engine'], # will be "postgres" EngineVersion=engine[ 'EngineVersion'], # most recent postgres version in this region. PreferredMaintenanceWindow=MAINTAINANCE_WINDOW, PreferredBackupWindow=BACKUP_WINDOW, AutoMinorVersionUpgrade=True, # auto-upgrades are fantastic BackupRetentionPeriod=BACKUP_RETENTION_PERIOD_DAYS, Tags=[ { 'Key': 'BEIWE-NAME', 'Value': 'Beiwe postgres database for %s' % eb_environment_name }, ], # Enhanced monitoring, leave disabled # MonitoringInterval=5, # in seconds, Valid Values: 0, 1, 5, 10, 15, 30, 60 # MonitoringRoleArn='string', # required for monitoring interval other than 0 # near as I can tell this is the "insert postgres paratmeters here" section. # DBParameterGroupName='string', # AvailabilityZone='string', # leave as default (random) # DBSecurityGroups=['strings'], # non-vpc rds instance settings # LicenseModel='string', # CharacterSetName='string', # OptionGroupName='string', # don't think this is required. # Domain='string', # has the phrase "active directory" in the description # DomainIAMRoleName='string', # CopyTagsToSnapshot=True | False, # Timezone='string', # only used by MSSQL # DBClusterIdentifier='string', # # EnablePerformanceInsights=True, # Aurora specific # PerformanceInsightsKMSKeyId='string' # Aurora specific # PromotionTier = 123, # Aurora specific ) while True: try: db = get_db_info(eb_environment_name) except DBInstanceNotFound: log.error( "couldn't find database %s, hopefully this is a momentary glitch. Retrying." ) sleep(5) continue log.info( '%s: RDS instance status is %s, waiting until status is "Ready"' % (current_time_string(), db['DBInstanceStatus'])) # RDS spinup goes creating > backing up > available. if db['DBInstanceStatus'] in ["creating", 'backing-up']: sleep(5) elif db['DBInstanceStatus'] == "available": log.info("Database status is no longer 'creating', it is '%s'" % db['DBInstanceStatus']) break else: raise Exception('encountered unknown database state "%s"' % db['DBInstanceStatus']) return db