def do_create_manager():
name = prompt_for_extant_eb_environment_name()
do_fail_if_environment_does_not_exist(name)
create_processing_server_configuration_file(name)
try:
settings = get_server_configuration_file(name)
except Exception as e:
log.error("could not read settings file")
log.error(e)
EXIT(1)
log.info("creating manager server for %s..." % name)
try:
instance = create_processing_control_server(
name, settings["MANAGER_SERVER_INSTANCE_TYPE"])
except Exception as e:
log.error(e)
EXIT(1)
public_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][
'Association']['PublicIp']
configure_fabric(name, public_ip)
push_files()
apt_installs(manager=True)
setup_rabbitmq()
load_git_repo()
setup_python()
push_beiwe_configuration(name)
push_manager_private_ip_and_password(name)
setup_manager_cron()
def do_create_manager():
name = prompt_for_extant_eb_environment_name()
do_fail_if_environment_does_not_exist(name)
create_processing_server_configuration_file(name)
try:
settings = get_server_configuration_file(name)
except Exception as e:
log.error("could not read settings file")
log.error(e)
EXIT(1)
log.info("creating manager server for %s..." % name)
try:
instance = create_processing_control_server(
name, settings["MANAGER_SERVER_INSTANCE_TYPE"])
except Exception as e:
log.error(e)
EXIT(1)
public_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][
'Association']['PublicIp']
log.info("Finished creating manager server for %s..." % name)
# TODO: fabric up the rabbitmq and cron task, ensure other servers can connect, watch data process
configure_fabric(name, public_ip)
push_files()
apt_installs(manager=True)
load_git_repo()
setup_python()
push_beiwe_configuration(name)
push_manager_private_ip(name)
# CC add script to create rabbitmq user
setup_celery_manager()
setup_manager_cron()
def do_fail_if_bad_environment_name(name):
if not (4 <= len(name) < 40):
log.error("That name is either too long or too short.")
EXIT(1)
if not re.match("^[a-zA-Z0-9-]+$", name) or name.endswith("-"):
log.error("that is not a valid Elastic Beanstalk environment name.")
EXIT(1)
def create_finalized_configuration(eb_environment_name):
# requires an rds server has been created for the environment.
# FLASK_SECRET_KEY
# S3_BUCKET
finalized_cred_path = get_finalized_credentials_file_path(
eb_environment_name)
if os.path.exists(finalized_cred_path):
log.error("Encountered a finalized configuration file at %s." %
finalized_cred_path)
log.error(
"This file contains autogenerated parameters which must be identical between "
"data processing servers and the Elastic Beanstalk frontend servers. This file "
"should not exist at this time, so the deployment process has been aborted."
)
EXIT(1)
config = validate_beiwe_environment_config(eb_environment_name)
config.update(get_full_db_credentials(eb_environment_name))
config['FLASK_SECRET_KEY'] = random_alphanumeric_string(80)
config["S3_BUCKET"] = create_data_bucket(eb_environment_name)
config.update(create_server_access_credentials(config["S3_BUCKET"]))
with open(finalized_cred_path, 'w') as f:
json.dump(config, f, indent=1)
return config
def do_help_setup_new_environment():
print(HELP_SETUP_NEW_ENVIRONMENT)
name = prompt_for_new_eb_environment_name()
do_fail_if_bad_environment_name(name)
do_fail_if_environment_exists(name)
beiwe_environment_fp = get_beiwe_python_environment_variables_file_path(
name)
processing_server_settings_fp = get_server_configuration_file_path(name)
extant_files = os.listdir(DEPLOYMENT_SPECIFIC_CONFIG_FOLDER)
for fp in (beiwe_environment_fp, processing_server_settings_fp):
if os.path.basename(fp) in extant_files:
log.error("is already a file at %s" %
relpath(beiwe_environment_fp))
EXIT(1)
with open(beiwe_environment_fp, 'w') as f:
json.dump(reference_environment_configuration_file(), f, indent=1)
with open(processing_server_settings_fp, 'w') as f:
json.dump(reference_data_processing_server_configuration(),
f,
indent=1)
print("Environment specific files have been created at %s and %s." % (
relpath(beiwe_environment_fp),
relpath(processing_server_settings_fp),
))
# Note: we actually cannot generate RDS credentials until we have a server, this is because
# the hostname cannot exist until the server exists.
print(
"""After filling in the required contents of these newly created files you will be able
to run the -create-environment command. Note that several more credentials files will be
generated as part of that process. """)
def do_create_worker():
name = prompt_for_extant_eb_environment_name()
do_fail_if_environment_does_not_exist(name)
manager_instance = get_manager_instance_by_eb_environment_name(name)
if manager_instance is None:
log.error(
"There is no manager server for the %s cluster, cannot deploy a worker until there is."
% name)
EXIT(1)
if manager_instance['State']['Name'] != 'running':
log.error(
"There is a manager server for the %s cluster, but it is not in the running state (%s)."
% (name, manager_instance['State']['Name']))
EXIT(1)
manager_public_ip = get_manager_public_ip(name)
manager_private_ip = get_manager_private_ip(name)
try:
settings = get_server_configuration_file(name)
except Exception as e:
log.error("could not read settings file")
log.error(e)
EXIT(1)
log.info("creating worker server for %s..." % name)
try:
instance = create_processing_server(
name, settings["MANAGER_SERVER_INSTANCE_TYPE"])
except Exception as e:
log.error(e)
EXIT(1)
instance_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][
'Association']['PublicIp']
# TODO: fabric up the worker with the celery/supervisord and ensure it can connect to manager.
configure_fabric(name, instance_ip)
push_files()
apt_installs()
load_git_repo()
setup_python()
push_beiwe_configuration(name)
push_manager_private_ip(name)
setup_celery_worker()
setup_worker_cron()
def prompt_for_extant_eb_environment_name():
print(EXTANT_ENVIRONMENT_PROMPT)
name = input()
environment_exists = check_if_eb_environment_exists(name)
if not environment_exists:
log.error("There is no environment with the name %s" % name)
EXIT(1)
validate_beiwe_environment_config(name)
return name
def cli_args_validation():
# Use '"count"' as the type, don't try and be fancy, argparse is a pain.
parser.add_argument(
'-create-environment',
action="count",
help="creates new environment with the provided environment name",
)
parser.add_argument(
'-create-manager',
action="count",
help="creates a data processing manager for the provided environment",
)
parser.add_argument(
'-create-worker',
action="count",
help="creates a data processing worker for the provided environment",
)
parser.add_argument(
"-help-setup-new-environment",
action="count",
help=
"assists in creation of configuration files for a beiwe environment deployment",
)
parser.add_argument(
"-fix-health-checks-blocking-deployment",
action="count",
help=
"sometimes deployment operations fail stating that health checks do not have sufficient permissions, run this command to fix that.",
)
parser.add_argument(
"-dev",
action="count",
help=
"Worker and Manager deploy operations will swap the server over to the development branch instead of master.",
)
parser.add_argument(
"-prod",
action="count",
help=
"Worker and Manager deploy operations will swap the server over to the production branch instead of master.",
)
parser.add_argument(
"-purge-instance-profiles",
action="count",
help=PURGE_COMMAND_BLURB,
)
# Note: this arguments variable is not iterable.
# access entities as arguments.long_name_of_argument, like arguments.update_manager
arguments = parser.parse_args()
# print help message if no arguments were supplied
if len(sys.argv) == 1:
parser.print_help()
EXIT()
return arguments
def do_create_worker():
name = prompt_for_extant_eb_environment_name()
do_fail_if_environment_does_not_exist(name)
manager_instance = get_manager_instance_by_eb_environment_name(name)
if manager_instance is None:
log.error(
"There is no manager server for the %s cluster, cannot deploy a worker until there is."
% name)
EXIT(1)
try:
settings = get_server_configuration_file(name)
except Exception as e:
log.error("could not read settings file")
log.error(e)
settings = None # ide warnings...
EXIT(1)
log.info("creating worker server for %s..." % name)
try:
instance = create_processing_server(
name, settings[WORKER_SERVER_INSTANCE_TYPE])
except Exception as e:
log.error(e)
instance = None # ide warnings...
EXIT(1)
instance_ip = instance['NetworkInterfaces'][0]['PrivateIpAddresses'][0][
'Association']['PublicIp']
configure_fabric(name, instance_ip)
create_swap()
push_home_directory_files()
apt_installs()
load_git_repo()
setup_python()
push_beiwe_configuration(name)
push_manager_private_ip_and_password(name)
setup_worker_cron()
setup_celery_worker() # run setup worker last.
log.warning(
"Server is almost up. Waiting 20 seconds to avoid a race condition..."
)
sleep(20)
run("supervisord")
def do_setup_eb_update():
print("\n", DO_SETUP_EB_UPDATE_OPEN)
files = sorted(
[f for f in os.listdir(STAGED_FILES) if f.lower().endswith(".zip")])
if not files:
print("Could not find any zip files in " + STAGED_FILES)
EXIT(1)
print("Enter the version of the codebase do you want to use:")
for i, file_name in enumerate(files):
print("[%s]: %s" % (i + 1, file_name))
print("(press CTL-C to cancel)\n")
try:
index = int(input("$ "))
except Exception:
log.error("Could not parse input.")
index = None # ide warnings
EXIT(1)
if index < 1 or index > len(files):
log.error("%s was not a valid option." % index)
EXIT(1)
# handle 1-indexing
file_name = files[index - 1]
# log.info("Processing %s..." % file_name)
time_ext = current_time_string().replace(" ", "_").replace(":", "_")
output_file_name = file_name[:-4] + "_processed_" + time_ext + ".zip"
do_zip_reduction(file_name, STAGED_FILES, output_file_name)
log.info("Done processing %s." % file_name)
log.info("The new file %s has been placed in %s" %
(output_file_name, STAGED_FILES))
print(
"You can now provide Elastic Beanstalk with %s to run an automated deployment of the new code."
% output_file_name)
EXIT(0)
def cli_args_validation():
# Warning: any change to format here requires you re-check all parameter validation
parser = argparse.ArgumentParser(
description="interactive set of commands for deploying a Beiwe Cluster"
)
# Use '"count"' as the type, don't try and be fancy, argparse is a pain.
parser.add_argument(
'-create-environment',
action="count",
help="creates new environment with the provided environment name",
)
parser.add_argument(
'-create-manager',
action="count",
help="creates a data processing manager for the provided environment",
)
parser.add_argument(
'-create-worker',
action="count",
help="creates a data processing worker for the provided environment",
)
parser.add_argument(
"-help-setup-new-environment",
action="count",
help=
"assists in creation of configuration files for a beiwe environment deployment",
)
parser.add_argument(
"-fix-health-checks-blocking-deployment",
action="count",
help=
"sometimes deployment operations fail stating that health checks do not have sufficient permissions, run this command to fix that.",
)
parser.add_argument(
"-purge-instance-profiles",
action="count",
help=PURGE_COMMAND_BLURB,
)
# Note: this arguments variable is not iterable.
# access entities as arguments.long_name_of_argument, like arguments.update_manager
arguments = parser.parse_args()
# print help message if no arguments were supplied
if len(sys.argv) == 1:
parser.print_help()
EXIT()
return arguments
def cli_args_validation():
# Use '"count"' as the type, don't try and be fancy, argparse is a pain.
parser.add_argument('-create-environment',
action="count",
help=CREATE_ENVIRONMENT_HELP)
parser.add_argument('-create-manager',
action="count",
help=CREATE_MANAGER_HELP)
parser.add_argument('-create-worker',
action="count",
help=CREATE_WORKER_HELP)
parser.add_argument("-help-setup-new-environment",
action="count",
help=HELP_SETUP_NEW_ENVIRONMENT_HELP)
parser.add_argument("-fix-health-checks-blocking-deployment",
action="count",
help=FIX_HEALTH_CHECKS_BLOCKING_DEPLOYMENT_HELP)
parser.add_argument("-dev", action="count", help=DEV_HELP)
parser.add_argument("-prod", action="count", help=PROD_HELP)
parser.add_argument("-purge-instance-profiles",
action="count",
help=PURGE_INSTANCE_PROFILES_HELP)
parser.add_argument("-terminate-processing-servers",
action="count",
help=TERMINATE_PROCESSING_SERVERS_HELP)
parser.add_argument('-get-manager-ip',
action="count",
help=GET_MANAGER_IP_ADDRESS_HELP)
parser.add_argument('-get-worker-ips',
action="count",
help=GET_WORKER_IP_ADDRESS_HELP)
# Note: this arguments variable is not iterable.
# access entities as arguments.long_name_of_argument, like arguments.update_manager
arguments = parser.parse_args()
# print help message if no arguments were supplied
if len(sys.argv) == 1:
parser.print_help()
EXIT()
return arguments
def validate_beiwe_environment_config(eb_environment_name):
# DOMAIN_NAME
# SENTRY_ANDROID_DSN
# SENTRY_DATA_PROCESSING_DSN
# SENTRY_ELASTIC_BEANSTALK_DSN
# SENTRY_JAVASCRIPT_DSN
# SYSADMIN_EMAILS
errors = []
try:
aws_credentials = get_aws_credentials()
global_config = get_global_config()
beiwe_variables = get_beiwe_environment_variables(eb_environment_name)
except Exception as e:
log.error(
"encountered an error while trying to read configuration files.")
log.error(e)
EXIT(1)
beiwe_variables_name = os.path.basename(
get_beiwe_python_environment_variables_file_path(eb_environment_name))
reference_environment_configuration_keys = reference_environment_configuration_file(
).keys()
# Validate the data
sysadmin_email = global_config.get('SYSTEM_ADMINISTRATOR_EMAIL', "")
if not sysadmin_email:
errors.append(
'(Global Configuration) System administrator email cannot be empty.'
)
else:
if not re.match('^[\S]+@[\S]+\.[\S]+$', sysadmin_email):
errors.append(
'(Global Configuration) Invalid email address: {}'.format(
sysadmin_email))
# check sentry urls
sentry_dsns = {
"SENTRY_ELASTIC_BEANSTALK_DSN":
beiwe_variables.get('SENTRY_ELASTIC_BEANSTALK_DSN', ''),
"SENTRY_DATA_PROCESSING_DSN":
beiwe_variables.get('SENTRY_DATA_PROCESSING_DSN', ''),
"SENTRY_ANDROID_DSN":
beiwe_variables.get('SENTRY_ANDROID_DSN', ''),
"SENTRY_JAVASCRIPT_DSN":
beiwe_variables.get('SENTRY_JAVASCRIPT_DSN', ''),
}
for name, dsn in sentry_dsns.iteritems():
if ensure_nonempty_string(dsn, name, errors, beiwe_variables_name):
if not DSN_REGEX.match(dsn):
errors.append('({}) Invalid DSN: {}'.format(
beiwe_variables_name, dsn))
# if name == "SENTRY_JAVASCRIPT_DSN":
# if not PUBLIC_DSN_REGEX.match(dsn):
# errors.append('({}) Invalid DSN: {}'.format(beiwe_variables_name, dsn))
# elif not PRIVATE_DSN_REGEX.match(dsn):
# errors.append('({}) Invalid DSN: {}'.format(beiwe_variables_name, dsn))
domain_name = beiwe_variables.get('DOMAIN', None)
ensure_nonempty_string(domain_name, 'Domain name', errors,
beiwe_variables_name)
for key in reference_environment_configuration_keys:
if key not in beiwe_variables:
errors.append("{} is missing.".format(key))
for key in beiwe_variables:
if key not in reference_environment_configuration_keys:
errors.append("{} is present but was not expected.".format(key))
# Raise any errors
if errors:
for e in errors:
log.error(e)
sleep(
0.1
) # python logging has some issues if you exit too fast... isn't it supposed to be synchronous?
EXIT(1) # forcibly exit, do not continue to run any code.
# Check for presence of the server settings file:
if not file_exists(
get_server_configuration_file_path(eb_environment_name)):
log.error("No server settings file exists at %s." %
get_server_configuration_file_path(eb_environment_name))
EXIT(1)
# Put the data into one dict to be returned
return {
'DOMAIN_NAME': domain_name,
'SYSADMIN_EMAILS': sysadmin_email,
'SENTRY_ELASTIC_BEANSTALK_DSN':
sentry_dsns['SENTRY_ELASTIC_BEANSTALK_DSN'],
'SENTRY_DATA_PROCESSING_DSN':
sentry_dsns['SENTRY_DATA_PROCESSING_DSN'],
'SENTRY_ANDROID_DSN': sentry_dsns['SENTRY_ANDROID_DSN'],
'SENTRY_JAVASCRIPT_DSN': sentry_dsns['SENTRY_JAVASCRIPT_DSN']
}
def do_fail_if_environment_exists(name):
environment_exists = check_if_eb_environment_exists(name)
if environment_exists:
log.error("There is already an environment named '%s'" % name.lower())
EXIT(1)
if len(sys.argv) == 1:
parser.print_help()
EXIT()
return arguments
####################################################################################################
##################################### Argument Parsing #############################################
####################################################################################################
if __name__ == "__main__":
# validate the global configuration file
if not all(
(are_aws_credentials_present(), is_global_configuration_valid())):
EXIT(1)
# get CLI arguments, see function for details
arguments = cli_args_validation()
if arguments.prod:
log.warning("RUNNING IN PROD MODE")
PROD_MODE.set(True)
if arguments.dev:
if PROD_MODE:
log.error("You cannot provide -prod and -dev at the same time.")
EXIT(1)
DEV_MODE.set(True)
log.warning("RUNNING IN DEV MODE")
def create_new_rds_instance(eb_environment_name):
db_instance_identifier = construct_db_name(eb_environment_name)
# identify whether there is already a database with this name, we don't want to
try:
_ = get_db_info(eb_environment_name)
log.error("There is already a database named %s" % eb_environment_name)
EXIT()
except DBInstanceNotFound:
pass
database_server_type = get_server_configuration_file(
eb_environment_name)['DB_SERVER_TYPE']
engine = get_most_recent_postgres_engine()
credentials = generate_valid_postgres_credentials()
log.info(
"writing database credentials to disk, database address will be added later."
)
write_rds_credentials(eb_environment_name, credentials, True)
# There is some weirdness involving security groups. It looks like there is this concept of
# non-vpc security groups, I am fairly certain that this interacts with cross-vpc, IAM based
# database access.
create_rds_security_groups(db_instance_identifier)
db_sec_grp_id = get_rds_security_groups(
db_instance_identifier)['database_sec_grp']['GroupId']
log.info("Creating RDS Postgres database named %s" %
db_instance_identifier)
rds_client = create_rds_client()
rds_instance = rds_client.create_db_instance(
# server details
DBInstanceIdentifier=db_instance_identifier,
DBInstanceClass="db." + database_server_type,
MultiAZ=False,
PubliclyAccessible=False,
Port=POSTGRES_PORT,
# attach the security group that will allow access
VpcSecurityGroupIds=[db_sec_grp_id],
#TODO: is this even relevant?
# providing the subnet is critical, not providing this value causes the db to be non-vpc
# DBSubnetGroupName='string',
# db storage
StorageType='gp2', # valid options are standard, gp2, io1
# Iops=1000, # multiple between 3 and 10 times the storage; only for use with io1.
# AllocatedStorage has weird constraints:
# General Purpose (SSD) storage (gp2): Must be an integer from 5 to 6144.
# Provisioned IOPS storage (io1): Must be an integer from 100 to 6144.
# Magnetic storage (standard): Must be an integer from 5 to 3072.
AllocatedStorage=50, # in gigabytes
# StorageEncrypted=True | False, # buh? drive encryption I think.
# KmsKeyId='string',
# TdeCredentialArn='string', # probably not something we will implement
# TdeCredentialPassword='******', # probably not something we will implement
# Security
MasterUsername=credentials['RDS_USERNAME'],
MasterUserPassword=credentials['RDS_PASSWORD'],
DBName=credentials['RDS_DB_NAME'],
EnableIAMDatabaseAuthentication=False,
Engine=engine['Engine'], # will be "postgres"
EngineVersion=engine[
'EngineVersion'], # most recent postgres version in this region.
PreferredMaintenanceWindow=MAINTAINANCE_WINDOW,
PreferredBackupWindow=BACKUP_WINDOW,
AutoMinorVersionUpgrade=True, # auto-upgrades are fantastic
BackupRetentionPeriod=BACKUP_RETENTION_PERIOD_DAYS,
Tags=[
{
'Key': 'BEIWE-NAME',
'Value': 'Beiwe postgres database for %s' % eb_environment_name
},
],
# Enhanced monitoring, leave disabled
# MonitoringInterval=5, # in seconds, Valid Values: 0, 1, 5, 10, 15, 30, 60
# MonitoringRoleArn='string', # required for monitoring interval other than 0
# near as I can tell this is the "insert postgres paratmeters here" section.
# DBParameterGroupName='string',
# AvailabilityZone='string', # leave as default (random)
# DBSecurityGroups=['strings'], # non-vpc rds instance settings
# LicenseModel='string',
# CharacterSetName='string',
# OptionGroupName='string', # don't think this is required.
# Domain='string', # has the phrase "active directory" in the description
# DomainIAMRoleName='string',
# CopyTagsToSnapshot=True | False,
# Timezone='string', # only used by MSSQL
# DBClusterIdentifier='string', #
# EnablePerformanceInsights=True, # Aurora specific
# PerformanceInsightsKMSKeyId='string' # Aurora specific
# PromotionTier = 123, # Aurora specific
)
while True:
try:
db = get_db_info(eb_environment_name)
except DBInstanceNotFound:
log.error(
"couldn't find database %s, hopefully this is a momentary glitch. Retrying."
)
sleep(5)
continue
log.info(
'%s: RDS instance status is %s, waiting until status is "Ready"' %
(current_time_string(), db['DBInstanceStatus']))
# RDS spinup goes creating > backing up > available.
if db['DBInstanceStatus'] in ["creating", 'backing-up']:
sleep(5)
elif db['DBInstanceStatus'] == "available":
log.info("Database status is no longer 'creating', it is '%s'" %
db['DBInstanceStatus'])
break
else:
raise Exception('encountered unknown database state "%s"' %
db['DBInstanceStatus'])
return db
