def deploy_resources():
rds_client = RDSClient().get_client()
rds = RDS(rds_client)
rds.create_postgresql_instance()
print("Creating RDS PostgreSQL Instance...")
def deprovision(instance_id):
"""Destroys an RDS instance.
"""
# The deprovision endpoint supports both sync and async requests.
# Ideally this would be async only since the operation is actually
# async, but in at least v208 (probably some later versions as
# well) it seems that the cloud controller does not include the
# accepts_incomplete param in the request. The last_operation
# endpoint supports async deprovisions so this should "just work"
# with either sync or async operations. The main difference is
# that the dynamodb reference will be removed here instead of
# opportunistically by the last_operation endpoint.
incompletes = bottle.request.query.getone('accepts_incomplete')
bottle.response.content_type = 'application/json'
dynamodb = utils.boto3_session(**CONFIG['aws']).resource('dynamodb')
table = dynamodb.Table(name=CONFIG['dynamodb_table'])
record = table.get_item(Key={'instance_id': instance_id})
if 'Item' not in record.keys():
bottle.response.status = 410
return json.dumps({})
record = record.pop('Item')
record['last_operation'] = 'destroy'
rds = RDS(DBInstanceIdentifier=record['hostname'], **CONFIG['aws'])
rds.destroy_instance()
if incompletes.lower() == 'true':
bottle.response.status = 202
table.put_item(Item=record)
else:
bottle.response.status = 200
table.delete_item(Key={'instance_id': instance_id})
return json.dumps({})
def create_from_latest_snapshot(config):
rds = RDS(aws_id=config['aws_access_key_id'],
aws_secret_key=config['aws_secret_access_key'],
aws_region=config['aws_region'])
snapshot = rds.get_latest_snapshot(
rds_instance_id=config['rds_instance_id'])
instance = rds.create_instace_from_snapshot(snapshot,
config['rds_subnet_group'])
return instance.id
def create_from_latest_snapshot(config):
rds = RDS(aws_id=config['aws_access_key_id'],
aws_secret_key=config['aws_secret_access_key'],
aws_region=config['aws_region'])
snapshot = rds.get_latest_snapshot(
rds_instance_id=config['rds_instance_id'])
instance = rds.create_instace_from_snapshot(
snapshot, config['rds_subnet_group'])
return instance.id
def deploy_resources():
rds_client = RDSClient().get_client()
print(f'RDs client {rds_client}')
rds = RDS(rds_client)
rds.create_postgresql_instance()
print("creating RDS PostGreSQL Instance")
def ephemeral_zombie_clean(self, args):
k8s = K8S()
rds = RDS(self.args)
route53 = ROUTE53(self.args)
namespaces = k8s.get_k8s_namespaces(pattern=self.args.target_instance)
rds_instances = rds.get_rds_instances(
pattern=self.args.target_instance)
#Check if any RDS instance does not have matching K8s NS
rds_zombies = []
for rds_instance in rds_instances:
found = False
for ns in namespaces:
if (re.search(ns, rds_instance['instance_name'])):
found = True
if (not found): rds_zombies.append(rds_instance)
if not rds_zombies:
logging.info("No ephemeral zombies found")
sys.exit(0)
logging.info(f"Found ephemeral zombies:{rds_zombies}")
attributes = {}
attributes['dns_suffix'] = self.args.dns_suffix
attributes['zone_id'] = self.args.zone_id
#Deleting Zombies and associated CNAME records
zombie_names = []
for zombie in rds_zombies:
ephemeral_name = zombie['instance_name'][:10]
attributes['cname_name'] = f"{ephemeral_name}-rds"
#Get All records for DNS Zone
route53_records = route53.get_records_set(attributes['zone_id'])
route53_record_name = f'{attributes["cname_name"]}.{attributes["dns_suffix"]}.'
#Check if record exists
zombie_cname_record = list(
filter(lambda d: d['Name'] in [route53_record_name],
route53_records))
if zombie_cname_record:
logging.info(
f'DNS record for {route53_record_name} found, deleting ...'
)
route53.update_dns(attributes['cname_name'],
attributes['dns_suffix'],
attributes['zone_id'],
zombie['instance_address'],
action='DELETE')
else:
logging.info(
f'DNS record for {route53_record_name} not found, skipping ...'
)
zombie_names.append(zombie['instance_name'])
rds.destroy_old_instances(zombie_names)
def update_polling(record):
"""Last operation polling logic for update action.
"""
rds = RDS(DBInstanceIdentifier=record['hostname'], **CONFIG['aws'])
details = rds.db_instance_details()
if details['DBInstanceStatus'] != 'available':
response = {'state': 'in progress', 'description': 'Updating service.'}
bottle.response.status = 200
return json.dumps(response)
else:
response = {'state': 'succeeded', 'description': 'Service updated.'}
bottle.response.status = 200
return json.dumps(response)
def backup_data(config):
"""
1. Create an instance from the latest snapshot.
2. Backup the data to s3 given an instance.
3. Delete the instance.
"""
instance_id = db_instance_creater.create_from_latest_snapshot(config)
print('Created instance', instance_id)
data_backup_creater.dump_gzip_backkup(instance_id, config)
print('Deleting instance', instance_id)
rds = RDS(aws_id=config['aws_access_key_id'],
aws_secret_key=config['aws_secret_access_key'],
aws_region=config['aws_region'])
rds.delete_instance(instance_id)
def destroy_polling(record):
"""Last operation polling logic for destroy action.
"""
rds = RDS(DBInstanceIdentifier=record['hostname'], **CONFIG['aws'])
if record['hostname'] in rds.get_all_identifiers():
response = {
'state': 'in progress',
'description': 'Destroying service.'
}
bottle.response.status = 200
return json.dumps(response)
else:
dynamodb = utils.boto3_session(**CONFIG['aws']).resource('dynamodb')
table = dynamodb.Table(name=CONFIG['dynamodb_table'])
table.delete_item(Key={'instance_id': record['instance_id']})
bottle.response.status = 410
return json.dumps({})
def dump_gzip_backkup(instance_id, config):
rds = RDS(aws_id=config['aws_access_key_id'],
aws_secret_key=config['aws_secret_access_key'],
aws_region=config['aws_region'])
instance = rds.get_instance(instance_id)
host = rds.get_host(instance)
compress_command = config.get('compress_command', 'gzip')
assert compress_command in command_to_extension
for db in config['databases']:
s3_location = 's3://{}/{}/{}-{}.sql.{}'.format(
config['s3_bucket'], config['s3_prefix'], db['name'],
datetime.now().strftime('%y-%b-%d-%H-%M-%S'),
command_to_extension[compress_command])
print('Backing Up', db['name'], 'to', s3_location)
create_backup(host, s3_location, db_user=db['user'],
db_password=db['password'], db_name=db['name'],
compress_command=compress_command)
def update(instance_id):
updateable_params = ('AllocatedStorage',)
incompletes = bottle.request.query.getone('accepts_incomplete')
bottle.response.content_type = 'application/json'
if incompletes is None:
return _abort_async_required()
data = json.loads(bottle.request.body.read())
for param in data['parameters'].keys():
if param not in updateable_params:
bottle.response.status = 400
msg = 'Updating of {0} is not supported'.format(param)
return json.dumps({'description': msg})
dynamodb = utils.boto3_session(**CONFIG['aws']).resource('dynamodb')
table = dynamodb.Table(name=CONFIG['dynamodb_table'])
record = table.get_item(Key={'instance_id': instance_id})
if 'Item' not in record.keys():
bottle.response.status = 410
return json.dumps({})
else:
record = record.pop('Item')
rds = RDS(DBInstanceIdentifier=record['hostname'], **CONFIG['aws'])
details = rds.db_instance_details()
if data['parameters']['AllocatedStorage'] <= details['AllocatedStorage']:
bottle.response.status = 400
return json.dumps({
'description': 'Decreasing AllocatedStorage is not supported.'
})
rds.update_instance(
DBInstanceIdentifier=record['hostname'],
**data['parameters']
)
for i in xrange(0,10):
details = rds.db_instance_details()
if details['DBInstanceStatus'] != 'available':
break
sleep(5)
else:
bottle.response.status = 408
return json.dumps({})
record['last_operation'] = 'update'
record['parameters'] = data['parameters']
table.put_item(Item=record)
bottle.response.status = 202
return json.dumps({})
def dump_gzip_backkup(instance_id, config):
rds = RDS(aws_id=config['aws_access_key_id'],
aws_secret_key=config['aws_secret_access_key'],
aws_region=config['aws_region'])
instance = rds.get_instance(instance_id)
host = rds.get_host(instance)
compress_command = config.get('compress_command', 'gzip')
assert compress_command in command_to_extension
for db in config['databases']:
s3_location = 's3://{}/{}/{}-{}.sql.{}'.format(
config['s3_bucket'], config['s3_prefix'], db['name'],
datetime.now().strftime('%y-%b-%d-%H-%M-%S'),
command_to_extension[compress_command])
print('Backing Up', db['name'], 'to', s3_location)
create_backup(host,
s3_location,
db_user=db['user'],
db_password=db['password'],
db_name=db['name'],
compress_command=compress_command)
def get_values(event):
region = None
if 'region' in event:
region = event['region']
modules_process = []
if 'custom' not in event:
beanstalk = Beanstalk()
modules_process.append(beanstalk.run(region))
ec2 = EC2()
modules_process.append(ec2.run(region))
rds = RDS()
modules_process.append(rds.run(region))
ssm = SSM()
modules_process.append(ssm.run(region))
ses = SES()
modules_process.append(ses.run(region))
elasticache = Elasticache()
modules_process.append(elasticache.run(region))
else:
data = str(event['custom'])
if 'BEANSTALK' in data:
beanstalk = Beanstalk()
modules_process.append(beanstalk.run(region))
if 'INSTANCES' in data:
ec2 = EC2()
modules_process.append(ec2.run(region))
if 'SSM' in data:
ssm = SSM()
modules_process.append(ssm.run(region))
if 'ELASTICACHE' in data:
elasticache = Elasticache()
modules_process.append(elasticache.run(region))
if 'RDS' in data:
rds = RDS()
modules_process.append(rds.run(region))
if 'SES' in data:
ses = SES()
modules_process.append(ses.run(region))
start_all(modules_process)
wait_all(modules_process)
ret_dict = read_all(modules_process)
return ret_dict
from rds import RDS
from decouple import config
rds_client = RDS.create_connection()
# AWS POSTGRES DB
db_name = config("DBNAME")
user = config("USER")
endpoint = config("ENTPOIND")
port = "5432"
rds_token = RDS.get_rds_token(rds_client, endpoint, port, user)
RDS.create_instance(rds_client)
RDS.connect_with_postgres(rds_token, db_name, endpoint, port, user)
def provision(instance_id):
"""Provisions an RDS instance.
"""
# TODO: Break this up into more maintainable chunks.
if bottle.request.content_type != 'application/json':
bottle.abort(
415,
'Unsupported Content-Type: expecting application/json'
)
incompletes = bottle.request.query.getone('accepts_incomplete')
bottle.response.content_type = 'application/json'
if incompletes is None:
return _abort_async_required()
if incompletes.lower() == 'true':
data = json.loads(bottle.request.body.read())
for plan in CONFIG['plan_settings']:
if plan['id'] == data['plan_id']:
plan_params = dict(plan)
# Remove the id value from the params so we can just
# pass the whole dict along to the RDS class.
del plan_params['id']
break
else:
bottle.response.status = 400
return json.dumps({'description': 'Plan ID does not exist'})
rds = RDS(**CONFIG['aws'])
# Update the rds class instance with the parameters for the
# plan as defined in the configuration.
rds.__dict__.update(plan_params)
# Parse and use extra parameters that have been passed in by
# the user.
#
# TODO: Move allowed_params to config so operator can determine
# what they want to allow.
allowed_params = ['DBName', 'AllocatedStorage']
if CONFIG['deploy_from_snapshots'] is True:
allowed_params.append('DBSnapshotIdentifier')
if 'parameters' in data.keys():
user_params = dict([
(k, v) for (k, v) in data['parameters'].items()
if k in allowed_params
])
rds.__dict__.update(user_params)
else:
user_params = {}
params_to_update = {}
rds.DBInstanceIdentifier = '-'.join([rds.Engine.lower(), instance_id])
rds.MasterUserPassword = utils.random_string()
if rds.DBSnapshotIdentifier is None:
last_operation = 'create'
source_snapshot = 'NONE'
step = 'NONE'
first_char = random.choice(string.letters)
rds.MasterUsername = ''.join([
first_char,
utils.random_string(15)
])
rds.create_instance()
else:
last_operation = 'create_from_snapshot'
source_snapshot = rds.DBSnapshotIdentifier
step = 'deploy'
try:
snapshot_metadata = rds.snapshot_metadata()
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'DBSnapshotNotFound':
bottle.response.status = 400
return json.dumps(
{'description': 'Invalid snapshot identifier'}
)
else:
raise
if snapshot_metadata['Engine'] != rds.Engine.lower():
bottle.response.status = 400
return json.dumps(
{'description': 'Database engine in snapshot differs from '
'database engine in plan settings.'}
)
rds.MasterUsername = snapshot_metadata['MasterUsername']
rds.Port = snapshot_metadata['Port']
# If the user is requesting a bigger disk than the snapshot
# was generated from store this parameter so we can change
# it during the modify operation after initial provisioning.
if rds.AllocatedStorage > snapshot_metadata['AllocatedStorage']:
params_to_update['AllocatedStorage'] = rds.AllocatedStorage
if rds.StorageType != snapshot_metadata['StorageType']:
params_to_update['StorageType']
# When deploying from snapshot the security groups is always
# set to the default security group. The only way to change
# it is to modify the instance after provisioning is done.
# If the security group IDs are provided then they take
# precedence over the named security groups. If named
# groups are provided they will be validated now before the
# instance is created and stored to be applied after the
# instance is done with initial bootstrapping.
if rds.VpcSecurityGroupIds:
params_to_update['VpcSecurityGroupIds'] = rds.VpcSecurityGroupIds
else:
group_ids = rds.validate_security_groups()
if group_ids[0]:
params_to_update['VpcSecurityGroupIds'] = group_ids[1]
else:
bottle.response.status = 400
return json.dumps(
{'description': 'Invalid AWS security group id'}
)
rds.create_from_snapshot()
iv = utils.Crypt.generate_iv()
credentials = {
'username': rds.MasterUsername,
'password': rds.MasterUserPassword,
'hostname': '',
'port': rds.Port,
'db_name': rds.DBName,
'uri': '',
}
with utils.Crypt(iv=iv, key=CONFIG['encryption_key']) as c:
creds = c.encrypt(json.dumps(credentials))
dynamodb = utils.boto3_session(**CONFIG['aws']).resource('dynamodb')
table = dynamodb.Table(name=CONFIG['dynamodb_table'])
record = {
'instance_id': instance_id,
'iv': iv,
'hostname': rds.DBInstanceIdentifier,
'credentials': creds,
'engine': rds.Engine,
'binding_ids': [],
'parameters': user_params,
'last_operation': last_operation,
'source_snapshot': source_snapshot,
'step': step,
'params_to_update': params_to_update
}
record.update(data)
table.put_item(Item=record)
else:
return _abort_async_required()
bottle.response.status = 202
return json.dumps({"dashboard_url": ""})
def create_from_snapshot_polling(record):
"""Last operation polling logic for create_from_snaphost action.
"""
bottle.response.content_type = 'application/json'
try:
dynamodb = utils.boto3_session(**CONFIG['aws']).resource('dynamodb')
table = dynamodb.Table(name=CONFIG['dynamodb_table'])
rds = RDS(name=record['hostname'], **CONFIG['aws'])
filters = {'DBInstanceIdentifier': record['hostname']}
details = rds.rds_conn.describe_db_instances(**filters)
details = details['DBInstances'][0]
except botocore.exceptions.ClientError as e:
# This exception will be raised if nothing matches the filter.
if e.response['Error']['Code'] == 'DBInstanceNotFound':
bottle.response.status = 410
return json.dumps({})
else:
raise
if record['step'] == 'deploy':
if details['DBInstanceStatus'] == 'available':
params = {'DBInstanceIdentifier': record['hostname']}
params.update(record['params_to_update'])
# Get the new password to pass along for modify.
with utils.Crypt(iv=record['iv'],
key=CONFIG['encryption_key']) as c:
creds = json.loads(c.decrypt(record['credentials']))
params['MasterUserPassword'] = creds['password']
rds.update_instance(**params)
record['step'] = 'modify'
table.put_item(Item=record)
msg = ('RDS Instance is currently in the {0} '
'state.'.format(details['DBInstanceStatus']))
response = {'state': 'in progress', 'description': msg}
bottle.response.status = 200
return json.dumps(response)
elif record['step'] == 'modify':
if details['DBInstanceStatus'] == 'available':
with utils.Crypt(iv=record['iv'],
key=CONFIG['encryption_key']) as c:
creds = json.loads(c.decrypt(record['credentials']))
creds['hostname'] = details['Endpoint']['Address']
uri = '{0}://{1}:{2}@{3}:{4}/{5}'.format(
details['Engine'].lower(),
creds['username'],
creds['password'],
creds['hostname'],
creds['port'],
creds['db_name']
)
creds['uri'] = uri
with utils.Crypt(iv=record['iv'],
key=CONFIG['encryption_key']) as c:
creds = c.encrypt(json.dumps(creds))
record['credentials'] = creds
record['step'] = 'complete'
table.put_item(Item=record)
msg = ('RDS Instance is currently in the {0} '
'state.'.format(details['DBInstanceStatus']))
response = {'state': 'in progress', 'description': msg}
bottle.response.status = 200
return json.dumps(response)
elif record['step'] == 'complete':
response = {'state': 'succeeded', 'description': 'Service Created.'}
bottle.response.status = 200
return json.dumps(response)
else:
msg = 'The instance failed to provision'
response = {'state': 'failed', 'description': msg}
bottle.response.status = 200
return json.dumps(response)
def get_rds():
rds_client = RDSClient().get_client()
rds = RDS(rds_client)
return rds
