def createrds(block_gb_size=12):
"""
Spin up a new database backend with Amazon RDS.
"""
loadconfig()
print("Connecting to Amazon RDS")
conn = boto.rds.connect_to_region(
env.AWS_REGION,
aws_access_key_id=env.AWS_ACCESS_KEY_ID,
aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
)
print("- Reserving an database")
db = conn.create_dbinstance(
"ccdc-%s" % random.choice(range(0, 99)),
block_gb_size,
'db.%s' % env.EC2_INSTANCE_TYPE,
'ccdc', # Username
'ccdcccdc', # Password
db_name='ccdc',
security_groups=[env.AWS_SECURITY_GROUP])
# Check up on its status every so often
print('- Waiting for instance to start')
status = db.update()
while status != 'available':
time.sleep(10)
status = db.update()
return db.endpoint[0]
def createserver(ami='ami-978dd9a7', block_gb_size=100):
"""
Spin up a new Ubuntu 14.04 server on Amazon EC2.
Returns the id and public address.
"""
loadconfig()
print("Connecting to Amazon EC2")
conn = boto.ec2.connect_to_region(
env.AWS_REGION,
aws_access_key_id=env.AWS_ACCESS_KEY_ID,
aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
)
print("- Reserving an instance")
bdt = boto.ec2.blockdevicemapping.BlockDeviceType(connection=conn)
bdt.size = block_gb_size
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping(connection=conn)
bdm['/dev/sda1'] = bdt
reservation = conn.run_instances(
ami,
key_name=env.key_name,
instance_type=env.EC2_INSTANCE_TYPE,
security_groups=(env.AWS_SECURITY_GROUP, ),
block_device_map=bdm,
)
instance = reservation.instances[0]
instance.add_tag("Name", "calaccess")
print('- Waiting for instance to start')
# Check up on its status every so often
status = instance.update()
while status == 'pending':
time.sleep(10)
status = instance.update()
print("- Provisioned at: %s" % instance.public_dns_name)
return (instance.id, instance.public_dns_name)
def clonerds(name=''):
"""
Create a new RDS instance from a backup snapshot.
"""
loadconfig()
print("Connecting to Amazon RDS")
conn = boto.rds.connect_to_region(
env.AWS_REGION,
aws_access_key_id=env.AWS_ACCESS_KEY_ID,
aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
)
print("- Restoring a database")
db = conn.restore_dbinstance_from_dbsnapshot(
'ccdc-20140107',
name,
'db.%s' % env.EC2_INSTANCE_TYPE,
)
# Check up on its status every so often
print('- Waiting for instance to start')
status = db.update()
while status != 'available':
time.sleep(10)
status = db.update()
return db.endpoint[0]
def createrds(block_gb_size=12):
"""
Spin up a new database backend with Amazon RDS.
"""
loadconfig()
client = boto3.client('rds')
db_instance_id = "calaccessraw-{0}".format(
random.choice(range(0, 99))
)
# check to see if there db instance already exists
while True:
try:
client.describe_db_instances(
DBInstanceIdentifier=db_instance_id
)
except ClientError as e:
if 'DBInstanceNotFound' in e.message:
break
else:
# if the db instance already exists, generate a new id
db_instance_id = "calaccessraw-{0}".format(
random.choice(range(0, 99))
)
print "- Reserving a database"
# full list of kwargs:
# http://boto3.readthedocs.io/en/latest/reference/services/rds.html#RDS.Client.create_db_instance # noqa
db = client.create_db_instance(
DBName='calaccess_raw',
DBInstanceIdentifier=db_instance_id,
AllocatedStorage=block_gb_size,
DBInstanceClass='db.t1.micro',
Engine='postgres',
MasterUsername='******',
MasterUserPassword=env.DB_USER_PASSWORD,
BackupRetentionPeriod=14,
PreferredBackupWindow='22:30-23:00',
Port=5432,
MultiAZ=False,
EngineVersion='9.4.5',
PubliclyAccessible=True,
StorageType='gp2',
StorageEncrypted=False,
)
# Check up on its status every so often
print '- Waiting for instance {0} to start'.format(db_instance_id)
waiter = client.get_waiter('db_instance_available')
waiter.wait(DBInstanceIdentifier=db_instance_id)
db = client.describe_db_instances(
DBInstanceIdentifier=db_instance_id)
return db['DBInstances'][0]['Endpoint']['Address']
def copys3(src_bucket, dest_bucket):
"""
Copy objects in the source AWS S3 bucket to the destination S3 bucket.
Ignores source bucket objects with the same name as objects already in the
destination bucket.
"""
loadconfig()
session = boto3.Session(
aws_access_key_id=env.AWS_ACCESS_KEY_ID,
aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
region_name=env.AWS_REGION_NAME
)
client = session.client('s3')
s3 = boto3.resource('s3')
src = s3.Bucket(src_bucket)
dest = s3.Bucket(dest_bucket)
src_objects = [
obj.key for obj
in src.objects.all()
]
dest_objects = [
obj.key for obj
in dest.objects.all()
]
objs_to_copy = [obj for obj in src_objects if obj not in dest_objects]
for obj in objs_to_copy:
print('- Copying {0} from {1} to {2}'.format(
obj,
src_bucket,
dest_bucket,
)
)
copy_source = {
'Bucket': src_bucket,
'Key': obj
}
client.copy(
copy_source,
dest_bucket,
obj,
)
print(green("Success!"))
def createkey(name):
"""
Creates an EC2 key pair and saves it to a .pem file
"""
# Make sure the key directory is there
os.path.exists(env.key_file_dir) or os.makedirs(env.key_file_dir)
# Connect to boto
session = boto3.Session(
aws_access_key_id=env.AWS_ACCESS_KEY_ID,
aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
region_name=env.AWS_REGION_NAME
)
client = session.client('ec2')
# Create the key
try:
key_pair = client.create_key_pair(KeyName=name)
except ClientError as e:
if 'InvalidKeyPair.Duplicate' in e.message:
print("A key with named {0} already exists".format(name))
return False
else:
raise e
# Save the key name to the configuration file
setconfig('KEY_NAME', name)
# Reboot the env
loadconfig()
# Save the key
with open(env.key_filename[0], 'w') as f:
f.write(key_pair['KeyMaterial'])
# Set it to tight permissions
os.chmod(env.key_filename[0], stat.S_IRUSR)
print(green("Success!"))
print("Key created at {}".format(env.key_filename[0]))
def createserver(block_gb_size=100):
"""
Spin up a new Ubuntu 14.04 server on Amazon EC2.
Returns the id and public address.
"""
loadconfig()
ec2 = boto3.resource('ec2')
# full list of kwargs:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances # noqa
new_instance = ec2.create_instances(
ImageId=env.AMI,
MinCount=1,
MaxCount=1,
InstanceType=env.EC2_INSTANCE_TYPE,
SecurityGroups=[env.AWS_SECURITY_GROUP],
BlockDeviceMappings=[
{
'DeviceName': '/dev/sda1',
'Ebs': {
'VolumeSize': block_gb_size,
},
},
],
KeyName=env.key_name,
)[0]
new_instance.create_tags(Tags=[{"Key": "Name", "Value": "calaccess"}])
print '- Waiting for instance to start'
new_instance.wait_until_running()
print "- Provisioned at: {0}".format(new_instance.public_dns_name)
return (new_instance.id, new_instance.public_dns_name)
def createkeypair():
"""
Creates an EC2 key pair and saves it to a .pem file
"""
loadconfig()
client = boto3.client('ec2')
key_file_dir = os.path.expanduser("~/.ec2/")
os.path.exists(key_file_dir) or os.makedirs(key_file_dir)
try:
key_pair = client.create_key_pair(KeyName=env.key_name)
except ClientError as e:
if 'InvalidKeyPair.Duplicate' in e.message:
print "A key with named {0} already exists".format(env.key_name)
else:
raise e
else:
print "- Saving to {0}".format(env.key_filename[0])
with open(env.key_filename[0], 'w') as f:
f.write(key_pair['KeyMaterial'])
os.chmod(env.key_filename[0], stat.S_IRUSR)
def copydb(src_db_instance_id, dest_db_instance_id, make_snapshot=False):
"""
Copy the most recent snapshot on the source AWS RDS instance to the
destination RDS instance.
"""
# Connect to boto
loadconfig()
session = boto3.Session(
aws_access_key_id=env.AWS_ACCESS_KEY_ID,
aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
region_name=env.AWS_REGION_NAME
)
client = session.client('rds')
if make_snapshot:
client.create_db_snapshot(
DBSnapshotIdentifier='{0}-{1}'.format(
src_db_instance_id,
datetime.now().strftime('%Y-%m-%d-%H-%M'),
),
DBInstanceIdentifier=src_db_instance_id,
)
print('- Creating snapshot of {0}'.format(src_db_instance_id))
# wait until snapshot completes
waiter = client.get_waiter('db_snapshot_completed')
waiter.wait(DBInstanceIdentifier=src_db_instance_id)
# get all the snapshots
snapshots = client.describe_db_snapshots(
DBInstanceIdentifier=src_db_instance_id
)['DBSnapshots']
# get the most recent completed snapshot
last_snapshot = sorted(
[s for s in snapshots if s['PercentProgress'] == 100],
key=lambda k: k['SnapshotCreateTime'],
reverse=True,
)[0]
# delete the current rds instance with the destination id (if it exists)
try:
client.delete_db_instance(
DBInstanceIdentifier=dest_db_instance_id,
SkipFinalSnapshot=True,
)
except ClientError as e:
if 'DBInstanceNotFound' in e.message:
pass
# if some other ClientError, just raise it
else:
raise
else:
print('- Deleting current {0} instance'.format(dest_db_instance_id))
# wait until existing destination instance is deleted
waiter = client.get_waiter('db_instance_deleted')
waiter.wait(DBInstanceIdentifier=dest_db_instance_id)
# restore destination instance from last snapshot of source
client.restore_db_instance_from_db_snapshot(
DBInstanceIdentifier=dest_db_instance_id,
DBSnapshotIdentifier=last_snapshot['DBSnapshotIdentifier']
)
print('- Restoring {0} from last snapshot of {1}'.format(
dest_db_instance_id,
src_db_instance_id,
))
# wait until restored destination instance becomes available
waiter = client.get_waiter('db_instance_available')
waiter.wait(DBInstanceIdentifier=dest_db_instance_id)
print(green("Success!"))