Ejemplo n.º 1
0
def createrds(block_gb_size=12):
    """
    Spin up a new database backend with Amazon RDS.
    """
    loadconfig()

    print("Connecting to Amazon RDS")
    conn = boto.rds.connect_to_region(
        env.AWS_REGION,
        aws_access_key_id=env.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
    )

    print("- Reserving an database")
    db = conn.create_dbinstance(
        "ccdc-%s" % random.choice(range(0, 99)),
        block_gb_size,
        'db.%s' % env.EC2_INSTANCE_TYPE,
        'ccdc',  # Username
        'ccdcccdc',  # Password
        db_name='ccdc',
        security_groups=[env.AWS_SECURITY_GROUP])

    # Check up on its status every so often
    print('- Waiting for instance to start')
    status = db.update()
    while status != 'available':
        time.sleep(10)
        status = db.update()

    return db.endpoint[0]
Ejemplo n.º 2
0
def createserver(ami='ami-978dd9a7', block_gb_size=100):
    """
    Spin up a new Ubuntu 14.04 server on Amazon EC2.

    Returns the id and public address.
    """
    loadconfig()
    print("Connecting to Amazon EC2")
    conn = boto.ec2.connect_to_region(
        env.AWS_REGION,
        aws_access_key_id=env.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
    )
    print("- Reserving an instance")
    bdt = boto.ec2.blockdevicemapping.BlockDeviceType(connection=conn)
    bdt.size = block_gb_size
    bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping(connection=conn)
    bdm['/dev/sda1'] = bdt
    reservation = conn.run_instances(
        ami,
        key_name=env.key_name,
        instance_type=env.EC2_INSTANCE_TYPE,
        security_groups=(env.AWS_SECURITY_GROUP, ),
        block_device_map=bdm,
    )
    instance = reservation.instances[0]
    instance.add_tag("Name", "calaccess")
    print('- Waiting for instance to start')
    # Check up on its status every so often
    status = instance.update()
    while status == 'pending':
        time.sleep(10)
        status = instance.update()
    print("- Provisioned at: %s" % instance.public_dns_name)
    return (instance.id, instance.public_dns_name)
Ejemplo n.º 3
0
def clonerds(name=''):
    """
    Create a new RDS instance from a backup snapshot.
    """
    loadconfig()

    print("Connecting to Amazon RDS")
    conn = boto.rds.connect_to_region(
        env.AWS_REGION,
        aws_access_key_id=env.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
    )

    print("- Restoring a database")
    db = conn.restore_dbinstance_from_dbsnapshot(
        'ccdc-20140107',
        name,
        'db.%s' % env.EC2_INSTANCE_TYPE,
    )

    # Check up on its status every so often
    print('- Waiting for instance to start')
    status = db.update()
    while status != 'available':
        time.sleep(10)
        status = db.update()

    return db.endpoint[0]
def createrds(block_gb_size=12):
    """
    Spin up a new database backend with Amazon RDS.
    """
    loadconfig()
    client = boto3.client('rds')

    db_instance_id = "calaccessraw-{0}".format(
        random.choice(range(0, 99))
    )

    # check to see if there db instance already exists
    while True:
        try:
            client.describe_db_instances(
                DBInstanceIdentifier=db_instance_id
            )
        except ClientError as e:
            if 'DBInstanceNotFound' in e.message:
                break
        else:
            # if the db instance already exists, generate a new id
            db_instance_id = "calaccessraw-{0}".format(
                random.choice(range(0, 99))
            )

    print "- Reserving a database"

    # full list of kwargs:
    # http://boto3.readthedocs.io/en/latest/reference/services/rds.html#RDS.Client.create_db_instance # noqa
    db = client.create_db_instance(
        DBName='calaccess_raw',
        DBInstanceIdentifier=db_instance_id,
        AllocatedStorage=block_gb_size,
        DBInstanceClass='db.t1.micro',
        Engine='postgres',
        MasterUsername='******',
        MasterUserPassword=env.DB_USER_PASSWORD,
        BackupRetentionPeriod=14,
        PreferredBackupWindow='22:30-23:00',
        Port=5432,
        MultiAZ=False,
        EngineVersion='9.4.5',
        PubliclyAccessible=True,
        StorageType='gp2',
        StorageEncrypted=False,
    )

    # Check up on its status every so often
    print '- Waiting for instance {0} to start'.format(db_instance_id)
    waiter = client.get_waiter('db_instance_available')
    waiter.wait(DBInstanceIdentifier=db_instance_id)

    db = client.describe_db_instances(
        DBInstanceIdentifier=db_instance_id)

    return db['DBInstances'][0]['Endpoint']['Address']
def copys3(src_bucket, dest_bucket):
    """
    Copy objects in the source AWS S3 bucket to the destination S3 bucket.

    Ignores source bucket objects with the same name as objects already in the
    destination bucket.
    """
    loadconfig()

    session = boto3.Session(
        aws_access_key_id=env.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
        region_name=env.AWS_REGION_NAME
    )
    client = session.client('s3')
    s3 = boto3.resource('s3')
    src = s3.Bucket(src_bucket)
    dest = s3.Bucket(dest_bucket)

    src_objects = [
        obj.key for obj
        in src.objects.all()
    ]

    dest_objects = [
        obj.key for obj
        in dest.objects.all()
    ]

    objs_to_copy = [obj for obj in src_objects if obj not in dest_objects]

    for obj in objs_to_copy:
        print('- Copying {0} from {1} to {2}'.format(
                obj,
                src_bucket,
                dest_bucket,
            )
        )
        copy_source = {
            'Bucket': src_bucket,
            'Key': obj
        }
        client.copy(
            copy_source,
            dest_bucket,
            obj,
        )

    print(green("Success!"))
def createkey(name):
    """
    Creates an EC2 key pair and saves it to a .pem file
    """
    # Make sure the key directory is there
    os.path.exists(env.key_file_dir) or os.makedirs(env.key_file_dir)

    # Connect to boto
    session = boto3.Session(
        aws_access_key_id=env.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
        region_name=env.AWS_REGION_NAME
    )
    client = session.client('ec2')

    # Create the key
    try:
        key_pair = client.create_key_pair(KeyName=name)
    except ClientError as e:
        if 'InvalidKeyPair.Duplicate' in e.message:
            print("A key with named {0} already exists".format(name))
            return False
        else:
            raise e

    # Save the key name to the configuration file
    setconfig('KEY_NAME', name)

    # Reboot the env
    loadconfig()

    # Save the key
    with open(env.key_filename[0], 'w') as f:
        f.write(key_pair['KeyMaterial'])
    # Set it to tight permissions
    os.chmod(env.key_filename[0], stat.S_IRUSR)

    print(green("Success!"))
    print("Key created at {}".format(env.key_filename[0]))
def createserver(block_gb_size=100):
    """
    Spin up a new Ubuntu 14.04 server on Amazon EC2.
    Returns the id and public address.
    """
    loadconfig()

    ec2 = boto3.resource('ec2')

    # full list of kwargs:
    # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances # noqa
    new_instance = ec2.create_instances(
        ImageId=env.AMI,
        MinCount=1,
        MaxCount=1,
        InstanceType=env.EC2_INSTANCE_TYPE,
        SecurityGroups=[env.AWS_SECURITY_GROUP],
        BlockDeviceMappings=[
            {
                'DeviceName': '/dev/sda1',
                'Ebs': {
                    'VolumeSize': block_gb_size,
                },
            },
        ],
        KeyName=env.key_name,
    )[0]

    new_instance.create_tags(Tags=[{"Key": "Name", "Value": "calaccess"}])

    print '- Waiting for instance to start'
    new_instance.wait_until_running()

    print "- Provisioned at: {0}".format(new_instance.public_dns_name)

    return (new_instance.id, new_instance.public_dns_name)
def createkeypair():
    """
    Creates an EC2 key pair and saves it to a .pem file
    """
    loadconfig()
    client = boto3.client('ec2')

    key_file_dir = os.path.expanduser("~/.ec2/")

    os.path.exists(key_file_dir) or os.makedirs(key_file_dir)

    try:
        key_pair = client.create_key_pair(KeyName=env.key_name)
    except ClientError as e:
        if 'InvalidKeyPair.Duplicate' in e.message:
            print "A key with named {0} already exists".format(env.key_name)
        else:
            raise e
    else:
        print "- Saving to {0}".format(env.key_filename[0])
        with open(env.key_filename[0], 'w') as f:
            f.write(key_pair['KeyMaterial'])

        os.chmod(env.key_filename[0], stat.S_IRUSR)
def copydb(src_db_instance_id, dest_db_instance_id, make_snapshot=False):
    """
    Copy the most recent snapshot on the source AWS RDS instance to the
    destination RDS instance.
    """
    # Connect to boto
    loadconfig()
    session = boto3.Session(
        aws_access_key_id=env.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=env.AWS_SECRET_ACCESS_KEY,
        region_name=env.AWS_REGION_NAME
    )
    client = session.client('rds')

    if make_snapshot:
        client.create_db_snapshot(
            DBSnapshotIdentifier='{0}-{1}'.format(
                src_db_instance_id,
                datetime.now().strftime('%Y-%m-%d-%H-%M'),
            ),
            DBInstanceIdentifier=src_db_instance_id,
        )
        print('- Creating snapshot of {0}'.format(src_db_instance_id))
        # wait until snapshot completes
        waiter = client.get_waiter('db_snapshot_completed')
        waiter.wait(DBInstanceIdentifier=src_db_instance_id)

    # get all the snapshots
    snapshots = client.describe_db_snapshots(
        DBInstanceIdentifier=src_db_instance_id
    )['DBSnapshots']

    # get the most recent completed snapshot
    last_snapshot = sorted(
        [s for s in snapshots if s['PercentProgress'] == 100],
        key=lambda k: k['SnapshotCreateTime'],
        reverse=True,
    )[0]

    # delete the current rds instance with the destination id (if it exists)
    try:
        client.delete_db_instance(
            DBInstanceIdentifier=dest_db_instance_id,
            SkipFinalSnapshot=True,
        )
    except ClientError as e:
        if 'DBInstanceNotFound' in e.message:
            pass
        # if some other ClientError, just raise it
        else:
            raise
    else:
        print('- Deleting current {0} instance'.format(dest_db_instance_id))
        # wait until existing destination instance is deleted
        waiter = client.get_waiter('db_instance_deleted')
        waiter.wait(DBInstanceIdentifier=dest_db_instance_id)

    # restore destination instance from last snapshot of source
    client.restore_db_instance_from_db_snapshot(
        DBInstanceIdentifier=dest_db_instance_id,
        DBSnapshotIdentifier=last_snapshot['DBSnapshotIdentifier']
    )

    print('- Restoring {0} from last snapshot of {1}'.format(
            dest_db_instance_id,
            src_db_instance_id,
        ))
    # wait until restored destination instance becomes available
    waiter = client.get_waiter('db_instance_available')
    waiter.wait(DBInstanceIdentifier=dest_db_instance_id)

    print(green("Success!"))