def volume_create(config, name, size=256, snapshot_name=None, wait=False):
"""Create an EBS volume.
Args:
config (aws_ml_helper.config.Config): Configuration
name (str): Name of the volume
size (int): Volume size
snapshot_name (str): Name of the snapshot from which the volume should
be created
wait (bool): Wait for the volume to become `available`
"""
ec2 = boto.resource('ec2', config)
from aws_ml_helper.snapshot import get_snapshot
snapshot = get_snapshot(config, snapshot_name)
snapshot_id = (snapshot and snapshot.id) or None
volume = ec2.create_volume(AvailabilityZone=config.availability_zone,
Size=size,
SnapshotId=snapshot_id,
VolumeType='gp2',
TagSpecifications=[{
'ResourceType':
'volume',
'Tags': [{
'Key': 'Name',
'Value': name
}]
}])
if wait:
while volume.state != 'available':
time.sleep(1)
volume.reload()
def snapshots(config):
"""List all snapshots
Args:
config (aws_ml_helper.config.Config): Configuration
"""
ec2 = boto.resource('ec2', config)
data = []
for i in ec2.snapshots.filter(OwnerIds=[config.account]):
data.append([
name_from_tags(i.tags), i.id, i.state, i.volume_size, i.description
])
print(
tabulate(data, ['name', 'id', 'state', 'size', 'description'],
config.table_format))
def get_snapshot(config, name):
"""Get snapshot by name"""
ec2 = boto.resource('ec2', config)
snapshot_list = list(
ec2.snapshots.filter(Filters=[{
'Name': 'tag:Name',
'Values': [name]
}]))
if len(snapshot_list) == 0:
click.secho(f'Snapshot "{name}" not found')
return None
elif len(snapshot_list) > 1:
click.secho(f'Multiple snapshots with name "{name}" found.')
return None
return snapshot_list[0]
def images(config):
"""List instances and their state
Args:
config (aws_ml_helper.config.Config): Configuration
"""
ec2 = boto.resource('ec2', config)
data = []
for i in ec2.images.filter(Owners=[config.account]):
data.append([
i.name,
i.id,
i.state,
])
print(tabulate(data, ['name', 'id', 'state'], config.table_format))
def instances(config):
"""List instances and their state
Args:
config (aws_ml_helper.config.Config): Configuration
"""
ec2 = boto.resource('ec2', config)
data = []
for i in ec2.instances.all():
data.append([
name_from_tags(i.tags), i.id, i.state['Name'], i.public_ip_address
or 'no ip'
])
print(
tabulate(data, ['name', 'id', 'state', 'public ip'],
config.table_format))
def volumes(config):
"""List volumes and their attributes
Args:
config (aws_ml_helper.config.Config): Configuration
"""
ec2 = boto.resource('ec2', config)
data = []
for v in ec2.volumes.all():
data.append([
name_from_tags(v.tags), v.id, v.size, v.state,
', '.join([a['InstanceId'] for a in v.attachments])
])
print(
tabulate(data, ['name', 'id', 'size', 'state', 'attachments'],
config.table_format))
def get_instance(config, name):
"""Returns an instance object with selected name or returns `None` if the
instance is not found.
Args:
config (aws_ml_helper.config.Config): Configuration
name (str): Instance name
Returns:
Instance if found or None
"""
ec2 = boto.resource('ec2', config)
instance_list = list(
ec2.instances.filter(Filters=[{
'Name': 'tag:Name',
'Values': [name]
}]))
if len(instance_list) == 0:
click.secho(f'Instance "{name}" not found')
return None
elif len(instance_list) > 1:
click.secho(f'Multiple instances with name "{name}" found.')
return None
return instance_list[0]
def get_image(config, name):
"""Returns an image object with selected name or returns `None` if the
image is not found.
Args:
config (aws_ml_helper.config.Config): Configuration
name (str): Volume name
Returns:
Volume if found or None
"""
ec2 = boto.resource('ec2', config)
image_list = ec2.images.filter(Owners=[config.account],
Filters=[{
'Name': 'Name',
'Values': [name]
}])
if len(image_list) == 0:
click.secho(f'Volume "{name}" not found')
return None
elif len(image_list) > 1:
click.secho(f'Multiple volumes with name "{name}" found.')
return None
return image_list[0]
def start_spot_instance(config, name, bid_price, ami_id=None,
instance_type=None, snapshot_name=None,
mount_point=None):
"""Starts a spot instance.
Args:
config (aws_ml_helper.config.Config): Configuration
name (str): Spot instance name
bid_price (int): Bidding price for the instance
ami_id (str): AMI id to use. If not provided, value form the
configuration will be used
instance_type (str): Instance type to use. If not provided, value from
the configuration will be used.
snapshot_name (str): Name of the snapshot from which the attached
volume will be created
mount_point (str): Path where the volume should be mounted
"""
ec2 = boto.client('ec2', config)
response = ec2.request_spot_instances(
InstanceCount=1,
Type='one-time',
LaunchSpecification={
'ImageId': ami_id or config.ami_id,
'InstanceType': instance_type or config.instance_type,
'KeyName': f'access-key-{config.vpc_name}',
'EbsOptimized': True,
'Placement': {
'AvailabilityZone': config.availability_zone,
},
# 'BlockDeviceMappings': [
# {
# 'DeviceName': '/dev/sda1',
# 'Ebs': {
# 'DeleteOnTermination': False,
# 'VolumeSize': 128,
# 'VolumeType': 'gp2'
# },
# },
# ],
'Monitoring': {'Enabled': True},
'NetworkInterfaces': [
{
'DeviceIndex': 0,
'AssociatePublicIpAddress': True,
'Groups': [config.ec2_security_group_id],
'SubnetId': config.subnet_id
},
],
},
SpotPrice=f'{bid_price}',
InstanceInterruptionBehavior='terminate'
)
click.echo('Spot instance request created.')
request_id = response['SpotInstanceRequests'][0]['SpotInstanceRequestId']
waiter = ec2.get_waiter('spot_instance_request_fulfilled')
waiter.wait(SpotInstanceRequestIds=[request_id])
response = ec2.describe_spot_instance_requests(
SpotInstanceRequestIds=[request_id]
)
instance_id = response['SpotInstanceRequests'][0]['InstanceId']
waiter = ec2.get_waiter('instance_running')
waiter.wait(InstanceIds=[instance_id])
click.echo(f'Spot Instance ID: {instance_id}')
ec2.create_tags(Resources=[instance_id],
Tags=[{'Key': 'Name', 'Value': name}])
response = ec2.describe_instances(
InstanceIds=[instance_id],
Filters=[{'Name': 'instance-state-name', 'Values': ['running']}]
)
instance_ip = (
response['Reservations'][0]['Instances'][0]['PublicIpAddress']
)
click.echo(f'Spot Instance IP: {instance_ip}')
mount_point = mount_point or config.mount_point
if mount_point in ('', None):
# Mount point is not defined we don't know where to mount the volume
return
ec2 = boto.resource('ec2', config)
# Search for a volume with the same name
volume = get_volume(config, name)
if volume is not None:
click.echo(f'Volume "{name}" found - attaching')
# Attach the volume
volume_attach(config, name, name, device='xvdh')
run(config, name, f'sudo mount /dev/xvdh {mount_point}')
else:
if snapshot_name is not None:
snapshot = get_snapshot(config, snapshot_name)
elif config.snapshot_id not in ('', None):
snapshot = ec2.Snapshot(config.snapshot_id)
snapshot_name = name_from_tags(snapshot.tags)
else:
# Snapshot not found return
return
click.echo(f'Creating volume "{name}" from snapshot "{snapshot_name}"')
volume_create(config, name, size=snapshot.volume_size,
snapshot_name=snapshot_name, wait=True)
click.echo(f'Attaching volume "{name}"')
volume_attach(config, name, name, device='xvdh')
run(config, name, f'sudo mount /dev/xvdh {mount_point}')
def start(config, name, ami_id, instance_type, ebs_size=128):
"""Start an instance.
If an instance with this name already exists and it's stopped, it will just
start the instance. If the instance does not exist, it will start it.
Args:
config (aws_ml_helper.config.Config): Configuration
name (str): Instance name
ami_id (str): AMI id to use. If not provided, value form the
configuration will be used
instance_type (str): Instance type to use. If not provided, value from
the configuration will be used.
ebs_size (int): Size of the EBS Volume in GB
"""
ec2 = boto.resource('ec2', config)
instance = get_instance(config, name)
if instance is None:
# Create an instance
instance_list = ec2.create_instances(
ImageId=ami_id or config.ami_id,
InstanceType=instance_type or config.instance_type,
KeyName=f'access-key-{config.vpc_name}',
MinCount=1,
MaxCount=1,
BlockDeviceMappings=[
{
'DeviceName': '/dev/sda1',
'Ebs': {
'DeleteOnTermination': True,
'VolumeSize': ebs_size,
'VolumeType': 'gp2'
},
},
],
Monitoring={'Enabled': True},
Placement={
'AvailabilityZone': config.availability_zone,
},
InstanceInitiatedShutdownBehavior='stop',
NetworkInterfaces=[
{
'DeviceIndex': 0,
'AssociatePublicIpAddress': True,
'Groups': [config.ec2_security_group_id],
'SubnetId': config.subnet_id
},
],
TagSpecifications=[
{
'ResourceType': 'instance',
'Tags': [{
'Key': 'Name',
'Value': name
}]
},
],
)
instance = instance_list[0]
else:
# Start an instance
instance.start()
# Wait for the instance
instance.wait_until_running()
print(f'Instance ID: {instance.id}')