def _GetDefaultImage(cls, machine_type, region):
"""Returns the default image given the machine type and region.
If no default is configured, this will return None.
"""
if cls.IMAGE_NAME_FILTER is None:
return None
prefix = machine_type.split('.')[0]
virt_type = 'paravirtual' if prefix in NON_HVM_PREFIXES else 'hvm'
describe_cmd = util.AWS_PREFIX + [
'--region=%s' % region, 'ec2', 'describe-images', '--query',
'Images[*].{Name:Name,ImageId:ImageId}', '--filters',
'Name=name,Values=%s' % cls.IMAGE_NAME_FILTER,
'Name=block-device-mapping.volume-type,Values=%s' %
cls.DEFAULT_ROOT_DISK_TYPE,
'Name=virtualization-type,Values=%s' % virt_type
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
if not stdout:
return None
images = json.loads(stdout)
# We want to return the latest version of the image, and since the wildcard
# portion of the image name is the image's creation date, we can just take
# the image with the 'largest' name.
return max(images, key=lambda image: image['Name'])['ImageId']
def Attach(self, vm):
"""Attaches the disk to a VM.
Args:
vm: The AwsVirtualMachine instance to which the disk will be attached.
"""
with self._lock:
self.attached_vm_id = vm.id
if self.attached_vm_id not in AwsDisk.vm_devices:
AwsDisk.vm_devices[self.attached_vm_id] = set(
string.ascii_lowercase)
self.device_letter = min(AwsDisk.vm_devices[self.attached_vm_id])
AwsDisk.vm_devices[self.attached_vm_id].remove(self.device_letter)
attach_cmd = util.AWS_PREFIX + [
'ec2', 'attach-volume',
'--region=%s' % self.region,
'--instance-id=%s' % self.attached_vm_id,
'--volume-id=%s' % self.id,
'--device=%s' % self.GetDevicePath()
]
logging.info(
'Attaching AWS volume %s. This may fail if the disk is not '
'ready, but will be retried.', self.id)
util.IssueRetryableCommand(attach_cmd)
def CreateRoute(self, internet_gateway_id):
"""Adds a route to the internet gateway."""
create_cmd = util.AWS_PREFIX + [
'ec2', 'create-route',
'--region=%s' % self.region,
'--route-table-id=%s' % self.id,
'--gateway-id=%s' % internet_gateway_id,
'--destination-cidr-block=0.0.0.0/0'
]
util.IssueRetryableCommand(create_cmd)
def _PostCreate(self):
"""Gets data about the route table."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-route-tables',
'--region=%s' % self.region,
'--filters=Name=vpc-id,Values=%s' % self.vpc_id
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
self.id = response['RouteTables'][0]['RouteTableId']
def Detach(self):
"""Detaches the internetgateway from the VPC."""
if self.attached:
detach_cmd = util.AWS_PREFIX + [
'ec2', 'detach-internet-gateway',
'--region=%s' % self.region,
'--internet-gateway-id=%s' % self.id,
'--vpc-id=%s' % self.vpc_id
]
util.IssueRetryableCommand(detach_cmd)
self.attached = False
def _Exists(self):
"""Returns true if the subnet exists."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-subnets',
'--region=%s' % self.region,
'--filter=Name=subnet-id,Values=%s' % self.id
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
subnets = response['Subnets']
assert len(subnets) < 2, 'Too many subnets.'
return len(subnets) > 0
def _Exists(self):
"""Returns true if the Placement Group exists."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-placement-groups',
'--region=%s' % self.region,
'--filter=Name=group-name,Values=%s' % self.name
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
placement_groups = response['PlacementGroups']
assert len(placement_groups) < 2, 'Too many placement groups.'
return len(placement_groups) > 0
def Attach(self, vpc_id):
"""Attaches the internetgateway to the VPC."""
if not self.attached:
self.vpc_id = vpc_id
attach_cmd = util.AWS_PREFIX + [
'ec2', 'attach-internet-gateway',
'--region=%s' % self.region,
'--internet-gateway-id=%s' % self.id,
'--vpc-id=%s' % self.vpc_id
]
util.IssueRetryableCommand(attach_cmd)
self.attached = True
def _Exists(self):
"""Returns true if the internet gateway exists."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-internet-gateways',
'--region=%s' % self.region,
'--filter=Name=internet-gateway-id,Values=%s' % self.id
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
internet_gateways = response['InternetGateways']
assert len(internet_gateways) < 2, 'Too many internet gateways.'
return len(internet_gateways) > 0
def _Exists(self):
"""Returns true if the VPC exists."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-vpcs',
'--region=%s' % self.region,
'--filter=Name=vpc-id,Values=%s' % self.id
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
vpcs = response['Vpcs']
assert len(vpcs) < 2, 'Too many VPCs.'
return len(vpcs) > 0
def DeleteKeyfile(self):
"""Deletes the imported keyfile for a region."""
with self._lock:
if self.region in self.deleted_keyfile_set:
return
delete_cmd = util.AWS_PREFIX + [
'ec2',
'--region=%s' % self.region, 'delete-key-pair',
'--key-name=%s' % 'perfkit-key-%s' % FLAGS.run_uri
]
util.IssueRetryableCommand(delete_cmd)
self.deleted_keyfile_set.add(self.region)
if self.region in self.imported_keyfile_set:
self.imported_keyfile_set.remove(self.region)
def Detach(self):
"""Detaches the disk from a VM."""
detach_cmd = util.AWS_PREFIX + [
'ec2', 'detach-volume',
'--region=%s' % self.region,
'--instance-id=%s' % self.attached_vm_id,
'--volume-id=%s' % self.id
]
util.IssueRetryableCommand(detach_cmd)
with self._lock:
assert self.attached_vm_id in AwsDisk.vm_devices
AwsDisk.vm_devices[self.attached_vm_id].add(self.device_letter)
self.attached_vm_id = None
self.device_letter = None
def AllowPort(self, vm, port):
"""Opens a port on the firewall.
Args:
vm: The BaseVirtualMachine object to open the port for.
port: The local port to open.
"""
if vm.is_static:
return
entry = (port, vm.group_id)
if entry in self.firewall_set:
return
with self._lock:
if entry in self.firewall_set:
return
authorize_cmd = util.AWS_PREFIX + [
'ec2', 'authorize-security-group-ingress',
'--region=%s' % vm.region,
'--group-id=%s' % vm.group_id,
'--port=%s' % port, '--cidr=0.0.0.0/0'
]
util.IssueRetryableCommand(authorize_cmd + ['--protocol=tcp'])
util.IssueRetryableCommand(authorize_cmd + ['--protocol=udp'])
self.firewall_set.add(entry)
def _EnableDnsHostnames(self):
"""Sets the enableDnsHostnames attribute of this VPC to True.
By default, instances launched in non-default VPCs are assigned an
unresolvable hostname. This breaks the hadoop benchmark. Setting the
enableDnsHostnames attribute to 'true' on the VPC resolves this. See:
http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html
"""
enable_hostnames_command = util.AWS_PREFIX + [
'ec2', 'modify-vpc-attribute',
'--region=%s' % self.region, '--vpc-id', self.id,
'--enable-dns-hostnames', '{ "Value": true }'
]
util.IssueRetryableCommand(enable_hostnames_command)
def _Exists(self):
"""Returns true if the disk exists."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-volumes',
'--region=%s' % self.region,
'--filter=Name=volume-id,Values=%s' % self.id
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
volumes = response['Volumes']
assert len(volumes) < 2, 'Too many volumes.'
if not volumes:
return False
status = volumes[0]['State']
assert status in VOLUME_KNOWN_STATUSES, status
return status in VOLUME_EXISTS_STATUSES
def _PostCreate(self):
"""Get the instance's data and tag it."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-instances',
'--region=%s' % self.region,
'--instance-ids=%s' % self.id
]
logging.info(
'Getting instance %s public IP. This will fail until '
'a public IP is available, but will be retried.', self.id)
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
instance = response['Reservations'][0]['Instances'][0]
self.ip_address = instance['PublicIpAddress']
self.internal_ip = instance['PrivateIpAddress']
self.group_id = instance['SecurityGroups'][0]['GroupId']
util.AddDefaultTags(self.id, self.region)
def ImportKeyfile(self):
"""Imports the public keyfile to AWS."""
with self._lock:
if self.region in self.imported_keyfile_set:
return
cat_cmd = ['cat', vm_util.GetPublicKeyPath()]
keyfile, _ = vm_util.IssueRetryableCommand(cat_cmd)
import_cmd = util.AWS_PREFIX + [
'ec2',
'--region=%s' % self.region, 'import-key-pair',
'--key-name=%s' % 'perfkit-key-%s' % FLAGS.run_uri,
'--public-key-material=%s' % keyfile
]
util.IssueRetryableCommand(import_cmd)
self.imported_keyfile_set.add(self.region)
if self.region in self.deleted_keyfile_set:
self.deleted_keyfile_set.remove(self.region)
def _GetDecodedPasswordData(self):
# Retreive a base64 encoded, encrypted password for the VM.
get_password_cmd = util.AWS_PREFIX + [
'ec2', 'get-password-data',
'--region=%s' % self.region,
'--instance-id=%s' % self.id
]
stdout, _ = util.IssueRetryableCommand(get_password_cmd)
response = json.loads(stdout)
password_data = response['PasswordData']
# AWS may not populate the password data until some time after
# the VM shows as running. Simply retry until the data shows up.
if not password_data:
raise ValueError('No PasswordData in response.')
# Decode the password data.
return base64.b64decode(password_data)
def _Exists(self):
"""Returns true if the VM exists."""
describe_cmd = util.AWS_PREFIX + [
'ec2', 'describe-instances',
'--region=%s' % self.region,
'--filter=Name=instance-id,Values=%s' % self.id
]
stdout, _ = util.IssueRetryableCommand(describe_cmd)
response = json.loads(stdout)
reservations = response['Reservations']
assert len(reservations) < 2, 'Too many reservations.'
if not reservations:
return False
instances = reservations[0]['Instances']
assert len(instances) == 1, 'Wrong number of instances.'
status = instances[0]['State']['Name']
assert status in INSTANCE_KNOWN_STATUSES, status
return status in INSTANCE_EXISTS_STATUSES
