def _GetDefaultImage(cls, machine_type, region): """Returns the default image given the machine type and region. If no default is configured, this will return None. """ if cls.IMAGE_NAME_FILTER is None: return None prefix = machine_type.split('.')[0] virt_type = 'paravirtual' if prefix in NON_HVM_PREFIXES else 'hvm' describe_cmd = util.AWS_PREFIX + [ '--region=%s' % region, 'ec2', 'describe-images', '--query', 'Images[*].{Name:Name,ImageId:ImageId}', '--filters', 'Name=name,Values=%s' % cls.IMAGE_NAME_FILTER, 'Name=block-device-mapping.volume-type,Values=%s' % cls.DEFAULT_ROOT_DISK_TYPE, 'Name=virtualization-type,Values=%s' % virt_type ] stdout, _ = util.IssueRetryableCommand(describe_cmd) if not stdout: return None images = json.loads(stdout) # We want to return the latest version of the image, and since the wildcard # portion of the image name is the image's creation date, we can just take # the image with the 'largest' name. return max(images, key=lambda image: image['Name'])['ImageId']
def Attach(self, vm): """Attaches the disk to a VM. Args: vm: The AwsVirtualMachine instance to which the disk will be attached. """ with self._lock: self.attached_vm_id = vm.id if self.attached_vm_id not in AwsDisk.vm_devices: AwsDisk.vm_devices[self.attached_vm_id] = set( string.ascii_lowercase) self.device_letter = min(AwsDisk.vm_devices[self.attached_vm_id]) AwsDisk.vm_devices[self.attached_vm_id].remove(self.device_letter) attach_cmd = util.AWS_PREFIX + [ 'ec2', 'attach-volume', '--region=%s' % self.region, '--instance-id=%s' % self.attached_vm_id, '--volume-id=%s' % self.id, '--device=%s' % self.GetDevicePath() ] logging.info( 'Attaching AWS volume %s. This may fail if the disk is not ' 'ready, but will be retried.', self.id) util.IssueRetryableCommand(attach_cmd)
def CreateRoute(self, internet_gateway_id): """Adds a route to the internet gateway.""" create_cmd = util.AWS_PREFIX + [ 'ec2', 'create-route', '--region=%s' % self.region, '--route-table-id=%s' % self.id, '--gateway-id=%s' % internet_gateway_id, '--destination-cidr-block=0.0.0.0/0' ] util.IssueRetryableCommand(create_cmd)
def _PostCreate(self): """Gets data about the route table.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-route-tables', '--region=%s' % self.region, '--filters=Name=vpc-id,Values=%s' % self.vpc_id ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) self.id = response['RouteTables'][0]['RouteTableId']
def Detach(self): """Detaches the internetgateway from the VPC.""" if self.attached: detach_cmd = util.AWS_PREFIX + [ 'ec2', 'detach-internet-gateway', '--region=%s' % self.region, '--internet-gateway-id=%s' % self.id, '--vpc-id=%s' % self.vpc_id ] util.IssueRetryableCommand(detach_cmd) self.attached = False
def _Exists(self): """Returns true if the subnet exists.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-subnets', '--region=%s' % self.region, '--filter=Name=subnet-id,Values=%s' % self.id ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) subnets = response['Subnets'] assert len(subnets) < 2, 'Too many subnets.' return len(subnets) > 0
def _Exists(self): """Returns true if the Placement Group exists.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-placement-groups', '--region=%s' % self.region, '--filter=Name=group-name,Values=%s' % self.name ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) placement_groups = response['PlacementGroups'] assert len(placement_groups) < 2, 'Too many placement groups.' return len(placement_groups) > 0
def Attach(self, vpc_id): """Attaches the internetgateway to the VPC.""" if not self.attached: self.vpc_id = vpc_id attach_cmd = util.AWS_PREFIX + [ 'ec2', 'attach-internet-gateway', '--region=%s' % self.region, '--internet-gateway-id=%s' % self.id, '--vpc-id=%s' % self.vpc_id ] util.IssueRetryableCommand(attach_cmd) self.attached = True
def _Exists(self): """Returns true if the internet gateway exists.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-internet-gateways', '--region=%s' % self.region, '--filter=Name=internet-gateway-id,Values=%s' % self.id ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) internet_gateways = response['InternetGateways'] assert len(internet_gateways) < 2, 'Too many internet gateways.' return len(internet_gateways) > 0
def _Exists(self): """Returns true if the VPC exists.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-vpcs', '--region=%s' % self.region, '--filter=Name=vpc-id,Values=%s' % self.id ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) vpcs = response['Vpcs'] assert len(vpcs) < 2, 'Too many VPCs.' return len(vpcs) > 0
def DeleteKeyfile(self): """Deletes the imported keyfile for a region.""" with self._lock: if self.region in self.deleted_keyfile_set: return delete_cmd = util.AWS_PREFIX + [ 'ec2', '--region=%s' % self.region, 'delete-key-pair', '--key-name=%s' % 'perfkit-key-%s' % FLAGS.run_uri ] util.IssueRetryableCommand(delete_cmd) self.deleted_keyfile_set.add(self.region) if self.region in self.imported_keyfile_set: self.imported_keyfile_set.remove(self.region)
def Detach(self): """Detaches the disk from a VM.""" detach_cmd = util.AWS_PREFIX + [ 'ec2', 'detach-volume', '--region=%s' % self.region, '--instance-id=%s' % self.attached_vm_id, '--volume-id=%s' % self.id ] util.IssueRetryableCommand(detach_cmd) with self._lock: assert self.attached_vm_id in AwsDisk.vm_devices AwsDisk.vm_devices[self.attached_vm_id].add(self.device_letter) self.attached_vm_id = None self.device_letter = None
def AllowPort(self, vm, port): """Opens a port on the firewall. Args: vm: The BaseVirtualMachine object to open the port for. port: The local port to open. """ if vm.is_static: return entry = (port, vm.group_id) if entry in self.firewall_set: return with self._lock: if entry in self.firewall_set: return authorize_cmd = util.AWS_PREFIX + [ 'ec2', 'authorize-security-group-ingress', '--region=%s' % vm.region, '--group-id=%s' % vm.group_id, '--port=%s' % port, '--cidr=0.0.0.0/0' ] util.IssueRetryableCommand(authorize_cmd + ['--protocol=tcp']) util.IssueRetryableCommand(authorize_cmd + ['--protocol=udp']) self.firewall_set.add(entry)
def _EnableDnsHostnames(self): """Sets the enableDnsHostnames attribute of this VPC to True. By default, instances launched in non-default VPCs are assigned an unresolvable hostname. This breaks the hadoop benchmark. Setting the enableDnsHostnames attribute to 'true' on the VPC resolves this. See: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html """ enable_hostnames_command = util.AWS_PREFIX + [ 'ec2', 'modify-vpc-attribute', '--region=%s' % self.region, '--vpc-id', self.id, '--enable-dns-hostnames', '{ "Value": true }' ] util.IssueRetryableCommand(enable_hostnames_command)
def _Exists(self): """Returns true if the disk exists.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-volumes', '--region=%s' % self.region, '--filter=Name=volume-id,Values=%s' % self.id ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) volumes = response['Volumes'] assert len(volumes) < 2, 'Too many volumes.' if not volumes: return False status = volumes[0]['State'] assert status in VOLUME_KNOWN_STATUSES, status return status in VOLUME_EXISTS_STATUSES
def _PostCreate(self): """Get the instance's data and tag it.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-instances', '--region=%s' % self.region, '--instance-ids=%s' % self.id ] logging.info( 'Getting instance %s public IP. This will fail until ' 'a public IP is available, but will be retried.', self.id) stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) instance = response['Reservations'][0]['Instances'][0] self.ip_address = instance['PublicIpAddress'] self.internal_ip = instance['PrivateIpAddress'] self.group_id = instance['SecurityGroups'][0]['GroupId'] util.AddDefaultTags(self.id, self.region)
def ImportKeyfile(self): """Imports the public keyfile to AWS.""" with self._lock: if self.region in self.imported_keyfile_set: return cat_cmd = ['cat', vm_util.GetPublicKeyPath()] keyfile, _ = vm_util.IssueRetryableCommand(cat_cmd) import_cmd = util.AWS_PREFIX + [ 'ec2', '--region=%s' % self.region, 'import-key-pair', '--key-name=%s' % 'perfkit-key-%s' % FLAGS.run_uri, '--public-key-material=%s' % keyfile ] util.IssueRetryableCommand(import_cmd) self.imported_keyfile_set.add(self.region) if self.region in self.deleted_keyfile_set: self.deleted_keyfile_set.remove(self.region)
def _GetDecodedPasswordData(self): # Retreive a base64 encoded, encrypted password for the VM. get_password_cmd = util.AWS_PREFIX + [ 'ec2', 'get-password-data', '--region=%s' % self.region, '--instance-id=%s' % self.id ] stdout, _ = util.IssueRetryableCommand(get_password_cmd) response = json.loads(stdout) password_data = response['PasswordData'] # AWS may not populate the password data until some time after # the VM shows as running. Simply retry until the data shows up. if not password_data: raise ValueError('No PasswordData in response.') # Decode the password data. return base64.b64decode(password_data)
def _Exists(self): """Returns true if the VM exists.""" describe_cmd = util.AWS_PREFIX + [ 'ec2', 'describe-instances', '--region=%s' % self.region, '--filter=Name=instance-id,Values=%s' % self.id ] stdout, _ = util.IssueRetryableCommand(describe_cmd) response = json.loads(stdout) reservations = response['Reservations'] assert len(reservations) < 2, 'Too many reservations.' if not reservations: return False instances = reservations[0]['Instances'] assert len(instances) == 1, 'Wrong number of instances.' status = instances[0]['State']['Name'] assert status in INSTANCE_KNOWN_STATUSES, status return status in INSTANCE_EXISTS_STATUSES