def _Create(self) -> None: # creating database database_input = { 'Name': self.db_name, 'Description': '\n'.join( f'{k}={v}' for k, v in util.MakeDefaultTags().items()), } cmd = util.AWS_PREFIX + [ 'glue', 'create-database', '--database-input', json.dumps(database_input), f'--region={self.region}', ] vm_util.IssueCommand(cmd) targets = {'S3Targets': [{'Path': self.data_discovery_path}]} if self.sample_size is not None: targets['S3Targets'][0]['SampleSize'] = self.sample_size # creating crawler cmd = util.AWS_PREFIX + [ 'glue', 'create-crawler', '--name', self.crawler_name, '--role', self.role, '--database-name', self.db_name, '--targets', json.dumps(targets), '--region', self.region, '--tags', ','.join( f'{k}={v}' for k, v in util.MakeDefaultTags().items()), ] vm_util.IssueCommand(cmd)
def _Create(self): """Creates the cluster.""" cmd = [ 'aws', 'elasticache', 'create-replication-group', '--engine', 'redis', '--engine-version', self.version, '--replication-group-id', self.cluster_name, '--replication-group-description', self.cluster_name, '--region', self.redis_region, '--cache-node-type', self.node_type, '--cache-subnet-group-name', self.subnet_group_name, '--preferred-cache-cluster-a-zs', self.spec.client_vm.zone ] if self.failover_style == cloud_redis.Failover.FAILOVER_SAME_REGION: cmd += [self.failover_zone] elif self.failover_style == cloud_redis.Failover.FAILOVER_SAME_ZONE: cmd += [self.spec.client_vm.zone] if self.failover_style != cloud_redis.Failover.FAILOVER_NONE: cmd += [ '--automatic-failover-enabled', '--num-cache-clusters', '2' ] cmd += ['--tags'] cmd += util.FormatTags(util.MakeDefaultTags()) vm_util.IssueCommand(cmd)
def ImportKeyfile(cls, region): """Imports the public keyfile to AWS.""" with cls._lock: if _GetKeyfileSetKey(region) in cls.imported_keyfile_set: return cat_cmd = ['cat', vm_util.GetPublicKeyPath()] keyfile, _ = vm_util.IssueRetryableCommand(cat_cmd) formatted_tags = util.FormatTagSpecifications('key-pair', util.MakeDefaultTags()) import_cmd = util.AWS_PREFIX + [ 'ec2', '--region=%s' % region, 'import-key-pair', '--key-name=%s' % cls.GetKeyNameForRun(), '--public-key-material=%s' % keyfile, '--tag-specifications=%s' % formatted_tags, ] _, stderr, retcode = vm_util.IssueCommand( import_cmd, raise_on_failure=False) if retcode: if 'KeyPairLimitExceeded' in stderr: raise errors.Benchmarks.QuotaFailure( 'KeyPairLimitExceeded in %s: %s' % (region, stderr)) else: raise errors.Benchmarks.PrepareException(stderr) cls.imported_keyfile_set.add(_GetKeyfileSetKey(region)) if _GetKeyfileSetKey(region) in cls.deleted_keyfile_set: cls.deleted_keyfile_set.remove(_GetKeyfileSetKey(region))
def _Create(self): """Creates the control plane and worker nodes.""" tags = util.MakeDefaultTags() eksctl_flags = { 'kubeconfig': FLAGS.kubeconfig, 'managed': True, 'name': self.name, 'nodegroup-name': 'eks', 'nodes': self.num_nodes, 'nodes-min': self.min_nodes, 'nodes-max': self.max_nodes, 'node-type': self.vm_config.machine_type, 'node-volume-size': self.vm_config.boot_disk_size, 'region': self.region, 'tags': ','.join('{}={}'.format(k, v) for k, v in tags.items()), 'ssh-public-key': aws_virtual_machine.AwsKeyFileManager.GetKeyNameForRun(), 'version': self.cluster_version, # NAT mode uses an EIP. 'vpc-nat-mode': 'Disable', 'zones': ','.join(self.zones), } cmd = [FLAGS.eksctl, 'create', 'cluster'] + sorted( '--{}={}'.format(k, v) for k, v in eksctl_flags.items() if v) vm_util.IssueCommand(cmd, timeout=1800)
def _Create(self): """Creates the Placement Group.""" formatted_tags = util.FormatTagSpecifications('placement-group', util.MakeDefaultTags()) create_cmd = util.AWS_PREFIX + [ 'ec2', 'create-placement-group', '--region=%s' % self.region, '--group-name=%s' % self.name, '--strategy=%s' % self.strategy, '--tag-specifications=%s' % formatted_tags ] vm_util.IssueCommand(create_cmd)
def _Create(self): """Creates the cluster.""" name = 'pkb_' + FLAGS.run_uri logs_bucket = FLAGS.aws_emr_loguri or self._CreateLogBucket() instance_groups = [] for group_type, group_spec in [('CORE', self.spec.worker_group), ('MASTER', self.spec.master_group)]: instance_properties = { 'InstanceCount': group_spec.vm_count, 'InstanceGroupType': group_type, 'InstanceType': group_spec.vm_spec.machine_type, 'Name': group_type + ' group' } if group_spec.disk_spec: # Make sure nothing we are ignoring is included in the disk spec assert group_spec.disk_spec.device_path is None assert group_spec.disk_spec.disk_number is None assert group_spec.disk_spec.mount_point is None assert group_spec.disk_spec.iops is None ebs_configuration = { 'EbsBlockDeviceConfigs': [{ 'VolumeSpecification': { 'SizeInGB': group_spec.disk_spec.disk_size, 'VolumeType': group_spec.disk_spec.disk_type }, 'VolumesPerInstance': group_spec.disk_spec.num_striped_disks }] } instance_properties.update( {'EbsConfiguration': ebs_configuration}) instance_groups.append(instance_properties) # we need to store the cluster id. cmd = self.cmd_prefix + [ 'emr', 'create-cluster', '--name', name, '--release-label', RELEASE_LABEL, '--use-default-roles', '--instance-groups', json.dumps(instance_groups), '--application', 'Name=Spark', 'Name=Hadoop', '--log-uri', logs_bucket ] if self.network: cmd += ['--ec2-attributes', 'SubnetId=' + self.network.subnet.id] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) self.cluster_id = result['ClusterId'] logging.info('Cluster created with id %s', self.cluster_id) for tag_key, tag_value in util.MakeDefaultTags().items(): self._AddTag(tag_key, tag_value)
def _GetNodeFlags(self, node_group: str, num_nodes: int, vm_config) -> Dict[str, Any]: """Get common flags for creating clusters and node_groups.""" tags = util.MakeDefaultTags() return { 'nodes': num_nodes, 'node-labels': f'pkb_nodepool={node_group}', 'node-type': vm_config.machine_type, 'node-volume-size': vm_config.boot_disk_size, 'region': self.region, 'tags': ','.join(f'{k}={v}' for k, v in tags.items()), 'ssh-public-key': aws_virtual_machine.AwsKeyFileManager.GetKeyNameForRun(), }
def _Create(self): """Creates the cluster.""" name = 'pkb_' + FLAGS.run_uri # Set up ebs details if disk_spec is present in the config ebs_configuration = None if self.spec.worker_group.disk_spec: # Make sure nothing we are ignoring is included in the disk spec assert self.spec.worker_group.disk_spec.device_path is None assert self.spec.worker_group.disk_spec.disk_number is None assert self.spec.worker_group.disk_spec.iops is None self.dpb_hdfs_type = disk_to_hdfs_map[ self.spec.worker_group.disk_spec.disk_type] if self.spec.worker_group.disk_spec.disk_type != disk.LOCAL: ebs_configuration = { 'EbsBlockDeviceConfigs': [{ 'VolumeSpecification': { 'SizeInGB': self.spec.worker_group.disk_spec.disk_size, 'VolumeType': self.spec.worker_group.disk_spec.disk_type }, 'VolumesPerInstance': self.spec.worker_group.disk_count }] } # Create the specification for the master and the worker nodes instance_groups = [] core_instances = { 'InstanceCount': self.spec.worker_count, 'InstanceGroupType': 'CORE', 'InstanceType': self.spec.worker_group.vm_spec.machine_type } if ebs_configuration: core_instances.update({'EbsConfiguration': ebs_configuration}) master_instance = { 'InstanceCount': 1, 'InstanceGroupType': 'MASTER', 'InstanceType': self.spec.worker_group.vm_spec.machine_type } if ebs_configuration: master_instance.update({'EbsConfiguration': ebs_configuration}) instance_groups.append(core_instances) instance_groups.append(master_instance) # Spark SQL needs to access Hive cmd = self.cmd_prefix + [ 'emr', 'create-cluster', '--name', name, '--release-label', self.dpb_version, '--use-default-roles', '--instance-groups', json.dumps(instance_groups), '--application', 'Name=Spark', 'Name=Hadoop', 'Name=Hive', '--log-uri', self.base_dir ] ec2_attributes = [ 'KeyName=' + aws_virtual_machine.AwsKeyFileManager.GetKeyNameForRun(), 'SubnetId=' + self.network.subnet.id, # Place all VMs in default security group for simplicity and speed of # provisioning 'EmrManagedMasterSecurityGroup=' + self.security_group_id, 'EmrManagedSlaveSecurityGroup=' + self.security_group_id, ] cmd += ['--ec2-attributes', ','.join(ec2_attributes)] if FLAGS.dpb_cluster_properties: cmd += ['--configurations', _GetClusterConfiguration()] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) self.cluster_id = result['ClusterId'] logging.info('Cluster created with id %s', self.cluster_id) for tag_key, tag_value in util.MakeDefaultTags().items(): self._AddTag(tag_key, tag_value)
def _Create(self): """Creates the cluster.""" name = 'pkb_' + FLAGS.run_uri # Set up ebs details if disk_spec is present int he config ebs_configuration = None if self.spec.worker_group.disk_spec: # Make sure nothing we are ignoring is included in the disk spec assert self.spec.worker_group.disk_spec.device_path is None assert self.spec.worker_group.disk_spec.disk_number is None assert self.spec.worker_group.disk_spec.iops is None ebs_configuration = { 'EbsBlockDeviceConfigs': [{ 'VolumeSpecification': { 'SizeInGB': self.spec.worker_group.disk_spec.disk_size, 'VolumeType': self.spec.worker_group.disk_spec.disk_type }, 'VolumesPerInstance': self.spec.worker_group.disk_spec.num_striped_disks }] } self.dpb_hdfs_type = disk_to_hdfs_map[ self.spec.worker_group.disk_spec.disk_type] # Create the specification for the master and the worker nodes instance_groups = [] core_instances = { 'InstanceCount': self.spec.worker_count, 'InstanceGroupType': 'CORE', 'InstanceType': self.spec.worker_group.vm_spec.machine_type } if ebs_configuration: core_instances.update({'EbsConfiguration': ebs_configuration}) master_instance = { 'InstanceCount': 1, 'InstanceGroupType': 'MASTER', 'InstanceType': self.spec.worker_group.vm_spec.machine_type } if ebs_configuration: master_instance.update({'EbsConfiguration': ebs_configuration}) instance_groups.append(core_instances) instance_groups.append(master_instance) # Create the log bucket to hold job's log output # TODO(saksena): Deprecate aws_emr_loguri flag and move # the log bucket creation to Create dependencies. logs_bucket = self._CreateLogBucket() # Spark SQL needs to access Hive cmd = self.cmd_prefix + [ 'emr', 'create-cluster', '--name', name, '--release-label', self.dpb_version, '--use-default-roles', '--instance-groups', json.dumps(instance_groups), '--application', 'Name=Spark', 'Name=Hadoop', 'Name=Hive', '--log-uri', logs_bucket ] ec2_attributes = [ 'KeyName=' + aws_virtual_machine.AwsKeyFileManager.GetKeyNameForRun() ] if self.network: ec2_attributes.append('SubnetId=' + self.network.subnet.id) cmd += ['--ec2-attributes', ','.join(ec2_attributes)] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) self.cluster_id = result['ClusterId'] logging.info('Cluster created with id %s', self.cluster_id) for tag_key, tag_value in util.MakeDefaultTags().items(): self._AddTag(tag_key, tag_value)
def _Create(self): """Create a VM instance.""" placement = [] if not util.IsRegion(self.zone): placement.append('AvailabilityZone=%s' % self.zone) if self.use_dedicated_host: placement.append('Tenancy=host,HostId=%s' % self.host.id) num_hosts = len(self.host_list) elif self.placement_group: if IsPlacementGroupCompatible(self.machine_type): placement.append('GroupName=%s' % self.placement_group.name) else: logging.warn( 'VM not placed in Placement Group. VM Type %s not supported', self.machine_type) placement = ','.join(placement) block_device_map = GetBlockDeviceMap(self.machine_type, self.boot_disk_size, self.image, self.region) tags = {} tags.update(self.vm_metadata) tags.update(util.MakeDefaultTags()) create_cmd = util.AWS_PREFIX + [ 'ec2', 'run-instances', '--region=%s' % self.region, '--subnet-id=%s' % self.network.subnet.id, '--client-token=%s' % self.client_token, '--image-id=%s' % self.image, '--instance-type=%s' % self.machine_type, '--key-name=%s' % AwsKeyFileManager.GetKeyNameForRun(), '--tag-specifications=%s' % util.FormatTagSpecifications('instance', tags)] if FLAGS.aws_efa: create_cmd.extend([ '--network-interfaces', ','.join(['%s=%s' % item for item in sorted(_EFA_PARAMS.items())]) ]) else: create_cmd.append('--associate-public-ip-address') if block_device_map: create_cmd.append('--block-device-mappings=%s' % block_device_map) if placement: create_cmd.append('--placement=%s' % placement) if FLAGS.aws_credit_specification: create_cmd.append('--credit-specification=%s' % FLAGS.aws_credit_specification) if self.user_data: create_cmd.append('--user-data=%s' % self.user_data) if self.capacity_reservation_id: create_cmd.append( '--capacity-reservation-specification=CapacityReservationTarget=' '{CapacityReservationId=%s}' % self.capacity_reservation_id) if self.use_spot_instance: instance_market_options = collections.OrderedDict() spot_options = collections.OrderedDict() spot_options['SpotInstanceType'] = 'one-time' spot_options['InstanceInterruptionBehavior'] = 'terminate' if self.spot_price: spot_options['MaxPrice'] = str(self.spot_price) instance_market_options['MarketType'] = 'spot' instance_market_options['SpotOptions'] = spot_options create_cmd.append( '--instance-market-options=%s' % json.dumps(instance_market_options)) _, stderr, retcode = vm_util.IssueCommand(create_cmd, raise_on_failure=False) machine_type_prefix = self.machine_type.split('.')[0] host_arch = _MACHINE_TYPE_PREFIX_TO_HOST_ARCH.get(machine_type_prefix) if host_arch: self.host_arch = host_arch if self.use_dedicated_host and 'InsufficientCapacityOnHost' in stderr: if self.num_vms_per_host: raise errors.Resource.CreationError( 'Failed to create host: %d vms of type %s per host exceeds ' 'memory capacity limits of the host' % (self.num_vms_per_host, self.machine_type)) else: logging.warning( 'Creation failed due to insufficient host capacity. A new host will ' 'be created and instance creation will be retried.') with self._lock: if num_hosts == len(self.host_list): host = AwsDedicatedHost(self.machine_type, self.zone) self.host_list.append(host) host.Create() self.host = self.host_list[-1] self.client_token = str(uuid.uuid4()) raise errors.Resource.RetryableCreationError() if 'InsufficientInstanceCapacity' in stderr: if self.use_spot_instance: self.spot_status_code = 'InsufficientSpotInstanceCapacity' self.early_termination = True raise errors.Benchmarks.InsufficientCapacityCloudFailure(stderr) if 'SpotMaxPriceTooLow' in stderr: self.spot_status_code = 'SpotMaxPriceTooLow' self.early_termination = True raise errors.Resource.CreationError(stderr) if 'InstanceLimitExceeded' in stderr: raise errors.Benchmarks.QuotaFailure(stderr) if retcode: raise errors.Resource.CreationError( 'Failed to create VM: %s return code: %s' % (retcode, stderr))
def _Create(self): """Creates the AWS CapacaityReservation. A reservation will be created given the VM shape in self.vm_groups. Count is determined by the number of VMs in said group. The reservation will have a lifetime determined by the general PKB concept of timeout_minutes. If the reservation exceeds this timeout, AWS will cancel it automatically. The VMs in the reservation will not be deleted. Note that an empty capacity reservation will encur costs for the VM shape / count, even if no VMs are using it. After the reservation is created, this method updates all the VMs in self.vm_groups by setting the capacity_reservation_id, as well as the zone attributes on the VM, and the VM's network instance. Raises: UnsupportedOsTypeError: If creating a capacity reservation for the given os type is not supported. CreationError: If a capacity reservation cannot be created in the region (typically indicates a stockout). """ if self.os_type in os_types.LINUX_OS_TYPES: instance_platform = 'Linux/UNIX' elif self.os_type in os_types.WINDOWS_OS_TYPES: instance_platform = 'Windows' else: raise UnsupportedOsTypeError( 'Unsupported os_type for AWS CapacityReservation: %s.' % self.os_type) # If the user did not specify an AZ, we need to try to create the # CapacityReservation in a specifc AZ until it succeeds. # Then update the zone attribute on all the VMs in the group, # as well as the zone attribute on the VMs' network instance. if util.IsRegion(self.zone_or_region): zones_to_try = util.GetZonesInRegion(self.region) else: zones_to_try = [self.zone_or_region] end_date = util.MakeDefaultTags()['timeout_utc'] for zone in zones_to_try: cmd = util.AWS_PREFIX + [ 'ec2', 'create-capacity-reservation', '--instance-type=%s' % self.machine_type, '--instance-platform=%s' % instance_platform, '--availability-zone=%s' % zone, '--instance-count=%s' % self.vm_count, '--instance-match-criteria=targeted', '--region=%s' % self.region, '--end-date-type=limited', '--end-date=%s' % end_date, ] stdout, stderr, retcode = vm_util.IssueCommand(cmd) if retcode: logging.info( 'Unable to create CapacityReservation in %s. ' 'This may be retried. Details: %s', zone, stderr) continue json_output = json.loads(stdout) self.capacity_reservation_id = ( json_output['CapacityReservation']['CapacityReservationId']) self._UpdateVmsInGroup(self.capacity_reservation_id, zone) return raise CreationError( 'Unable to create CapacityReservation in any of the ' 'following zones: %s.' % zones_to_try)
def _Create(self): """Creates the cluster.""" name = 'pkb_' + FLAGS.run_uri # Set up ebs details if disk_spec is present int he config ebs_configuration = None if self.spec.worker_group.disk_spec: # Make sure nothing we are ignoring is included in the disk spec assert self.spec.worker_group.disk_spec.device_path is None assert self.spec.worker_group.disk_spec.disk_number is None assert self.spec.worker_group.disk_spec.mount_point is None assert self.spec.worker_group.disk_spec.iops is None ebs_configuration = { 'EbsBlockDeviceConfigs': [{ 'VolumeSpecification': { 'SizeInGB': self.spec.worker_group.disk_spec.disk_size, 'VolumeType': self.spec.worker_group.disk_spec.disk_type }, 'VolumesPerInstance': self.spec.worker_group.disk_spec.num_striped_disks }] } # Create the specification for the master and the worker nodes instance_groups = [] core_instances = { 'InstanceCount': self.spec.worker_count, 'InstanceGroupType': 'CORE', 'InstanceType': self.spec.worker_group.vm_spec.machine_type } if ebs_configuration: core_instances.update({'EbsConfiguration': ebs_configuration}) master_instance = { 'InstanceCount': 1, 'InstanceGroupType': 'MASTER', 'InstanceType': self.spec.worker_group.vm_spec.machine_type } if ebs_configuration: master_instance.update({'EbsConfiguration': ebs_configuration}) instance_groups.append(core_instances) instance_groups.append(master_instance) # Create the log bucket to hold job's log output logs_bucket = FLAGS.aws_emr_loguri or self._CreateLogBucket() cmd = self.cmd_prefix + [ 'emr', 'create-cluster', '--name', name, '--release-label', self.emr_release_label, '--use-default-roles', '--instance-groups', json.dumps(instance_groups), '--application', 'Name=Spark', 'Name=Hadoop', '--log-uri', logs_bucket ] if self.network: cmd += ['--ec2-attributes', 'SubnetId=' + self.network.subnet.id] stdout, _, _ = vm_util.IssueCommand(cmd) result = json.loads(stdout) self.cluster_id = result['ClusterId'] logging.info('Cluster created with id %s', self.cluster_id) for tag_key, tag_value in util.MakeDefaultTags().items(): self._AddTag(tag_key, tag_value)
def _BuildContext(launcher_vm, booter_template_vm): """Returns the context variables for Jinja2 template during rendering.""" context = { 'boot_machine_type': booter_template_vm.machine_type, 'cloud': FLAGS.cloud, 'contact_launcher': FLAGS.vms_contact_launcher, 'launcher_vm_name': launcher_vm.name, 'os_type': 'linux' if _IsLinux() else 'windows', 'server_ip': launcher_vm.internal_ip, 'server_port': _PORT, 'start_time_file': _START_TIME_FILE_PATH, 'timeout': _TIMEOUT_SECONDS, 'vm_count': FLAGS.boots_per_launcher, 'zone': launcher_vm.zone, 'use_public_ip': '' if FLAGS.use_public_ip else 'no-', } cloud = FLAGS.cloud if cloud == 'GCP': context.update({ 'boot_disk_size': booter_template_vm.boot_disk_size, 'boot_vm_name_prefix': _BOOT_VM_NAME_PREFIX.format(launcher_name=launcher_vm.name), 'image_family': booter_template_vm.image_family, 'image_project': booter_template_vm.image_project, 'gcloud_path': FLAGS.gcloud_path, 'project': FLAGS.project, 'tags': gcp_util.MakeFormattedDefaultTags(), }) elif cloud == 'AWS': tags = aws_util.MakeDefaultTags() tags.update({'launcher_id': launcher_vm.name}) context.update({ 'group_name': booter_template_vm.placement_group.name, 'image': booter_template_vm.image, 'key_name': 'perfkit-key-{0}'.format(FLAGS.run_uri), 'region': aws_util.GetRegionFromZone(launcher_vm.zone), 'subnet_id': booter_template_vm.network.subnet.id, 'tags': aws_util.FormatTagSpecifications('instance', tags), }) elif cloud == 'Azure': context.update({ 'boot_vm_name_prefix': launcher_vm.name.split('-', 1)[1], 'location': launcher_vm.region, 'image': booter_template_vm.image, 'storage_sku': booter_template_vm.os_disk.disk_type, 'resource_group': launcher_vm.resource_group.name, 'nic': _BOOT_NIC_NAME_PREFIX.format(run_uri=FLAGS.run_uri), 'password': booter_template_vm.password, 'start_id': GetAzBootVMStartIdByLauncher(launcher_vm.name), }) return context