def _GetResourceDict(self, time_format, timeout_minutes=None): """Gets a list of tags to be used to tag resources.""" now_utc = datetime.datetime.utcnow() if not timeout_minutes: timeout_minutes = FLAGS.timeout_minutes timeout_utc = ( now_utc + datetime.timedelta(minutes=timeout_minutes)) tags = { 'timeout_utc': timeout_utc.strftime(time_format), 'create_time_utc': now_utc.strftime(time_format), 'benchmark': self.name, 'perfkit_uuid': self.uuid, 'owner': FLAGS.owner, 'benchmark_uid': self.uid, } # add metadata key value pairs metadata_dict = (flag_util.ParseKeyValuePairs(FLAGS.metadata) if hasattr(FLAGS, 'metadata') else dict()) for key, value in metadata_dict.items(): tags[self._SafeLabelKeyOrValue(key)] = self._SafeLabelKeyOrValue(value) return tags
def AddMetadata(self, metadata, benchmark_spec): metadata = metadata.copy() metadata['perfkitbenchmarker_version'] = version.VERSION if FLAGS.hostname_metadata: metadata['hostnames'] = ','.join( [vm.hostname for vm in benchmark_spec.vms]) for name, vms in benchmark_spec.vm_groups.iteritems(): if len(vms) == 0: continue # Get a representative VM so that we can publish the cloud, zone, # machine type, and image. vm = vms[-1] name_prefix = '' if name == 'default' else name + '_' metadata[name_prefix + 'cloud'] = vm.CLOUD metadata[name_prefix + 'zone'] = vm.zone metadata[name_prefix + 'image'] = vm.image for k, v in vm.GetMachineTypeDict().iteritems(): metadata[name_prefix + k] = v metadata[name_prefix + 'vm_count'] = len(vms) if vm.scratch_disks: data_disk = vm.scratch_disks[0] disk_type = data_disk.disk_type disk_size = data_disk.disk_size num_stripes = data_disk.num_striped_disks # Legacy metadata keys metadata[name_prefix + 'scratch_disk_type'] = disk_type metadata[name_prefix + 'scratch_disk_size'] = disk_size metadata[name_prefix + 'num_striped_disks'] = (data_disk.num_striped_disks) if getattr(data_disk, 'iops', None) is not None: metadata[name_prefix + 'scratch_disk_iops'] = data_disk.iops metadata[name_prefix + 'aws_provisioned_iops'] = data_disk.iops # Modern metadata keys metadata[name_prefix + 'data_disk_0_type'] = disk_type metadata[name_prefix + 'data_disk_0_size'] = (disk_size * num_stripes if disk_size else disk_size) metadata[name_prefix + 'data_disk_0_num_stripes'] = num_stripes if getattr(data_disk, 'metadata', None) is not None: if disk.LEGACY_DISK_TYPE in data_disk.metadata: metadata[name_prefix + 'scratch_disk_type'] = ( data_disk.metadata[disk.LEGACY_DISK_TYPE]) for key, value in data_disk.metadata.iteritems(): metadata[name_prefix + 'data_disk_0_%s' % (key, )] = value if FLAGS.set_files: metadata['set_files'] = ','.join(FLAGS.set_files) if FLAGS.sysctl: metadata['sysctl'] = ','.join(FLAGS.sysctl) # Flatten all user metadata into a single list (since each string in the # FLAGS.metadata can actually be several key-value pairs) and then iterate # over it. parsed_metadata = flag_util.ParseKeyValuePairs(FLAGS.metadata) metadata.update(parsed_metadata) return metadata
def AddMetadata(self, metadata, benchmark_spec): metadata = metadata.copy() metadata['perfkitbenchmarker_version'] = version.VERSION if FLAGS.simulate_maintenance: metadata['simulate_maintenance'] = True if FLAGS.hostname_metadata: metadata['hostnames'] = ','.join( [vm.hostname for vm in benchmark_spec.vms]) if benchmark_spec.container_cluster: cluster = benchmark_spec.container_cluster for k, v in cluster.GetResourceMetadata().iteritems(): metadata['container_cluster_' + k] = v if benchmark_spec.managed_relational_db: managed_db = benchmark_spec.managed_relational_db for k, v in managed_db.GetResourceMetadata().iteritems(): metadata['managed_relational_db_' + k] = v if benchmark_spec.cloud_tpu: cloud_tpu = benchmark_spec.cloud_tpu for k, v in cloud_tpu.GetResourceMetadata().iteritems(): metadata['cloud_tpu_' + k] = v if benchmark_spec.cloud_redis: cloud_redis = benchmark_spec.cloud_redis for k, v in cloud_redis.GetResourceMetadata().iteritems(): metadata['cloud_redis_' + k] = v for name, vms in benchmark_spec.vm_groups.iteritems(): if len(vms) == 0: continue # Get a representative VM so that we can publish the cloud, zone, # machine type, and image. vm = vms[-1] name_prefix = '' if name == 'default' else name + '_' for k, v in vm.GetResourceMetadata().iteritems(): metadata[name_prefix + k] = v metadata[name_prefix + 'vm_count'] = len(vms) for k, v in vm.GetOSResourceMetadata().iteritems(): metadata[name_prefix + k] = v if vm.scratch_disks: data_disk = vm.scratch_disks[0] metadata[name_prefix + 'data_disk_count'] = len( vm.scratch_disks) for key, value in data_disk.GetResourceMetadata().iteritems(): metadata[name_prefix + 'data_disk_0_%s' % (key, )] = value if FLAGS.set_files: metadata['set_files'] = ','.join(FLAGS.set_files) if FLAGS.sysctl: metadata['sysctl'] = ','.join(FLAGS.sysctl) # Flatten all user metadata into a single list (since each string in the # FLAGS.metadata can actually be several key-value pairs) and then iterate # over it. parsed_metadata = flag_util.ParseKeyValuePairs(FLAGS.metadata) metadata.update(parsed_metadata) return metadata
def _ApplyFlags(cls, config_values, flag_values): """Overrides config values with flag values. Can be overridden by derived classes to add support for specific flags. Args: config_values: dict mapping config option names to provided values. Is modified by this function. flag_values: flags.FlagValues. Runtime flags that may override the provided config values. Returns: dict mapping config option names to values derived from the config values or flag values. """ super(KubernetesDiskSpec, cls)._ApplyFlags(config_values, flag_values) if flag_values['k8s_volume_provisioner'].present: config_values['provisioner'] = flag_values.k8s_volume_provisioner if flag_values['k8s_volume_parameters'].present: config_values['parameters'] = config_values.get('parameters', {}) config_values['parameters'].update( flag_util.ParseKeyValuePairs(flag_values.k8s_volume_parameters))
def _GenerateCreateCommand(self, ssh_keys_path): """Generates a command to create the VM instance. Args: ssh_keys_path: string. Path to a file containing the sshKeys metadata. Returns: GcloudCommand. gcloud command to issue in order to create the VM instance. """ args = [] if self.node_group: args = ['alpha'] args.extend(['compute', 'instances', 'create', self.name]) cmd = util.GcloudCommand(self, *args) if self.network.subnet_resource is not None: cmd.flags['subnet'] = self.network.subnet_resource.name else: cmd.flags['network'] = self.network.network_resource.name if self.image: cmd.flags['image'] = self.image elif self.image_family: cmd.flags['image-family'] = self.image_family if self.image_project is not None: cmd.flags['image-project'] = self.image_project cmd.flags['boot-disk-auto-delete'] = True cmd.flags['boot-disk-size'] = self.boot_disk_size or self.BOOT_DISK_SIZE_GB cmd.flags['boot-disk-type'] = self.boot_disk_type or self.BOOT_DISK_TYPE if self.machine_type is None: cmd.flags['custom-cpu'] = self.cpus cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib) else: cmd.flags['machine-type'] = self.machine_type if self.gpu_count: cmd.flags['accelerator'] = GenerateAcceleratorSpecString(self.gpu_type, self.gpu_count) cmd.flags['tags'] = 'perfkitbenchmarker' cmd.flags['no-restart-on-failure'] = True if self.node_group: cmd.flags['node-group'] = self.node_group.name if self.min_cpu_platform: cmd.flags['min-cpu-platform'] = self.min_cpu_platform metadata_from_file = {'sshKeys': ssh_keys_path} parsed_metadata_from_file = flag_util.ParseKeyValuePairs( FLAGS.gcp_instance_metadata_from_file) for key, value in parsed_metadata_from_file.iteritems(): if key in metadata_from_file: logging.warning('Metadata "%s" is set internally. Cannot be overridden ' 'from command line.', key) continue metadata_from_file[key] = value cmd.flags['metadata-from-file'] = ','.join([ '%s=%s' % (k, v) for k, v in metadata_from_file.iteritems() ]) metadata = {'owner': FLAGS.owner} if FLAGS.owner else {} metadata.update(self.boot_metadata) parsed_metadata = flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata) for key, value in parsed_metadata.iteritems(): if key in metadata: logging.warning('Metadata "%s" is set internally. Cannot be overridden ' 'from command line.', key) continue metadata[key] = value cmd.flags['metadata'] = ','.join( ['%s=%s' % (k, v) for k, v in metadata.iteritems()]) # TODO(gareth-ferneyhough): If GCE one day supports live migration on GPUs # this can be revised. if (FLAGS['gce_migrate_on_maintenance'].present and FLAGS.gce_migrate_on_maintenance and self.gpu_count): raise errors.Config.InvalidValue( 'Cannot set flag gce_migrate_on_maintenance on instances with GPUs, ' 'as it is not supported by GCP.') if not FLAGS.gce_migrate_on_maintenance or self.gpu_count: cmd.flags['maintenance-policy'] = 'TERMINATE' ssd_interface_option = FLAGS.gce_ssd_interface cmd.flags['local-ssd'] = (['interface={0}'.format(ssd_interface_option)] * self.max_local_disks) if FLAGS.gcloud_scopes: cmd.flags['scopes'] = ','.join(re.split(r'[,; ]', FLAGS.gcloud_scopes)) if self.preemptible: cmd.flags['preemptible'] = True return cmd
def _GenerateCreateCommand(self, ssh_keys_path): """Generates a command to create the VM instance. Args: ssh_keys_path: string. Path to a file containing the sshKeys metadata. Returns: GcloudCommand. gcloud command to issue in order to create the VM instance. """ args = ['compute', 'instances', 'create', self.name] cmd = util.GcloudCommand(self, *args) if self.network.subnet_resource is not None: cmd.flags['subnet'] = self.network.subnet_resource.name else: cmd.flags['network'] = self.network.network_resource.name if self.image: cmd.flags['image'] = self.image elif self.image_family: cmd.flags['image-family'] = self.image_family if self.image_project is not None: cmd.flags['image-project'] = self.image_project cmd.flags['boot-disk-auto-delete'] = True if self.boot_disk_size: cmd.flags['boot-disk-size'] = self.boot_disk_size if self.boot_disk_type: cmd.flags['boot-disk-type'] = self.boot_disk_type if self.machine_type is None: cmd.flags['custom-cpu'] = self.cpus cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib) if self.min_cpu_platform: cmd.flags['min-cpu-platform'] = self.min_cpu_platform else: cmd.flags['machine-type'] = self.machine_type if self.min_cpu_platform and 'n1-' in self.machine_type: cmd.flags['min-cpu-platform'] = self.min_cpu_platform elif self.min_cpu_platform: logging.warning('Cannot set min-cpu-platform for %s', self.machine_type) if self.gpu_count and self.machine_type and 'a2-' not in self.machine_type: # A2 machine type already has predefined GPU type and count. cmd.flags['accelerator'] = GenerateAcceleratorSpecString( self.gpu_type, self.gpu_count) cmd.flags['tags'] = ','.join(['perfkitbenchmarker'] + (self.gce_tags or [])) cmd.flags['no-restart-on-failure'] = True if self.node_group: cmd.flags['node-group'] = self.node_group.name if self.gce_shielded_secure_boot: cmd.flags['shielded-secure-boot'] = True if self.network.placement_group: self.metadata.update( self.network.placement_group.GetResourceMetadata()) cmd.flags['resource-policies'] = self.network.placement_group.name cmd.flags['maintenance-policy'] = 'TERMINATE' else: self.metadata[ 'placement_group_style'] = placement_group.PLACEMENT_GROUP_NONE metadata_from_file = {'sshKeys': ssh_keys_path} parsed_metadata_from_file = flag_util.ParseKeyValuePairs( FLAGS.gcp_instance_metadata_from_file) for key, value in six.iteritems(parsed_metadata_from_file): if key in metadata_from_file: logging.warning( 'Metadata "%s" is set internally. Cannot be overridden ' 'from command line.', key) continue metadata_from_file[key] = value cmd.flags['metadata-from-file'] = ','.join( ['%s=%s' % (k, v) for k, v in six.iteritems(metadata_from_file)]) metadata = {} metadata.update(self.boot_metadata) metadata.update(util.GetDefaultTags()) additional_metadata = {} additional_metadata.update(self.vm_metadata) additional_metadata.update( flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata)) for key, value in six.iteritems(additional_metadata): if key in metadata: logging.warning( 'Metadata "%s" is set internally. Cannot be overridden ' 'from command line.', key) continue metadata[key] = value if self.preemptible: cmd.flags['preemptible'] = True preemptible_status_bucket = ( f'gs://{FLAGS.gcp_preemptible_status_bucket}/{FLAGS.run_uri}/') self.preempt_marker = f'{preemptible_status_bucket}{self.name}' metadata.update([self._PreemptibleMetadataKeyValue()]) cmd.flags['metadata'] = util.FormatTags(metadata) # TODO(user): If GCE one day supports live migration on GPUs # this can be revised. if (FLAGS['gce_migrate_on_maintenance'].present and FLAGS.gce_migrate_on_maintenance and self.gpu_count): raise errors.Config.InvalidValue( 'Cannot set flag gce_migrate_on_maintenance on instances with GPUs, ' 'as it is not supported by GCP.') if not FLAGS.gce_migrate_on_maintenance or self.gpu_count: cmd.flags['maintenance-policy'] = 'TERMINATE' cmd.flags['local-ssd'] = ( ['interface={0}'.format(FLAGS.gce_ssd_interface)] * self.max_local_disks) if FLAGS.gcloud_scopes: cmd.flags['scopes'] = ','.join( re.split(r'[,; ]', FLAGS.gcloud_scopes)) cmd.flags['network-tier'] = self.gce_network_tier.upper() cmd.flags['labels'] = util.MakeFormattedDefaultTags() return cmd
def _GenerateCreateCommand(self, ssh_keys_path): """Generates a command to create the VM instance. Args: ssh_keys_path: string. Path to a file containing the sshKeys metadata. Returns: GcloudCommand. gcloud command to issue in order to create the VM instance. """ cmd = util.GcloudCommand(self, 'compute', 'instances', 'create', self.name) cmd.flags['network'] = self.network.network_resource.name cmd.flags['image'] = self.image cmd.flags['boot-disk-auto-delete'] = True if FLAGS.image_project: cmd.flags['image-project'] = FLAGS.image_project cmd.flags['boot-disk-size'] = self.BOOT_DISK_SIZE_GB cmd.flags['boot-disk-type'] = self.BOOT_DISK_TYPE if self.machine_type is None: cmd.flags['custom-cpu'] = self.cpus cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib) else: cmd.flags['machine-type'] = self.machine_type cmd.flags['tags'] = 'perfkitbenchmarker' cmd.flags['no-restart-on-failure'] = True metadata_from_file = {'sshKeys': ssh_keys_path} parsed_metadata_from_file = flag_util.ParseKeyValuePairs( FLAGS.gcp_instance_metadata_from_file) for key, value in parsed_metadata_from_file.iteritems(): if key in metadata_from_file: logging.warning( 'Metadata "%s" is set internally. Cannot be overridden ' 'from command line.' % key) continue metadata_from_file[key] = value cmd.flags['metadata-from-file'] = ','.join( ['%s=%s' % (k, v) for k, v in metadata_from_file.iteritems()]) metadata = {'owner': FLAGS.owner} if FLAGS.owner else {} metadata.update(self.boot_metadata) parsed_metadata = flag_util.ParseKeyValuePairs( FLAGS.gcp_instance_metadata) for key, value in parsed_metadata.iteritems(): if key in metadata: logging.warning( 'Metadata "%s" is set internally. Cannot be overridden ' 'from command line.' % key) continue metadata[key] = value cmd.flags['metadata'] = ','.join( ['%s=%s' % (k, v) for k, v in metadata.iteritems()]) if not FLAGS.gce_migrate_on_maintenance: cmd.flags['maintenance-policy'] = 'TERMINATE' ssd_interface_option = NVME if NVME in self.image else SCSI cmd.flags['local-ssd'] = ( ['interface={0}'.format(ssd_interface_option)] * self.max_local_disks) if FLAGS.gcloud_scopes: cmd.flags['scopes'] = ','.join( re.split(r'[,; ]', FLAGS.gcloud_scopes)) if self.preemptible: cmd.flags['preemptible'] = True return cmd
def _Create(self): """Creates the cluster.""" cmd = self.DataprocGcloudCommand('clusters', 'create', self.cluster_id) if self.project is not None: cmd.flags['project'] = self.project if self.spec.worker_count: # The number of worker machines in the cluster cmd.flags['num-workers'] = self.spec.worker_count else: cmd.flags['single-node'] = True # Initialize applications on the dataproc cluster if self.spec.applications: logging.info('Include the requested applications') cmd.flags['optional-components'] = ','.join(self.spec.applications) # Enable component gateway for debuggability. Does not impact performance. cmd.flags['enable-component-gateway'] = True # TODO(pclay): stop ignoring spec.master_group? for role in ['worker', 'master']: # Set machine type if self.spec.worker_group.vm_spec.machine_type: self._AddToCmd(cmd, '{0}-machine-type'.format(role), self.spec.worker_group.vm_spec.machine_type) # Set boot_disk_size if self.spec.worker_group.disk_spec.disk_size: size_in_gb = '{}GB'.format( str(self.spec.worker_group.disk_spec.disk_size)) self._AddToCmd(cmd, '{0}-boot-disk-size'.format(role), size_in_gb) # Set boot_disk_type if self.spec.worker_group.disk_spec.disk_type: self._AddToCmd(cmd, '{0}-boot-disk-type'.format(role), self.spec.worker_group.disk_spec.disk_type) self.dpb_hdfs_type = disk_to_hdfs_map[ self.spec.worker_group.disk_spec.disk_type] # Set ssd count if self.spec.worker_group.vm_spec.num_local_ssds: self._AddToCmd(cmd, 'num-{0}-local-ssds'.format(role), self.spec.worker_group.vm_spec.num_local_ssds) # This will actually be used for storage self.dpb_hdfs_type = 'Local SSD' # Set zone cmd.flags['zone'] = self.dpb_service_zone if self.dpb_version: cmd.flags['image-version'] = self.dpb_version if FLAGS.gcp_dataproc_image: cmd.flags['image'] = FLAGS.gcp_dataproc_image if FLAGS.dpb_cluster_properties: cmd.flags['properties'] = ','.join(FLAGS.dpb_cluster_properties) # Ideally DpbServiceSpec would have a network spec, which we would create to # Resolve the name, but because EMR provisions its own VPC and we are # generally happy using pre-existing networks for Dataproc. Just use the # underlying flag instead. if FLAGS.gce_network_name: cmd.flags['network'] = FLAGS.gce_network_name metadata = util.GetDefaultTags() metadata.update( flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata)) cmd.flags['metadata'] = util.FormatTags(metadata) cmd.flags['labels'] = util.MakeFormattedDefaultTags() timeout = 900 # 15 min stdout, stderr, retcode = cmd.Issue(timeout=timeout, raise_on_failure=False) self._cluster_create_time = self._ParseClusterCreateTime(stdout) if retcode: util.CheckGcloudResponseKnownFailures(stderr, retcode) raise errors.Resource.CreationError(stderr)