def AddMetadata(self, **kwargs): """Adds metadata to the VM via 'gcloud compute instances add-metadata'.""" if not kwargs: return cmd = util.GcloudCommand(self, 'compute', 'instances', 'add-metadata', self.name) cmd.flags['metadata'] = util.MakeFormattedDefaultTags() if kwargs: cmd.flags['metadata'] = '{metadata},{kwargs}'.format( metadata=cmd.flags['metadata'], kwargs=util.FormatTags(kwargs)) cmd.Issue()
def _GenerateCreateCommand(self, ssh_keys_path): """Generates a command to create the VM instance. Args: ssh_keys_path: string. Path to a file containing the sshKeys metadata. Returns: GcloudCommand. gcloud command to issue in order to create the VM instance. """ args = ['compute', 'instances', 'create', self.name] cmd = util.GcloudCommand(self, *args) if self.network.subnet_resource is not None: cmd.flags['subnet'] = self.network.subnet_resource.name else: cmd.flags['network'] = self.network.network_resource.name if self.image: cmd.flags['image'] = self.image elif self.image_family: cmd.flags['image-family'] = self.image_family if self.image_project is not None: cmd.flags['image-project'] = self.image_project cmd.flags['boot-disk-auto-delete'] = True if self.boot_disk_size: cmd.flags['boot-disk-size'] = self.boot_disk_size if self.boot_disk_type: cmd.flags['boot-disk-type'] = self.boot_disk_type if self.machine_type is None: cmd.flags['custom-cpu'] = self.cpus cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib) if self.min_cpu_platform: cmd.flags['min-cpu-platform'] = self.min_cpu_platform else: cmd.flags['machine-type'] = self.machine_type if self.min_cpu_platform and 'n1-' in self.machine_type: cmd.flags['min-cpu-platform'] = self.min_cpu_platform elif self.min_cpu_platform: logging.warning('Cannot set min-cpu-platform for %s', self.machine_type) if self.gpu_count and self.machine_type and 'a2-' not in self.machine_type: # A2 machine type already has predefined GPU type and count. cmd.flags['accelerator'] = GenerateAcceleratorSpecString( self.gpu_type, self.gpu_count) cmd.flags['tags'] = ','.join(['perfkitbenchmarker'] + (self.gce_tags or [])) cmd.flags['no-restart-on-failure'] = True if self.node_group: cmd.flags['node-group'] = self.node_group.name if self.gce_shielded_secure_boot: cmd.flags['shielded-secure-boot'] = True if self.network.placement_group: self.metadata.update( self.network.placement_group.GetResourceMetadata()) cmd.flags['resource-policies'] = self.network.placement_group.name cmd.flags['maintenance-policy'] = 'TERMINATE' else: self.metadata[ 'placement_group_style'] = placement_group.PLACEMENT_GROUP_NONE metadata_from_file = {'sshKeys': ssh_keys_path} parsed_metadata_from_file = flag_util.ParseKeyValuePairs( FLAGS.gcp_instance_metadata_from_file) for key, value in six.iteritems(parsed_metadata_from_file): if key in metadata_from_file: logging.warning( 'Metadata "%s" is set internally. Cannot be overridden ' 'from command line.', key) continue metadata_from_file[key] = value cmd.flags['metadata-from-file'] = ','.join( ['%s=%s' % (k, v) for k, v in six.iteritems(metadata_from_file)]) metadata = {} metadata.update(self.boot_metadata) metadata.update(util.GetDefaultTags()) additional_metadata = {} additional_metadata.update(self.vm_metadata) additional_metadata.update( flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata)) for key, value in six.iteritems(additional_metadata): if key in metadata: logging.warning( 'Metadata "%s" is set internally. Cannot be overridden ' 'from command line.', key) continue metadata[key] = value if self.preemptible: cmd.flags['preemptible'] = True preemptible_status_bucket = ( f'gs://{FLAGS.gcp_preemptible_status_bucket}/{FLAGS.run_uri}/') self.preempt_marker = f'{preemptible_status_bucket}{self.name}' metadata.update([self._PreemptibleMetadataKeyValue()]) cmd.flags['metadata'] = util.FormatTags(metadata) # TODO(user): If GCE one day supports live migration on GPUs # this can be revised. if (FLAGS['gce_migrate_on_maintenance'].present and FLAGS.gce_migrate_on_maintenance and self.gpu_count): raise errors.Config.InvalidValue( 'Cannot set flag gce_migrate_on_maintenance on instances with GPUs, ' 'as it is not supported by GCP.') if not FLAGS.gce_migrate_on_maintenance or self.gpu_count: cmd.flags['maintenance-policy'] = 'TERMINATE' cmd.flags['local-ssd'] = ( ['interface={0}'.format(FLAGS.gce_ssd_interface)] * self.max_local_disks) if FLAGS.gcloud_scopes: cmd.flags['scopes'] = ','.join( re.split(r'[,; ]', FLAGS.gcloud_scopes)) cmd.flags['network-tier'] = self.gce_network_tier.upper() cmd.flags['labels'] = util.MakeFormattedDefaultTags() return cmd
def _Create(self): """Creates the cluster.""" cmd = self.DataprocGcloudCommand('clusters', 'create', self.cluster_id) if self.project is not None: cmd.flags['project'] = self.project if self.spec.worker_count: # The number of worker machines in the cluster cmd.flags['num-workers'] = self.spec.worker_count else: cmd.flags['single-node'] = True # Initialize applications on the dataproc cluster if self.spec.applications: logging.info('Include the requested applications') cmd.flags['optional-components'] = ','.join(self.spec.applications) # Enable component gateway for debuggability. Does not impact performance. cmd.flags['enable-component-gateway'] = True # TODO(pclay): stop ignoring spec.master_group? for role in ['worker', 'master']: # Set machine type if self.spec.worker_group.vm_spec.machine_type: self._AddToCmd(cmd, '{0}-machine-type'.format(role), self.spec.worker_group.vm_spec.machine_type) # Set boot_disk_size if self.spec.worker_group.disk_spec.disk_size: size_in_gb = '{}GB'.format( str(self.spec.worker_group.disk_spec.disk_size)) self._AddToCmd(cmd, '{0}-boot-disk-size'.format(role), size_in_gb) # Set boot_disk_type if self.spec.worker_group.disk_spec.disk_type: self._AddToCmd(cmd, '{0}-boot-disk-type'.format(role), self.spec.worker_group.disk_spec.disk_type) self.dpb_hdfs_type = disk_to_hdfs_map[ self.spec.worker_group.disk_spec.disk_type] # Set ssd count if self.spec.worker_group.vm_spec.num_local_ssds: self._AddToCmd(cmd, 'num-{0}-local-ssds'.format(role), self.spec.worker_group.vm_spec.num_local_ssds) # This will actually be used for storage self.dpb_hdfs_type = 'Local SSD' # Set zone cmd.flags['zone'] = self.dpb_service_zone if self.dpb_version: cmd.flags['image-version'] = self.dpb_version if FLAGS.gcp_dataproc_image: cmd.flags['image'] = FLAGS.gcp_dataproc_image if FLAGS.dpb_cluster_properties: cmd.flags['properties'] = ','.join(FLAGS.dpb_cluster_properties) # Ideally DpbServiceSpec would have a network spec, which we would create to # Resolve the name, but because EMR provisions its own VPC and we are # generally happy using pre-existing networks for Dataproc. Just use the # underlying flag instead. if FLAGS.gce_network_name: cmd.flags['network'] = FLAGS.gce_network_name metadata = util.GetDefaultTags() metadata.update( flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata)) cmd.flags['metadata'] = util.FormatTags(metadata) cmd.flags['labels'] = util.MakeFormattedDefaultTags() timeout = 900 # 15 min stdout, stderr, retcode = cmd.Issue(timeout=timeout, raise_on_failure=False) self._cluster_create_time = self._ParseClusterCreateTime(stdout) if retcode: util.CheckGcloudResponseKnownFailures(stderr, retcode) raise errors.Resource.CreationError(stderr)
def testFormatAndSplitAreInverses(self): tags = {'a': 'b', 'c': 'd'} self.assertEqual(tags, util.SplitTags(util.FormatTags(tags)))