def _GetResourceDict(self, time_format, timeout_minutes=None):
"""Gets a list of tags to be used to tag resources."""
now_utc = datetime.datetime.utcnow()
if not timeout_minutes:
timeout_minutes = FLAGS.timeout_minutes
timeout_utc = (
now_utc +
datetime.timedelta(minutes=timeout_minutes))
tags = {
'timeout_utc': timeout_utc.strftime(time_format),
'create_time_utc': now_utc.strftime(time_format),
'benchmark': self.name,
'perfkit_uuid': self.uuid,
'owner': FLAGS.owner,
'benchmark_uid': self.uid,
}
# add metadata key value pairs
metadata_dict = (flag_util.ParseKeyValuePairs(FLAGS.metadata)
if hasattr(FLAGS, 'metadata') else dict())
for key, value in metadata_dict.items():
tags[self._SafeLabelKeyOrValue(key)] = self._SafeLabelKeyOrValue(value)
return tags
def AddMetadata(self, metadata, benchmark_spec):
metadata = metadata.copy()
metadata['perfkitbenchmarker_version'] = version.VERSION
if FLAGS.hostname_metadata:
metadata['hostnames'] = ','.join(
[vm.hostname for vm in benchmark_spec.vms])
for name, vms in benchmark_spec.vm_groups.iteritems():
if len(vms) == 0:
continue
# Get a representative VM so that we can publish the cloud, zone,
# machine type, and image.
vm = vms[-1]
name_prefix = '' if name == 'default' else name + '_'
metadata[name_prefix + 'cloud'] = vm.CLOUD
metadata[name_prefix + 'zone'] = vm.zone
metadata[name_prefix + 'image'] = vm.image
for k, v in vm.GetMachineTypeDict().iteritems():
metadata[name_prefix + k] = v
metadata[name_prefix + 'vm_count'] = len(vms)
if vm.scratch_disks:
data_disk = vm.scratch_disks[0]
disk_type = data_disk.disk_type
disk_size = data_disk.disk_size
num_stripes = data_disk.num_striped_disks
# Legacy metadata keys
metadata[name_prefix + 'scratch_disk_type'] = disk_type
metadata[name_prefix + 'scratch_disk_size'] = disk_size
metadata[name_prefix +
'num_striped_disks'] = (data_disk.num_striped_disks)
if getattr(data_disk, 'iops', None) is not None:
metadata[name_prefix +
'scratch_disk_iops'] = data_disk.iops
metadata[name_prefix +
'aws_provisioned_iops'] = data_disk.iops
# Modern metadata keys
metadata[name_prefix + 'data_disk_0_type'] = disk_type
metadata[name_prefix +
'data_disk_0_size'] = (disk_size * num_stripes
if disk_size else disk_size)
metadata[name_prefix + 'data_disk_0_num_stripes'] = num_stripes
if getattr(data_disk, 'metadata', None) is not None:
if disk.LEGACY_DISK_TYPE in data_disk.metadata:
metadata[name_prefix + 'scratch_disk_type'] = (
data_disk.metadata[disk.LEGACY_DISK_TYPE])
for key, value in data_disk.metadata.iteritems():
metadata[name_prefix + 'data_disk_0_%s' %
(key, )] = value
if FLAGS.set_files:
metadata['set_files'] = ','.join(FLAGS.set_files)
if FLAGS.sysctl:
metadata['sysctl'] = ','.join(FLAGS.sysctl)
# Flatten all user metadata into a single list (since each string in the
# FLAGS.metadata can actually be several key-value pairs) and then iterate
# over it.
parsed_metadata = flag_util.ParseKeyValuePairs(FLAGS.metadata)
metadata.update(parsed_metadata)
return metadata
def AddMetadata(self, metadata, benchmark_spec):
metadata = metadata.copy()
metadata['perfkitbenchmarker_version'] = version.VERSION
if FLAGS.simulate_maintenance:
metadata['simulate_maintenance'] = True
if FLAGS.hostname_metadata:
metadata['hostnames'] = ','.join(
[vm.hostname for vm in benchmark_spec.vms])
if benchmark_spec.container_cluster:
cluster = benchmark_spec.container_cluster
for k, v in cluster.GetResourceMetadata().iteritems():
metadata['container_cluster_' + k] = v
if benchmark_spec.managed_relational_db:
managed_db = benchmark_spec.managed_relational_db
for k, v in managed_db.GetResourceMetadata().iteritems():
metadata['managed_relational_db_' + k] = v
if benchmark_spec.cloud_tpu:
cloud_tpu = benchmark_spec.cloud_tpu
for k, v in cloud_tpu.GetResourceMetadata().iteritems():
metadata['cloud_tpu_' + k] = v
if benchmark_spec.cloud_redis:
cloud_redis = benchmark_spec.cloud_redis
for k, v in cloud_redis.GetResourceMetadata().iteritems():
metadata['cloud_redis_' + k] = v
for name, vms in benchmark_spec.vm_groups.iteritems():
if len(vms) == 0:
continue
# Get a representative VM so that we can publish the cloud, zone,
# machine type, and image.
vm = vms[-1]
name_prefix = '' if name == 'default' else name + '_'
for k, v in vm.GetResourceMetadata().iteritems():
metadata[name_prefix + k] = v
metadata[name_prefix + 'vm_count'] = len(vms)
for k, v in vm.GetOSResourceMetadata().iteritems():
metadata[name_prefix + k] = v
if vm.scratch_disks:
data_disk = vm.scratch_disks[0]
metadata[name_prefix + 'data_disk_count'] = len(
vm.scratch_disks)
for key, value in data_disk.GetResourceMetadata().iteritems():
metadata[name_prefix + 'data_disk_0_%s' % (key, )] = value
if FLAGS.set_files:
metadata['set_files'] = ','.join(FLAGS.set_files)
if FLAGS.sysctl:
metadata['sysctl'] = ','.join(FLAGS.sysctl)
# Flatten all user metadata into a single list (since each string in the
# FLAGS.metadata can actually be several key-value pairs) and then iterate
# over it.
parsed_metadata = flag_util.ParseKeyValuePairs(FLAGS.metadata)
metadata.update(parsed_metadata)
return metadata
def _ApplyFlags(cls, config_values, flag_values):
"""Overrides config values with flag values.
Can be overridden by derived classes to add support for specific flags.
Args:
config_values: dict mapping config option names to provided values. Is
modified by this function.
flag_values: flags.FlagValues. Runtime flags that may override the
provided config values.
Returns:
dict mapping config option names to values derived from the config
values or flag values.
"""
super(KubernetesDiskSpec, cls)._ApplyFlags(config_values, flag_values)
if flag_values['k8s_volume_provisioner'].present:
config_values['provisioner'] = flag_values.k8s_volume_provisioner
if flag_values['k8s_volume_parameters'].present:
config_values['parameters'] = config_values.get('parameters', {})
config_values['parameters'].update(
flag_util.ParseKeyValuePairs(flag_values.k8s_volume_parameters))
def _GenerateCreateCommand(self, ssh_keys_path):
"""Generates a command to create the VM instance.
Args:
ssh_keys_path: string. Path to a file containing the sshKeys metadata.
Returns:
GcloudCommand. gcloud command to issue in order to create the VM instance.
"""
args = []
if self.node_group:
args = ['alpha']
args.extend(['compute', 'instances', 'create', self.name])
cmd = util.GcloudCommand(self, *args)
if self.network.subnet_resource is not None:
cmd.flags['subnet'] = self.network.subnet_resource.name
else:
cmd.flags['network'] = self.network.network_resource.name
if self.image:
cmd.flags['image'] = self.image
elif self.image_family:
cmd.flags['image-family'] = self.image_family
if self.image_project is not None:
cmd.flags['image-project'] = self.image_project
cmd.flags['boot-disk-auto-delete'] = True
cmd.flags['boot-disk-size'] = self.boot_disk_size or self.BOOT_DISK_SIZE_GB
cmd.flags['boot-disk-type'] = self.boot_disk_type or self.BOOT_DISK_TYPE
if self.machine_type is None:
cmd.flags['custom-cpu'] = self.cpus
cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib)
else:
cmd.flags['machine-type'] = self.machine_type
if self.gpu_count:
cmd.flags['accelerator'] = GenerateAcceleratorSpecString(self.gpu_type,
self.gpu_count)
cmd.flags['tags'] = 'perfkitbenchmarker'
cmd.flags['no-restart-on-failure'] = True
if self.node_group:
cmd.flags['node-group'] = self.node_group.name
if self.min_cpu_platform:
cmd.flags['min-cpu-platform'] = self.min_cpu_platform
metadata_from_file = {'sshKeys': ssh_keys_path}
parsed_metadata_from_file = flag_util.ParseKeyValuePairs(
FLAGS.gcp_instance_metadata_from_file)
for key, value in parsed_metadata_from_file.iteritems():
if key in metadata_from_file:
logging.warning('Metadata "%s" is set internally. Cannot be overridden '
'from command line.', key)
continue
metadata_from_file[key] = value
cmd.flags['metadata-from-file'] = ','.join([
'%s=%s' % (k, v) for k, v in metadata_from_file.iteritems()
])
metadata = {'owner': FLAGS.owner} if FLAGS.owner else {}
metadata.update(self.boot_metadata)
parsed_metadata = flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata)
for key, value in parsed_metadata.iteritems():
if key in metadata:
logging.warning('Metadata "%s" is set internally. Cannot be overridden '
'from command line.', key)
continue
metadata[key] = value
cmd.flags['metadata'] = ','.join(
['%s=%s' % (k, v) for k, v in metadata.iteritems()])
# TODO(gareth-ferneyhough): If GCE one day supports live migration on GPUs
# this can be revised.
if (FLAGS['gce_migrate_on_maintenance'].present and
FLAGS.gce_migrate_on_maintenance and self.gpu_count):
raise errors.Config.InvalidValue(
'Cannot set flag gce_migrate_on_maintenance on instances with GPUs, '
'as it is not supported by GCP.')
if not FLAGS.gce_migrate_on_maintenance or self.gpu_count:
cmd.flags['maintenance-policy'] = 'TERMINATE'
ssd_interface_option = FLAGS.gce_ssd_interface
cmd.flags['local-ssd'] = (['interface={0}'.format(ssd_interface_option)] *
self.max_local_disks)
if FLAGS.gcloud_scopes:
cmd.flags['scopes'] = ','.join(re.split(r'[,; ]', FLAGS.gcloud_scopes))
if self.preemptible:
cmd.flags['preemptible'] = True
return cmd
def _GenerateCreateCommand(self, ssh_keys_path):
"""Generates a command to create the VM instance.
Args:
ssh_keys_path: string. Path to a file containing the sshKeys metadata.
Returns:
GcloudCommand. gcloud command to issue in order to create the VM instance.
"""
args = ['compute', 'instances', 'create', self.name]
cmd = util.GcloudCommand(self, *args)
if self.network.subnet_resource is not None:
cmd.flags['subnet'] = self.network.subnet_resource.name
else:
cmd.flags['network'] = self.network.network_resource.name
if self.image:
cmd.flags['image'] = self.image
elif self.image_family:
cmd.flags['image-family'] = self.image_family
if self.image_project is not None:
cmd.flags['image-project'] = self.image_project
cmd.flags['boot-disk-auto-delete'] = True
if self.boot_disk_size:
cmd.flags['boot-disk-size'] = self.boot_disk_size
if self.boot_disk_type:
cmd.flags['boot-disk-type'] = self.boot_disk_type
if self.machine_type is None:
cmd.flags['custom-cpu'] = self.cpus
cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib)
if self.min_cpu_platform:
cmd.flags['min-cpu-platform'] = self.min_cpu_platform
else:
cmd.flags['machine-type'] = self.machine_type
if self.min_cpu_platform and 'n1-' in self.machine_type:
cmd.flags['min-cpu-platform'] = self.min_cpu_platform
elif self.min_cpu_platform:
logging.warning('Cannot set min-cpu-platform for %s',
self.machine_type)
if self.gpu_count and self.machine_type and 'a2-' not in self.machine_type:
# A2 machine type already has predefined GPU type and count.
cmd.flags['accelerator'] = GenerateAcceleratorSpecString(
self.gpu_type, self.gpu_count)
cmd.flags['tags'] = ','.join(['perfkitbenchmarker'] +
(self.gce_tags or []))
cmd.flags['no-restart-on-failure'] = True
if self.node_group:
cmd.flags['node-group'] = self.node_group.name
if self.gce_shielded_secure_boot:
cmd.flags['shielded-secure-boot'] = True
if self.network.placement_group:
self.metadata.update(
self.network.placement_group.GetResourceMetadata())
cmd.flags['resource-policies'] = self.network.placement_group.name
cmd.flags['maintenance-policy'] = 'TERMINATE'
else:
self.metadata[
'placement_group_style'] = placement_group.PLACEMENT_GROUP_NONE
metadata_from_file = {'sshKeys': ssh_keys_path}
parsed_metadata_from_file = flag_util.ParseKeyValuePairs(
FLAGS.gcp_instance_metadata_from_file)
for key, value in six.iteritems(parsed_metadata_from_file):
if key in metadata_from_file:
logging.warning(
'Metadata "%s" is set internally. Cannot be overridden '
'from command line.', key)
continue
metadata_from_file[key] = value
cmd.flags['metadata-from-file'] = ','.join(
['%s=%s' % (k, v) for k, v in six.iteritems(metadata_from_file)])
metadata = {}
metadata.update(self.boot_metadata)
metadata.update(util.GetDefaultTags())
additional_metadata = {}
additional_metadata.update(self.vm_metadata)
additional_metadata.update(
flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata))
for key, value in six.iteritems(additional_metadata):
if key in metadata:
logging.warning(
'Metadata "%s" is set internally. Cannot be overridden '
'from command line.', key)
continue
metadata[key] = value
if self.preemptible:
cmd.flags['preemptible'] = True
preemptible_status_bucket = (
f'gs://{FLAGS.gcp_preemptible_status_bucket}/{FLAGS.run_uri}/')
self.preempt_marker = f'{preemptible_status_bucket}{self.name}'
metadata.update([self._PreemptibleMetadataKeyValue()])
cmd.flags['metadata'] = util.FormatTags(metadata)
# TODO(user): If GCE one day supports live migration on GPUs
# this can be revised.
if (FLAGS['gce_migrate_on_maintenance'].present
and FLAGS.gce_migrate_on_maintenance and self.gpu_count):
raise errors.Config.InvalidValue(
'Cannot set flag gce_migrate_on_maintenance on instances with GPUs, '
'as it is not supported by GCP.')
if not FLAGS.gce_migrate_on_maintenance or self.gpu_count:
cmd.flags['maintenance-policy'] = 'TERMINATE'
cmd.flags['local-ssd'] = (
['interface={0}'.format(FLAGS.gce_ssd_interface)] *
self.max_local_disks)
if FLAGS.gcloud_scopes:
cmd.flags['scopes'] = ','.join(
re.split(r'[,; ]', FLAGS.gcloud_scopes))
cmd.flags['network-tier'] = self.gce_network_tier.upper()
cmd.flags['labels'] = util.MakeFormattedDefaultTags()
return cmd
def _GenerateCreateCommand(self, ssh_keys_path):
"""Generates a command to create the VM instance.
Args:
ssh_keys_path: string. Path to a file containing the sshKeys metadata.
Returns:
GcloudCommand. gcloud command to issue in order to create the VM instance.
"""
cmd = util.GcloudCommand(self, 'compute', 'instances', 'create',
self.name)
cmd.flags['network'] = self.network.network_resource.name
cmd.flags['image'] = self.image
cmd.flags['boot-disk-auto-delete'] = True
if FLAGS.image_project:
cmd.flags['image-project'] = FLAGS.image_project
cmd.flags['boot-disk-size'] = self.BOOT_DISK_SIZE_GB
cmd.flags['boot-disk-type'] = self.BOOT_DISK_TYPE
if self.machine_type is None:
cmd.flags['custom-cpu'] = self.cpus
cmd.flags['custom-memory'] = '{0}MiB'.format(self.memory_mib)
else:
cmd.flags['machine-type'] = self.machine_type
cmd.flags['tags'] = 'perfkitbenchmarker'
cmd.flags['no-restart-on-failure'] = True
metadata_from_file = {'sshKeys': ssh_keys_path}
parsed_metadata_from_file = flag_util.ParseKeyValuePairs(
FLAGS.gcp_instance_metadata_from_file)
for key, value in parsed_metadata_from_file.iteritems():
if key in metadata_from_file:
logging.warning(
'Metadata "%s" is set internally. Cannot be overridden '
'from command line.' % key)
continue
metadata_from_file[key] = value
cmd.flags['metadata-from-file'] = ','.join(
['%s=%s' % (k, v) for k, v in metadata_from_file.iteritems()])
metadata = {'owner': FLAGS.owner} if FLAGS.owner else {}
metadata.update(self.boot_metadata)
parsed_metadata = flag_util.ParseKeyValuePairs(
FLAGS.gcp_instance_metadata)
for key, value in parsed_metadata.iteritems():
if key in metadata:
logging.warning(
'Metadata "%s" is set internally. Cannot be overridden '
'from command line.' % key)
continue
metadata[key] = value
cmd.flags['metadata'] = ','.join(
['%s=%s' % (k, v) for k, v in metadata.iteritems()])
if not FLAGS.gce_migrate_on_maintenance:
cmd.flags['maintenance-policy'] = 'TERMINATE'
ssd_interface_option = NVME if NVME in self.image else SCSI
cmd.flags['local-ssd'] = (
['interface={0}'.format(ssd_interface_option)] *
self.max_local_disks)
if FLAGS.gcloud_scopes:
cmd.flags['scopes'] = ','.join(
re.split(r'[,; ]', FLAGS.gcloud_scopes))
if self.preemptible:
cmd.flags['preemptible'] = True
return cmd
def _Create(self):
"""Creates the cluster."""
cmd = self.DataprocGcloudCommand('clusters', 'create', self.cluster_id)
if self.project is not None:
cmd.flags['project'] = self.project
if self.spec.worker_count:
# The number of worker machines in the cluster
cmd.flags['num-workers'] = self.spec.worker_count
else:
cmd.flags['single-node'] = True
# Initialize applications on the dataproc cluster
if self.spec.applications:
logging.info('Include the requested applications')
cmd.flags['optional-components'] = ','.join(self.spec.applications)
# Enable component gateway for debuggability. Does not impact performance.
cmd.flags['enable-component-gateway'] = True
# TODO(pclay): stop ignoring spec.master_group?
for role in ['worker', 'master']:
# Set machine type
if self.spec.worker_group.vm_spec.machine_type:
self._AddToCmd(cmd, '{0}-machine-type'.format(role),
self.spec.worker_group.vm_spec.machine_type)
# Set boot_disk_size
if self.spec.worker_group.disk_spec.disk_size:
size_in_gb = '{}GB'.format(
str(self.spec.worker_group.disk_spec.disk_size))
self._AddToCmd(cmd, '{0}-boot-disk-size'.format(role),
size_in_gb)
# Set boot_disk_type
if self.spec.worker_group.disk_spec.disk_type:
self._AddToCmd(cmd, '{0}-boot-disk-type'.format(role),
self.spec.worker_group.disk_spec.disk_type)
self.dpb_hdfs_type = disk_to_hdfs_map[
self.spec.worker_group.disk_spec.disk_type]
# Set ssd count
if self.spec.worker_group.vm_spec.num_local_ssds:
self._AddToCmd(cmd, 'num-{0}-local-ssds'.format(role),
self.spec.worker_group.vm_spec.num_local_ssds)
# This will actually be used for storage
self.dpb_hdfs_type = 'Local SSD'
# Set zone
cmd.flags['zone'] = self.dpb_service_zone
if self.dpb_version:
cmd.flags['image-version'] = self.dpb_version
if FLAGS.gcp_dataproc_image:
cmd.flags['image'] = FLAGS.gcp_dataproc_image
if FLAGS.dpb_cluster_properties:
cmd.flags['properties'] = ','.join(FLAGS.dpb_cluster_properties)
# Ideally DpbServiceSpec would have a network spec, which we would create to
# Resolve the name, but because EMR provisions its own VPC and we are
# generally happy using pre-existing networks for Dataproc. Just use the
# underlying flag instead.
if FLAGS.gce_network_name:
cmd.flags['network'] = FLAGS.gce_network_name
metadata = util.GetDefaultTags()
metadata.update(
flag_util.ParseKeyValuePairs(FLAGS.gcp_instance_metadata))
cmd.flags['metadata'] = util.FormatTags(metadata)
cmd.flags['labels'] = util.MakeFormattedDefaultTags()
timeout = 900 # 15 min
stdout, stderr, retcode = cmd.Issue(timeout=timeout,
raise_on_failure=False)
self._cluster_create_time = self._ParseClusterCreateTime(stdout)
if retcode:
util.CheckGcloudResponseKnownFailures(stderr, retcode)
raise errors.Resource.CreationError(stderr)
