def _Create(self):
"""Create a GCE VM instance."""
with open(self.ssh_public_key) as f:
public_key = f.read().rstrip('\n')
with vm_util.NamedTemporaryFile(dir=vm_util.GetTempDir(),
prefix='key-metadata') as tf:
tf.write('%s:%s\n' % (self.user_name, public_key))
tf.close()
create_cmd = [
FLAGS.gcloud_path, 'compute', 'instances', 'create', self.name,
'--disk',
'name=%s' % self.boot_disk.name, 'boot=yes', 'mode=rw',
'--machine-type', self.machine_type,
'--tags=perfkitbenchmarker', '--no-restart-on-failure',
'--metadata-from-file',
'sshKeys=%s' % tf.name, '--metadata',
'owner=%s' % FLAGS.owner
]
for key, value in self.boot_metadata.iteritems():
create_cmd.append('%s=%s' % (key, value))
if not FLAGS.gce_migrate_on_maintenance:
create_cmd.extend(['--maintenance-policy', 'TERMINATE'])
ssd_interface_option = NVME if NVME in self.image else SCSI
for _ in range(self.max_local_disks):
create_cmd.append('--local-ssd')
create_cmd.append('interface=%s' % ssd_interface_option)
if FLAGS.gcloud_scopes:
create_cmd.extend(['--scopes'] +
re.split(r'[,; ]', FLAGS.gcloud_scopes))
create_cmd.extend(util.GetDefaultGcloudFlags(self))
vm_util.IssueCommand(create_cmd)
def _Create(self):
run_cmd = [
FLAGS.kubectl,
'run',
self.name,
'--image=%s' % self.image,
'--limits=cpu=%sm,memory=%sMi' % (int(1000 * self.cpus), self.memory),
'--port', str(self.port)
]
if self.command:
run_cmd.extend(['--command', '--'])
run_cmd.extend(self.command)
vm_util.IssueCommand(run_cmd)
expose_cmd = [
FLAGS.kubectl,
'--kubeconfig', FLAGS.kubeconfig,
'expose', 'deployment', self.name,
'--type', 'NodePort',
'--target-port', str(self.port)
]
vm_util.IssueCommand(expose_cmd)
with vm_util.NamedTemporaryFile() as tf:
tf.write(_K8S_INGRESS.format(service_name=self.name))
tf.close()
kubernetes_helper.CreateFromFile(tf.name)
def _Create(self):
"""Create a GCE VM instance."""
num_hosts = len(self.host_list)
with open(self.ssh_public_key) as f:
public_key = f.read().rstrip('\n')
with vm_util.NamedTemporaryFile(dir=vm_util.GetTempDir(),
prefix='key-metadata') as tf:
tf.write('%s:%s\n' % (self.user_name, public_key))
tf.close()
create_cmd = self._GenerateCreateCommand(tf.name)
_, stderr, retcode = create_cmd.Issue()
if (self.use_dedicated_host and retcode
and _INSUFFICIENT_HOST_CAPACITY in stderr
and not self.num_vms_per_host):
logging.warning(
'Creation failed due to insufficient host capacity. A new host will '
'be created and instance creation will be retried.')
with self._host_lock:
if num_hosts == len(self.host_list):
host = GceSoleTenantHost(self.host_type, self.zone,
self.project)
self.host_list.append(host)
host.Create()
self.host = self.host_list[-1]
raise errors.Resource.RetryableCreationError()
def _Create(self):
run_cmd = [
'run', self.name,
'--image=%s' % self.image, '--port',
str(self.port)
]
limits = []
if self.cpus:
limits.append(f'cpu={int(1000 * self.cpus)}m')
if self.memory:
limits.append(f'memory={self.memory}Mi')
if limits:
run_cmd.append('--limits=' + ','.join(limits))
if self.command:
run_cmd.extend(['--command', '--'])
run_cmd.extend(self.command)
RunKubectlCommand(run_cmd)
expose_cmd = [
'expose', 'deployment', self.name, '--type', 'NodePort',
'--target-port',
str(self.port)
]
RunKubectlCommand(expose_cmd)
with vm_util.NamedTemporaryFile() as tf:
tf.write(_K8S_INGRESS.format(service_name=self.name))
tf.close()
kubernetes_helper.CreateFromFile(tf.name)
def RenderTemplate(self, template_path, remote_path, context):
"""Renders a local Jinja2 template and copies it to the remote host.
The template will be provided variables defined in 'context', as well as a
variable named 'vm' referencing this object.
Args:
template_path: string. Local path to jinja2 template.
remote_path: string. Remote path for rendered file on the remote vm.
context: dict. Variables to pass to the Jinja2 template during rendering.
Raises:
jinja2.UndefinedError: if template contains variables not present in
'context'.
RemoteCommandError: If there was a problem copying the file.
"""
with open(template_path) as fp:
template_contents = fp.read()
environment = jinja2.Environment(undefined=jinja2.StrictUndefined)
template = environment.from_string(template_contents)
prefix = 'pkb-' + os.path.basename(template_path)
with vm_util.NamedTemporaryFile(prefix=prefix,
dir=vm_util.GetTempDir(),
delete=False) as tf:
tf.write(template.render(vm=self, **context))
tf.close()
self.RemoteCopy(tf.name, remote_path)
def PublishSamples(self, samples):
if not samples:
logging.warn('No samples: not publishing to BigQuery')
return
with vm_util.NamedTemporaryFile(prefix='perfkit-bq-pub',
dir=vm_util.GetTempDir(),
suffix='.json') as tf:
json_publisher = NewlineDelimitedJSONPublisher(
tf.name, collapse_labels=True)
json_publisher.PublishSamples(samples)
tf.close()
logging.info('Publishing %d samples to %s', len(samples),
self.bigquery_table)
load_cmd = [self.bq_path]
if self.project_id:
load_cmd.append('--project_id=' + self.project_id)
if self.service_account:
assert self.service_account_private_key_file is not None
load_cmd.extend([
'--service_account=' + self.service_account,
'--service_account_credential_file=' +
self._credentials_file,
'--service_account_private_key_file=' +
self.service_account_private_key_file
])
load_cmd.extend([
'load', '--source_format=NEWLINE_DELIMITED_JSON',
self.bigquery_table, tf.name
])
vm_util.IssueRetryableCommand(load_cmd)
def _Create(self):
"""Create a GCE VM instance."""
num_hosts = len(self.host_list)
with open(self.ssh_public_key) as f:
public_key = f.read().rstrip('\n')
with vm_util.NamedTemporaryFile(dir=vm_util.GetTempDir(),
prefix='key-metadata') as tf:
tf.write('%s:%s\n' % (self.user_name, public_key))
tf.close()
create_cmd = self._GenerateCreateCommand(tf.name)
_, stderr, retcode = create_cmd.Issue(timeout=_GCE_VM_CREATE_TIMEOUT)
if (self.use_dedicated_host and retcode and
_INSUFFICIENT_HOST_CAPACITY in stderr and not self.num_vms_per_host):
logging.warning(
'Creation failed due to insufficient host capacity. A new host will '
'be created and instance creation will be retried.')
with self._host_lock:
if num_hosts == len(self.host_list):
host = GceSoleTenantNodeGroup(self.node_template,
self.zone, self.project)
self.host_list.append(host)
host.Create()
self.node_group = self.host_list[-1]
raise errors.Resource.RetryableCreationError()
if (not self.use_dedicated_host and retcode and
_INSUFFICIENT_HOST_CAPACITY in stderr):
logging.error(STOCKOUT_MESSAGE)
raise errors.Benchmarks.InsufficientCapacityCloudFailure(STOCKOUT_MESSAGE)
util.CheckGcloudResponseKnownFailures(stderr, retcode)
def CreateResource(resource_body):
with vm_util.NamedTemporaryFile() as tf:
tf.write(resource_body)
tf.close()
create_cmd = [
FLAGS.kubectl,
'--kubeconfig=%s' % FLAGS.kubeconfig, 'create', '-f', tf.name
]
return vm_util.IssueCommand(create_cmd)
def _Delete(self):
"""Deletes the service."""
with vm_util.NamedTemporaryFile() as tf:
tf.write(_K8S_INGRESS.format(service_name=self.name))
tf.close()
kubernetes_helper.DeleteFromFile(tf.name)
delete_cmd = ['delete', 'deployment', self.name]
RunKubectlCommand(delete_cmd, raise_on_failure=False)
def JujuConfigureEnvironment(self):
"""Configure a bootstrapped Juju environment."""
if self.is_controller:
resp, _ = self.RemoteHostCommand('mkdir -p ~/.juju')
with vm_util.NamedTemporaryFile() as tf:
tf.write(self.environments_yaml.format(self.internal_ip))
tf.close()
self.PushFile(tf.name, '~/.juju/environments.yaml')
def _RunDiskpartScript(self, script):
"""Runs the supplied Diskpart script on the VM."""
with vm_util.NamedTemporaryFile(prefix='diskpart') as tf:
tf.write(script)
tf.close()
script_path = ntpath.join(self.temp_dir, os.path.basename(tf.name))
self.RemoteCopy(tf.name, script_path)
self.RemoteCommand(
'diskpart /s {script_path}'.format(script_path=script_path))
def _Create(self):
"""Create a GCE VM instance."""
with open(self.ssh_public_key) as f:
public_key = f.read().rstrip('\n')
with vm_util.NamedTemporaryFile(dir=vm_util.GetTempDir(),
prefix='key-metadata') as tf:
tf.write('%s:%s\n' % (self.user_name, public_key))
tf.close()
create_cmd = self._GenerateCreateCommand(tf.name)
create_cmd.Issue()
def _ConfigureSpec(prime_client,
clients,
benchmark,
load=None,
num_runs=None,
incr_load=None):
"""Configures SPEC SFS 2014 on the prime client.
This function modifies the default configuration file (sfs_rc) which
can be found either in the SPEC SFS 2014 user guide or within the iso.
It also creates a file containing the client mountpoints so that SPEC
can run in a distributed manner.
Args:
prime_client: The VM from which SPEC will be controlled.
clients: A list of SPEC client VMs (including the prime_client).
benchmark: The sub-benchmark to run.
load: List of ints. The LOAD parameter to SPECSFS.
num_runs: The NUM_RUNS parameter to SPECSFS.
incr_load: The INCR_LOAD parameter to SPECSFS.
"""
config_path = posixpath.join(_SPEC_DIR, _SPEC_CONFIG)
prime_client.RemoteCommand('sudo cp {0}.bak {0}'.format(config_path))
stdout, _ = prime_client.RemoteCommand('pwd')
exec_path = posixpath.join(stdout.strip(), _SPEC_DIR, 'binaries', 'linux',
'x86_64', 'netmist')
load = load or FLAGS.specsfs2014_load
num_runs = num_runs or FLAGS.specsfs2014_load
incr_load = incr_load or FLAGS.specsfs2014_incr_load
configuration_overrides = {
'USER': prime_client.user_name,
'EXEC_PATH': exec_path.replace('/', r'\/'),
'CLIENT_MOUNTPOINTS': _MOUNTPOINTS_FILE,
'BENCHMARK': benchmark,
'LOAD': ' '.join([str(x) for x in load]),
'NUM_RUNS': num_runs,
'INCR_LOAD': incr_load,
'WARMUP_TIME': 60,
}
# Any special characters in the overrides dictionary should be escaped so
# that they don't interfere with sed.
sed_expressions = ' '.join([
'-e "s/{0}=.*/{0}={1}/"'.format(k, v)
for k, v in six.iteritems(configuration_overrides)
])
sed_cmd = 'sudo sed -i {0} {1}'.format(sed_expressions, config_path)
prime_client.RemoteCommand(sed_cmd)
with vm_util.NamedTemporaryFile(mode='w') as tf:
for client in clients:
tf.write('%s %s\n' % (client.internal_ip, _MOUNT_POINT))
tf.close()
prime_client.PushFile(tf.name,
posixpath.join(_SPEC_DIR, _MOUNTPOINTS_FILE))
def _Create(self):
"""Create a GCE VM instance."""
num_hosts = len(self.host_list)
with open(self.ssh_public_key) as f:
public_key = f.read().rstrip('\n')
with vm_util.NamedTemporaryFile(mode='w',
dir=vm_util.GetTempDir(),
prefix='key-metadata') as tf:
tf.write('%s:%s\n' % (self.user_name, public_key))
tf.close()
create_cmd = self._GenerateCreateCommand(tf.name)
_, stderr, retcode = create_cmd.Issue(
timeout=_GCE_VM_CREATE_TIMEOUT, raise_on_failure=False)
if (self.use_dedicated_host and retcode
and _INSUFFICIENT_HOST_CAPACITY in stderr):
if self.num_vms_per_host:
raise errors.Resource.CreationError(
'Failed to create host: %d vms of type %s per host exceeds '
'memory capacity limits of the host' %
(self.num_vms_per_host, self.machine_type))
else:
logging.warning(
'Creation failed due to insufficient host capacity. A new host will '
'be created and instance creation will be retried.')
with self._host_lock:
if num_hosts == len(self.host_list):
host = GceSoleTenantNodeGroup(self.node_template,
self.zone, self.project)
self.host_list.append(host)
host.Create()
self.node_group = self.host_list[-1]
raise errors.Resource.RetryableCreationError()
if (not self.use_dedicated_host and retcode
and _INSUFFICIENT_HOST_CAPACITY in stderr):
logging.error(util.STOCKOUT_MESSAGE)
raise errors.Benchmarks.InsufficientCapacityCloudFailure(
util.STOCKOUT_MESSAGE)
util.CheckGcloudResponseKnownFailures(stderr, retcode)
if retcode:
if (create_cmd.rate_limited and 'already exists' in stderr
and FLAGS.retry_on_rate_limited):
# Gcloud create commands may still create VMs despite being rate
# limited.
return
if util.RATE_LIMITED_MESSAGE in stderr:
raise errors.Benchmarks.QuotaFailure.RateLimitExceededError(
stderr)
if self.preemptible and _FAILED_TO_START_DUE_TO_PREEMPTION in stderr:
self.spot_early_termination = True
raise errors.Benchmarks.InsufficientCapacityCloudFailure(
'Interrupted before VM started')
raise errors.Resource.CreationError(
'Failed to create VM: %s return code: %s' % (stderr, retcode))
def AuthenticateVm(self):
"""Authenticate a remote machine to access all peers."""
if not self.is_static and not self.has_private_key:
self.RemoteHostCopy(vm_util.GetPrivateKeyPath(), REMOTE_KEY_PATH)
with vm_util.NamedTemporaryFile() as tf:
tf.write('Host *\n')
tf.write(' StrictHostKeyChecking no\n')
tf.close()
self.PushFile(tf.name, '~/.ssh/config')
self.has_private_key = True
def _AuthorizeNodes(self):
"""Allow the nodes to be added to the cluster."""
with vm_util.NamedTemporaryFile() as tf:
tf.write(_CONFIG_MAP.format(
node_instance_role=self.eks_workers.node_instance_role))
tf.close()
apply_cmd = [
FLAGS.kubectl, '--kubeconfig', FLAGS.kubeconfig,
'apply', '-f', tf.name,
]
vm_util.IssueCommand(apply_cmd)
def _Delete(self):
"""Deletes the service."""
with vm_util.NamedTemporaryFile() as tf:
tf.write(_K8S_INGRESS.format(service_name=self.name))
tf.close()
kubernetes_helper.DeleteFromFile(tf.name)
delete_cmd = [
FLAGS.kubectl, '--kubeconfig', FLAGS.kubeconfig, 'delete',
'deployment', self.name
]
vm_util.IssueCommand(delete_cmd)
def PublishSamples(self, samples):
with vm_util.NamedTemporaryFile(prefix='perfkit-gcs-pub',
dir=vm_util.GetTempDir(),
suffix='.json') as tf:
json_publisher = NewlineDelimitedJSONPublisher(tf.name)
json_publisher.PublishSamples(samples)
tf.close()
object_name = self._GenerateObjectName()
storage_uri = 'gs://{0}/{1}'.format(self.bucket, object_name)
logging.info('Publishing %d samples to %s', len(samples), storage_uri)
copy_cmd = [self.gsutil_path, 'cp', tf.name, storage_uri]
vm_util.IssueRetryableCommand(copy_cmd)
def CreateMachineFile(vms):
"""Create a file with the IP of each machine in the cluster on its own line.
Args:
vms: The list of vms which will be in the cluster.
"""
with vm_util.NamedTemporaryFile() as machine_file:
master_vm = vms[0]
machine_file.write('localhost slots=%d\n' % master_vm.num_cpus)
for vm in vms[1:]:
machine_file.write('%s slots=%d\n' % (vm.internal_ip, vm.num_cpus))
machine_file.close()
master_vm.PushFile(machine_file.name, MACHINEFILE)
def _Create(self):
"""Creates the cluster."""
# Create the cluster spec but don't provision any resources.
create_cmd = [
FLAGS.kops, 'create', 'cluster',
'--name=%s' % self.name,
'--zones=%s' % self.zone,
'--node-count=%s' % self.num_nodes,
'--node-size=%s' % self.machine_type
]
env = os.environ.copy()
env['KUBECONFIG'] = FLAGS.kubeconfig
env['KOPS_STATE_STORE'] = 's3://%s' % self.config_bucket
vm_util.IssueCommand(create_cmd, env=env)
# Download the cluster spec and modify it.
get_cmd = [
FLAGS.kops, 'get', 'cluster', self.name, '--output=yaml'
]
stdout, _, _ = vm_util.IssueCommand(get_cmd, env=env)
spec = yaml.load(stdout)
spec['metadata']['creationTimestamp'] = None
spec['spec']['api']['loadBalancer']['idleTimeoutSeconds'] = 3600
benchmark_spec = context.GetThreadBenchmarkSpec()
spec['spec']['cloudLabels'] = {
'owner': FLAGS.owner,
'perfkitbenchmarker-run': FLAGS.run_uri,
'benchmark': benchmark_spec.name,
'perfkit_uuid': benchmark_spec.uuid,
'benchmark_uid': benchmark_spec.uid
}
# Replace the cluster spec.
with vm_util.NamedTemporaryFile() as tf:
yaml.dump(spec, tf)
tf.close()
replace_cmd = [
FLAGS.kops, 'replace', '--filename=%s' % tf.name
]
vm_util.IssueCommand(replace_cmd, env=env)
# Create the actual cluster.
update_cmd = [
FLAGS.kops, 'update', 'cluster', self.name, '--yes'
]
vm_util.IssueCommand(update_cmd, env=env)
def _PostCreate(self):
"""Retrieve generic VM info and then retrieve the VM's password."""
super(WindowsAwsVirtualMachine, self)._PostCreate()
# Get the decoded password data.
decoded_password_data = self._GetDecodedPasswordData()
# Write the encrypted data to a file, and use openssl to
# decrypt the password.
with vm_util.NamedTemporaryFile() as tf:
tf.write(decoded_password_data)
tf.close()
decrypt_cmd = [
'openssl', 'rsautl', '-decrypt', '-in', tf.name, '-inkey',
vm_util.GetPrivateKeyPath()
]
password, _ = vm_util.IssueRetryableCommand(decrypt_cmd)
self.password = password
def CreateMachineFile(vms, num_slots=lambda vm: vm.NumCpusForBenchmark(),
remote_path='MACHINEFILE'):
"""Create a file with the IP of each machine in the cluster on its own line.
The file is then pushed to the provided path on the master vm.
Args:
vms: The list of vms which will be in the cluster.
num_slots: The function to use to calculate the number of slots
for each vm. Defaults to vm.NumCpusForBenchmark()
remote_path: remote path of the machine file. Defaults to MACHINEFILE
"""
with vm_util.NamedTemporaryFile(mode='w') as machine_file:
master_vm = vms[0]
machine_file.write('localhost slots=%d\n' % num_slots(master_vm))
for vm in vms[1:]:
machine_file.write('%s slots=%d\n' % (vm.internal_ip,
num_slots(vm)))
machine_file.close()
master_vm.PushFile(machine_file.name, remote_path)
def ApplyManifest(self, manifest_file, **kwargs):
"""Applies a declarative Kubernetes manifest; possibly with jinja.
Args:
manifest_file: The name of the YAML file or YAML template.
**kwargs: Arguments to the jinja template.
"""
filename = data.ResourcePath(manifest_file)
if not filename.endswith('.j2'):
assert not kwargs
RunKubectlCommand(['apply', '-f', filename])
return
environment = jinja2.Environment(undefined=jinja2.StrictUndefined)
with open(filename) as template_file, vm_util.NamedTemporaryFile(
mode='w', suffix='.yaml') as rendered_template:
manifest = environment.from_string(template_file.read()).render(kwargs)
rendered_template.write(manifest)
rendered_template.close()
RunKubectlCommand(['apply', '-f', rendered_template.name])
def CreateMachineFile(vms,
num_slots=lambda vm: vm.NumCpusForBenchmark(),
remote_path='MACHINEFILE',
mpi_vendor='openmpi'):
"""Create a file with the IP of each machine in the cluster on its own line.
The file is then pushed to the provided path on the master vm.
Pass in "num_slots=lambda vm: 0" to create a machine file without a defined
number of slots.
OpenMPI's format: "<host> slots=<slots>"
https://www.open-mpi.org/faq/?category=running#mpirun-hostfile
IntelMPI's format: "<host>:<slots>"
https://software.intel.com/content/www/us/en/develop/articles/controlling-process-placement-with-the-intel-mpi-library.html
Args:
vms: The list of vms which will be in the cluster.
num_slots: The function to use to calculate the number of slots
for each vm. Defaults to vm.NumCpusForBenchmark()
remote_path: remote path of the machine file. Defaults to MACHINEFILE
mpi_vendor: Implementation of MPI. Can be openmpi or intel.
"""
def Line(vm, vm_name=None):
vm_name = vm_name or vm.internal_ip
slots = num_slots(vm)
if not slots:
return vm_name
if mpi_vendor == 'intel':
return f'{vm_name}:{slots}'
return f'{vm_name} slots={slots}'
with vm_util.NamedTemporaryFile(mode='w') as machine_file:
master_vm = vms[0]
machine_file.write(Line(master_vm, 'localhost') + '\n')
for vm in vms[1:]:
machine_file.write(Line(vm) + '\n')
machine_file.close()
master_vm.PushFile(machine_file.name, remote_path)
def CreateResource(resource_body):
with vm_util.NamedTemporaryFile() as tf:
tf.write(resource_body)
tf.close()
return kubernetes_helper.CreateFromFile(tf.name)
def DeleteResource(resource_body):
with vm_util.NamedTemporaryFile() as tf:
tf.write(resource_body)
tf.close()
DeleteFromFile(tf.name)
def CreateResource(resource_body):
with vm_util.NamedTemporaryFile(mode='w') as tf:
tf.write(resource_body)
tf.close()
CreateFromFile(tf.name)
