def Install(vm):
"""Installs NVIDIA CUDA Deep Neural Network library."""
src_path = data.ResourcePath(FLAGS.cudnn)
dest_path = os.path.join('/tmp', FLAGS.cudnn)
vm.RemoteCopy(src_path, dest_path)
vm.RemoteCommand('tar -zxf %s' % dest_path, should_log=True)
vm.RemoteCommand('sudo cp cuda/lib64/* %s/lib64/' %
CUDA_TOOLKIT_INSTALL_DIR)
vm.RemoteCommand('sudo cp cuda/include/cudnn.h %s/include/' %
CUDA_TOOLKIT_INSTALL_DIR)
def _PrepareWithIsoFile(vm, speccpu_vm_state):
"""Prepares the VM to run using the iso file.
Copies the iso to the VM, mounts it, and extracts the contents. Copies the
config file to the VM. Runs the SPEC install.sh script on the VM.
Args:
vm: BaseVirtualMachine. Recipient of the iso file.
speccpu_vm_state: SpecInstallConfigurations. Modified by this function to
reflect any changes to the VM that may need to be cleaned up.
"""
scratch_dir = vm.GetScratchDir()
# Make cpu2006 or cpu2017 directory on the VM.
vm.RemoteCommand('mkdir {0}'.format(speccpu_vm_state.spec_dir))
# Copy the iso to the VM.
local_iso_file_path = data.ResourcePath(speccpu_vm_state.base_iso_file_path)
vm.PushFile(local_iso_file_path, scratch_dir)
# Extract files from the iso to the cpu2006 or cpu2017 directory.
vm.RemoteCommand('mkdir {0}'.format(speccpu_vm_state.mount_dir))
vm.RemoteCommand('sudo mount -t iso9660 -o loop {0} {1}'.format(
speccpu_vm_state.iso_file_path, speccpu_vm_state.mount_dir))
vm.RemoteCommand('cp -r {0}/* {1}'.format(speccpu_vm_state.mount_dir,
speccpu_vm_state.spec_dir))
# cpu2017 iso does not come with config directory nor clang.xml
if speccpu_vm_state.clang_flag_file_path:
vm.RemoteCommand('mkdir -p {0}'.format(
os.path.dirname(speccpu_vm_state.clang_flag_file_path)))
vm.PushFile(data.ResourcePath(speccpu_vm_state.base_clang_flag_file_path),
speccpu_vm_state.clang_flag_file_path)
vm.RemoteCommand('chmod -R 777 {0}'.format(speccpu_vm_state.spec_dir))
# Copy the cfg to the VM.
local_cfg_file_path = data.ResourcePath(speccpu_vm_state.runspec_config)
vm.PushFile(local_cfg_file_path, speccpu_vm_state.cfg_file_path)
# Run SPEC CPU2006 or 2017 installation.
install_script_path = posixpath.join(speccpu_vm_state.spec_dir, 'install.sh')
vm.RobustRemoteCommand('yes | {0}'.format(install_script_path))
def CheckPrerequisites(benchmark_config):
"""Verifies that the required resources are present.
Args:
benchmark_config: Unused
Raises:
perfkitbenchmarker.data.ResourceNotFound: On missing resource.
"""
del benchmark_config # unused
data.ResourcePath(DATA_FILE)
def CheckPrerequisites(_):
"""Verifies that the required resources are present.
Raises:
perfkitbenchmarker.data.ResourceNotFound: On missing resource.
"""
data.ResourcePath(_BOOT_TEMPLATE)
data.ResourcePath(_LISTENER_SERVER)
data.ResourcePath(_CLEAN_UP_TEMPLATE)
if FLAGS.cloud == 'Azure' and FLAGS.vms_contact_launcher and not _IsLinux(
):
raise errors.Benchmarks.PrepareException(
'Booting Windows VMs on Azure with a start-up script is not supported. '
'See https://github.com/Azure/azure-powershell/issues/9600.')
if FLAGS.vms_contact_launcher and FLAGS.use_public_ip:
raise errors.Benchmarks.PrepareException(
'After VMs contact launcher server, launcher will check connectivity '
'of the VMs using the client address of the curl request. This option '
'is only applicable when launcher makes the initial contact.')
def Prepare(self, benchmark_name: str) -> None:
"""Prepares the client vm to execute query.
Installs the bq tool dependencies and authenticates using a service account.
Args:
benchmark_name: String name of the benchmark, to allow extraction and
usage of benchmark specific artifacts (certificates, etc.) during client
vm preparation.
"""
self.client_vm.Install('pip')
self.client_vm.RemoteCommand('sudo pip install absl-py')
self.client_vm.Install('google_cloud_sdk')
# Push the service account file to the working directory on client vm
key_file_name = FLAGS.gcp_service_account_key_file.split('/')[-1]
if '/' in FLAGS.gcp_service_account_key_file:
self.client_vm.PushFile(FLAGS.gcp_service_account_key_file)
else:
self.client_vm.InstallPreprovisionedBenchmarkData(
benchmark_name, [FLAGS.gcp_service_account_key_file], '')
# Authenticate using the service account file
vm_gcloud_path = google_cloud_sdk.GCLOUD_PATH
activate_cmd = (
'{} auth activate-service-account {} --key-file={}'.format(
vm_gcloud_path, FLAGS.gcp_service_account, key_file_name))
self.client_vm.RemoteCommand(activate_cmd)
# Push the framework to execute a sql query and gather performance details
service_specific_dir = os.path.join('edw', Bigquery.SERVICE_TYPE)
self.client_vm.PushFile(
data.ResourcePath(
os.path.join(service_specific_dir, 'script_runner.sh')))
runner_permission_update_cmd = 'chmod 755 {}'.format(
'script_runner.sh')
self.client_vm.RemoteCommand(runner_permission_update_cmd)
self.client_vm.PushFile(
data.ResourcePath(os.path.join('edw', 'script_driver.py')))
self.client_vm.PushFile(
data.ResourcePath(
os.path.join(service_specific_dir,
'provider_specific_script_driver.py')))
def _GetWorkloadFileList():
"""Returns the list of workload files to run.
Returns:
In order of preference:
* The argument to --ycsb_workload_files.
* Bundled YCSB workloads A and B.
"""
return [data.ResourcePath(workload)
for workload in FLAGS.ycsb_workload_files]
def PrepareDataFile(vm):
"""Generate data file on vm to destination directory.
Args:
vm: The VM needs data file.
"""
file_path = data.ResourcePath(DATA_FILE)
vm.PushFile(file_path, '%s/' % vm.GetScratchDir(0))
vm.RemoteCommand('cd %s/; bash cloud-storage-workload.sh' %
vm.GetScratchDir(0))
def _Install(launcher_vm, booter_template_vm):
"""Installs benchmark scripts and packages on the launcher vm."""
launcher_vm.InstallCli()
# Render boot script on launcher server VM(s)
context = _BuildContext(launcher_vm, booter_template_vm)
launcher_vm.RenderTemplate(data.ResourcePath(_BOOT_TEMPLATE), _BOOT_PATH,
context)
launcher_vm.RenderTemplate(data.ResourcePath(_STATUS_TEMPLATE), _STATUS_PATH,
context)
# Installs and start listener server on launcher VM(s).
launcher_vm.InstallPackages('netcat')
launcher_vm.PushDataFile(_LISTENER_SERVER, _REMOTE_DIR)
client_port = _SSH_PORT if _IsLinux() else _RDP_PORT
launcher_vm.RemoteCommand('touch log')
launcher_vm.RemoteCommand(_GetServerStartCommand(client_port, launcher_vm))
# Render clean up script on launcher server VM(s).
launcher_vm.RenderTemplate(data.ResourcePath(_CLEAN_UP_TEMPLATE),
_CLEAN_UP_SCRIPT_PATH, context)
def _CopyLib(vm):
# If the cudnn flag was passed on the command line,
# use that value for the cudnn path. Otherwise, chose
# an intelligent default given the cuda toolkit version
# specified.
cudnn_path = FLAGS.cudnn
src_path = data.ResourcePath(cudnn_path)
dest_path = posixpath.join('/tmp', cudnn_path)
vm.RemoteCopy(src_path, dest_path)
return dest_path
def PushDataFile(self, data_file):
"""Upload a file in perfkitbenchmarker.data directory to the VM.
Args:
data_file: The filename of the file to upload.
Raises:
perfkitbenchmarker.data.ResourceNotFound: if 'data_file' does not exist.
"""
file_path = data.ResourcePath(data_file)
self.PushFile(file_path)
def CreateRenderedManifestFile(filename, config):
"""Returns a file containing a rendered Jinja manifest (.j2) template."""
manifest_filename = data.ResourcePath(filename)
environment = jinja2.Environment(undefined=jinja2.StrictUndefined)
with open(manifest_filename) as manifest_file:
manifest_template = environment.from_string(manifest_file.read())
rendered_yaml = tempfile.NamedTemporaryFile(mode='w')
rendered_yaml.write(manifest_template.render(config))
rendered_yaml.flush()
return rendered_yaml
def _LocalDataPath(local_file):
"""Return local data path for given file.
Args:
local_file: name of local file to create full data path for
Returns:
path of local_file, in the data directory
"""
return data.ResourcePath(local_file)
def Prepare(benchmark_spec):
"""Installs and sets up dataset on the Spark clusters.
Copies scripts and all the queries to cloud.
Creates external Hive tables for data (unless BigQuery is being used).
Args:
benchmark_spec: The benchmark specification
"""
dpb_service_instance = benchmark_spec.dpb_service
run_uri = benchmark_spec.uuid.split('-')[0]
dpb_service_instance.CreateBucket(run_uri)
temp_run_dir = temp_dir.GetRunDirPath()
spark_sql_perf_dir = os.path.join(temp_run_dir, 'spark_sql_perf_dir')
vm_util.IssueCommand(['git', 'clone', SPARK_SQL_PERF_GIT, spark_sql_perf_dir])
vm_util.IssueCommand(['git', 'checkout', SPARK_SQL_PERF_GIT_COMMIT],
cwd=spark_sql_perf_dir)
query_dir = os.path.join(spark_sql_perf_dir, 'src', 'main', 'resources',
FLAGS.dpb_sparksql_query)
storage_service = object_storage_service.GetObjectStorageClass(FLAGS.cloud)()
dst_url = '{prefix}{uri}'.format(
prefix=dpb_service_instance.PERSISTENT_FS_PREFIX, uri=run_uri)
for dir_name, _, files in os.walk(query_dir):
for filename in files:
match = re.match(r'q?([0-9]+)a?.sql', filename)
if match:
query_id = match.group(1)
# if order is specified only upload those queries
if not FLAGS.dpb_sparksql_order or query_id in FLAGS.dpb_sparksql_order:
query = '{}.sql'.format(query_id)
src_url = os.path.join(dir_name, filename)
storage_service.Copy(src_url, os.path.join(dst_url, query))
for script in [SPARK_TABLE_SCRIPT, SPARK_SQL_RUNNER_SCRIPT]:
src_url = data.ResourcePath(script)
storage_service.Copy(src_url, dst_url)
benchmark_spec.base_dir = dst_url
# Create external Hive tables if not reading the data from BigQuery
if FLAGS.dpb_sparksql_data:
stdout = storage_service.List(FLAGS.dpb_sparksql_data)
for table_dir in stdout.split('\n'):
# The directory name is the table name.
if not table_dir:
continue
table = re.split(' |/', table_dir.rstrip('/')).pop()
stats = dpb_service_instance.SubmitJob(
pyspark_file=os.path.join(dst_url, SPARK_TABLE_SCRIPT),
job_type=BaseDpbService.PYSPARK_JOB_TYPE,
job_arguments=[FLAGS.dpb_sparksql_data, table])
logging.info(stats)
if not stats['success']:
logging.warning('Creates table %s from %s failed', table, table_dir)
def Prepare(benchmark_spec):
"""Installs and sets up dataset on the Spark clusters.
Copies scripts and all the queries to cloud.
Creates external Hive tables for data (unless BigQuery is being used).
Args:
benchmark_spec: The benchmark specification
"""
dpb_service_instance = benchmark_spec.dpb_service
# buckets must start with a letter
bucket = 'pkb-' + benchmark_spec.uuid.split('-')[0]
storage_service = dpb_service_instance.storage_service
storage_service.MakeBucket(bucket)
benchmark_spec.base_dir = dpb_service_instance.PERSISTENT_FS_PREFIX + bucket
temp_run_dir = temp_dir.GetRunDirPath()
spark_sql_perf_dir = os.path.join(temp_run_dir, 'spark_sql_perf_dir')
vm_util.IssueCommand(
['git', 'clone', SPARK_SQL_PERF_GIT, spark_sql_perf_dir])
vm_util.IssueCommand(['git', 'checkout', SPARK_SQL_PERF_GIT_COMMIT],
cwd=spark_sql_perf_dir)
query_dir = os.path.join(spark_sql_perf_dir, 'src', 'main', 'resources',
FLAGS.dpb_sparksql_query)
for dir_name, _, files in os.walk(query_dir):
for filename in files:
match = re.match(r'q?([0-9]+)a?.sql', filename)
if match:
query_id = match.group(1)
# if order is specified only upload those queries
if not FLAGS.dpb_sparksql_order or query_id in FLAGS.dpb_sparksql_order:
query = '{}.sql'.format(query_id)
src_url = os.path.join(dir_name, filename)
storage_service.CopyToBucket(src_url, bucket, query)
for script in [SPARK_TABLE_SCRIPT, SPARK_SQL_RUNNER_SCRIPT]:
src_url = data.ResourcePath(script)
storage_service.CopyToBucket(src_url, bucket, script)
# Create external Hive tables if not reading the data from BigQuery
if FLAGS.dpb_sparksql_data:
stdout = storage_service.List(FLAGS.dpb_sparksql_data)
table_subdirs = [
re.split(' |/', line.rstrip('/')).pop()
for line in stdout.split('\n') if line
]
stats = dpb_service_instance.SubmitJob(
pyspark_file=os.path.join(benchmark_spec.base_dir,
SPARK_TABLE_SCRIPT),
job_type=BaseDpbService.PYSPARK_JOB_TYPE,
job_arguments=[FLAGS.dpb_sparksql_data + ','.join(table_subdirs)])
logging.info(stats)
if not stats['success']:
raise errors.Benchmarks.PrepareException(
'Creating tables from {}/* failed'.format(FLAGS.sparksql_data))
def Prepare(benchmark_spec):
"""Installs and sets up dataset on the Spark clusters.
Copies scripts and all the queries to cloud.
Creates external Hive tables for data (unless BigQuery is being used).
Args:
benchmark_spec: The benchmark specification
"""
dpb_service_instance = benchmark_spec.dpb_service
# buckets must start with a letter
bucket = 'pkb-' + benchmark_spec.uuid.split('-')[0]
storage_service = dpb_service_instance.storage_service
storage_service.MakeBucket(bucket)
benchmark_spec.bucket = bucket
benchmark_spec.base_dir = dpb_service_instance.PERSISTENT_FS_PREFIX + bucket
benchmark_spec.staged_queries = _LoadAndStageQueries(
storage_service, benchmark_spec.base_dir)
for script in [SPARK_TABLE_SCRIPT, SPARK_SQL_RUNNER_SCRIPT]:
src_url = data.ResourcePath(script)
storage_service.CopyToBucket(src_url, bucket, script)
benchmark_spec.table_subdirs = []
if FLAGS.dpb_sparksql_data:
table_dir = FLAGS.dpb_sparksql_data.rstrip('/') + '/'
stdout = storage_service.List(table_dir)
for line in stdout.split('\n'):
# GCS will sometimes list the directory itself.
if line and line != table_dir:
benchmark_spec.table_subdirs.append(
re.split(' |/', line.rstrip('/')).pop())
benchmark_spec.data_dir = FLAGS.dpb_sparksql_data
if FLAGS.dpb_sparksql_copy_to_hdfs:
benchmark_spec.data_dir = 'hdfs:/tmp/spark_sql/'
dpb_service_instance.DistributedCopy(table_dir,
benchmark_spec.data_dir)
# Create external Hive tables
if FLAGS.dpb_sparksql_create_hive_tables:
try:
result = dpb_service_instance.SubmitJob(
pyspark_file=os.path.join(benchmark_spec.base_dir,
SPARK_TABLE_SCRIPT),
job_type=BaseDpbService.PYSPARK_JOB_TYPE,
job_arguments=[
benchmark_spec.data_dir,
','.join(benchmark_spec.table_subdirs)
])
logging.info(result)
except dpb_service.JobSubmissionError as e:
raise errors.Benchmarks.PrepareException(
'Creating tables from {}/* failed'.format(
benchmark_spec.data_dir)) from e
def _PrepareWithIsoFile(vm, speccpu_vm_state):
"""Prepares the VM to run using the iso file.
Copies the iso to the VM, mounts it, and extracts the contents. Copies the
config file to the VM. Runs the SPEC install.sh script on the VM.
Args:
vm: BaseVirtualMachine. Recipient of the iso file.
speccpu_vm_state: _SpecCpu2006SpecificState. Modified by this function to
reflect any changes to the VM that may need to be cleaned up.
"""
scratch_dir = vm.GetScratchDir()
# Make cpu2006 directory on the VM.
vm.RemoteCommand('mkdir {0}'.format(speccpu_vm_state.spec_dir))
# Copy the iso to the VM.
local_iso_file_path = data.ResourcePath(_SPECCPU2006_ISO)
speccpu_vm_state.iso_file_path = posixpath.join(scratch_dir,
_SPECCPU2006_ISO)
vm.PushFile(local_iso_file_path, scratch_dir)
# Extract files from the iso to the cpu2006 directory.
speccpu_vm_state.mount_dir = posixpath.join(scratch_dir, _MOUNT_DIR)
vm.RemoteCommand('mkdir {0}'.format(speccpu_vm_state.mount_dir))
vm.RemoteCommand('sudo mount -t iso9660 -o loop {0} {1}'.format(
speccpu_vm_state.iso_file_path, speccpu_vm_state.mount_dir))
vm.RemoteCommand('cp -r {0}/* {1}'.format(speccpu_vm_state.mount_dir,
speccpu_vm_state.spec_dir))
vm.RemoteCommand('chmod -R 777 {0}'.format(speccpu_vm_state.spec_dir))
# Copy the cfg to the VM.
local_cfg_file_path = data.ResourcePath(FLAGS.runspec_config)
cfg_file_name = os.path.basename(local_cfg_file_path)
speccpu_vm_state.cfg_file_path = posixpath.join(speccpu_vm_state.spec_dir,
'config', cfg_file_name)
vm.PushFile(local_cfg_file_path, speccpu_vm_state.cfg_file_path)
# Run SPEC CPU2006 installation.
install_script_path = posixpath.join(speccpu_vm_state.spec_dir,
'install.sh')
vm.RobustRemoteCommand('yes | {0}'.format(install_script_path))
def CreateHpccinf(vm: linux_vm.BaseLinuxVirtualMachine,
benchmark_spec: bm_spec.BenchmarkSpec) -> HpccDimensions:
"""Creates the HPCC input file."""
dimensions = _CalculateHpccDimensions(len(benchmark_spec.vms),
vm.NumCpusForBenchmark(),
vm.total_free_memory_kb)
vm.RemoteCommand(f'rm -f {HPCCINF_FILE}')
vm.RenderTemplate(data.ResourcePath(LOCAL_HPCCINF_FILE),
remote_path=HPCCINF_FILE,
context=dataclasses.asdict(dimensions))
return dimensions
def CheckPrerequisites():
"""Verifies that the required resources are present.
Raises:
perfkitbenchmarker.data.ResourceNotFound: On missing resource.
"""
if FLAGS['num_vms'].present and FLAGS.num_vms < 3:
raise ValueError('cassandra_ycsb requires at least 3 Cassandra VMs.')
cassandra.CheckPrerequisites()
ycsb.CheckPrerequisites()
data.ResourcePath(CREATE_TABLE_SCRIPT)
def _RenderConfig(vm,
leader,
workers,
memory_fraction=SPARK_MEMORY_FRACTION,
configure_s3=False):
"""Load Spark Condfiguration on VM."""
# Use first worker to get worker configuration
worker = workers[0]
worker_cores = worker.NumCpusForBenchmark()
worker_memory_mb = int((worker.total_memory_kb / 1024) * memory_fraction)
driver_memory_mb = int((leader.total_memory_kb / 1024) * memory_fraction)
spark_conf = GetConfiguration(driver_memory_mb=driver_memory_mb,
worker_memory_mb=worker_memory_mb,
worker_cores=worker_cores,
num_workers=len(workers),
configure_s3=configure_s3)
if vm.scratch_disks:
# TODO(pclay): support multiple scratch disks. A current suboptimal
# workaround is RAID0 local_ssds with --num_striped_disks.
scratch_dir = posixpath.join(vm.GetScratchDir(), 'spark')
else:
scratch_dir = posixpath.join('/tmp/pkb/local_scratch', 'spark')
optional_tools = None
if configure_s3:
optional_tools = 'hadoop-aws'
context = {
'spark_conf': spark_conf,
'leader_ip': leader.internal_ip,
'worker_ips': [vm.internal_ip for vm in workers],
'scratch_dir': scratch_dir,
'worker_vcpus': worker_cores,
'spark_private_key': SPARK_PRIVATE_KEY,
'worker_memory': spark_conf[SPARK_WORKER_MEMORY],
'hadoop_cmd': hadoop.HADOOP_CMD,
'python_cmd': 'python3',
'optional_tools': optional_tools
}
for file_name in DATA_FILES:
file_path = data.ResourcePath(file_name)
if file_name == 'spark/workers.j2':
# Spark calls its worker list slaves.
file_name = 'spark/slaves.j2'
remote_path = posixpath.join(SPARK_CONF_DIR,
os.path.basename(file_name))
if file_name.endswith('.j2'):
vm.RenderTemplate(file_path,
os.path.splitext(remote_path)[0], context)
else:
vm.RemoteCopy(file_path, remote_path)
def _InstallPostgresServer(self):
if self.spec.engine_version == POSTGRES_13_VERSION:
self.server_vm.Install('postgres13')
else:
raise UnsupportedError('Only postgres version 13 is currently supported')
vm = self.server_vm
version = self.spec.engine_version
postgres_conf_path = POSTGRES_CONFIG_PATH.format(version)
postgres_conf_file = postgres_conf_path + POSTGRES_CONFIG
postgres_hba_conf_file = postgres_conf_path + POSTGRES_HBA_CONFIG
vm.PushFile(data.ResourcePath(
posixpath.join(POSTGRES_RESOURCE_PATH, POSTGRES_HBA_CONFIG)))
vm.RemoteCommand('sudo -u postgres psql postgres -c '
'"ALTER USER postgres PASSWORD \'%s\';"'
% self.spec.database_password)
vm.RemoteCommand('sudo -u postgres psql postgres -c '
'"CREATE ROLE %s LOGIN SUPERUSER PASSWORD \'%s\';"' %
(self.spec.database_username,
self.spec.database_password))
# Change the directory to scratch
vm.RemoteCommand(
'sudo sed -i.bak '
'"s:\'/var/lib/postgresql/{0}/main\':\'{1}/postgresql/{0}/main\':" '
'/etc/postgresql/{0}/main/postgresql.conf'.format(
version, self.server_vm.GetScratchDir()))
# Accept remote connection
vm.RemoteCommand(
'sudo sed -i.bak '
r'"s:\#listen_addresses ='
' \'localhost\':listen_addresses = \'*\':" '
'{}'.format(postgres_conf_file))
# Set the size of the shared buffer
vm.RemoteCommand(
'sudo sed -i.bak "s:shared_buffers = 128MB:shared_buffers = {}GB:" '
'{}'.format(self.postgres_shared_buffer_size, postgres_conf_file))
# Update data path to new location
vm.RemoteCommand('sudo rsync -av /var/lib/postgresql /scratch')
# # Use cat to move files because mv will override file permissions
self.server_vm.RemoteCommand(
"sudo bash -c "
"'cat pg_hba.conf > "
"{}'".format(postgres_hba_conf_file))
self.server_vm.RemoteCommand(
'sudo cat {}'.format(postgres_conf_file))
self.server_vm.RemoteCommand(
'sudo cat {}'.format(postgres_hba_conf_file))
vm.RemoteCommand('sudo systemctl restart postgresql')
def _PrepareSpec(vm):
"""Prepares a SPEC client by copying SPEC to the VM."""
mount_dir = 'spec_mnt'
vm.RemoteCommand('mkdir %s' % mount_dir)
vm.RemoteCommand('mkdir %s' % _SPEC_DIR)
vm.InstallPreprovisionedBenchmarkData('specsfs2014', [_SPEC_SFS_2014_ISO],
'~/')
vm.PushFile(data.ResourcePath(_SPEC_SFS_2014_LICENSE), _SPEC_DIR)
vm.RemoteCommand('sudo mount -t iso9660 -o loop %s %s' %
(_SPEC_SFS_2014_ISO, mount_dir))
vm.RemoteCommand('cp -r %s/* %s' % (mount_dir, _SPEC_DIR))
vm.RemoteCommand('sudo umount {0} && sudo rm -rf {0}'.format(mount_dir))
def _GetWorkloadFileList():
"""Returns the list of workload files to run.
Returns:
In order of preference:
* The argument to --ycsb_workload_files.
* Bundled YCSB workloads A and B.
"""
if FLAGS.ycsb_workload_files:
return FLAGS.ycsb_workload_files
return [data.ResourcePath(os.path.join('ycsb', workload))
for workload in ('workloada', 'workloadb')]
def PushDataFile(self, data_file, remote_path=''):
"""Upload a file in perfkitbenchmarker.data directory to the VM.
Args:
data_file: The filename of the file to upload.
remote_path: The destination for 'data_file' on the VM. If not specified,
the file will be placed in the user's home directory.
Raises:
perfkitbenchmarker.data.ResourceNotFound: if 'data_file' does not exist.
"""
file_path = data.ResourcePath(data_file)
self.PushFile(file_path, remote_path)
def PrepareBinaries(vms: List[linux_vm.BaseLinuxVirtualMachine]) -> None:
"""Prepare binaries on all vms."""
if hpcc.USE_INTEL_COMPILED_HPL.value:
intelmpi.NfsExportIntelDirectory(vms)
vm_util.RunThreaded(lambda vm: vm.Install('numactl'), vms)
return
headnode_vm = vms[0]
if FLAGS.hpcc_binary:
headnode_vm.PushFile(data.ResourcePath(FLAGS.hpcc_binary), './hpcc')
else:
headnode_vm.RemoteCommand(f'cp {hpcc.HPCC_DIR}/hpcc hpcc')
vm_util.RunThreaded(lambda vm: _PrepareBinaries(headnode_vm, vm), vms[1:])
def ConfigureAndStart(master, workers, start_yarn=True):
"""Configure hadoop on a cluster.
Args:
master: VM. Master VM - will be the HDFS NameNode, YARN ResourceManager.
workers: List of VMs. Each VM will run an HDFS DataNode, YARN node.
start_yarn: bool. Start YARN and JobHistory server? Set to False if HDFS is
the only service required. Default: True.
"""
vms = [master] + workers
fn = functools.partial(_RenderConfig, master_ip=master.internal_ip,
worker_ips=[worker.internal_ip for worker in workers])
vm_util.RunThreaded(fn, vms)
master.RemoteCommand(
"rm -f {0} && ssh-keygen -q -t rsa -N '' -f {0}".format(
HADOOP_PRIVATE_KEY))
public_key = master.RemoteCommand('cat {0}.pub'.format(HADOOP_PRIVATE_KEY))[0]
def AddKey(vm):
vm.RemoteCommand('echo "{0}" >> ~/.ssh/authorized_keys'.format(public_key))
vm_util.RunThreaded(AddKey, vms)
context = {'hadoop_dir': HADOOP_DIR,
'vm_ips': [vm.internal_ip for vm in vms],
'start_yarn': start_yarn}
# HDFS setup and formatting, YARN startup
script_path = posixpath.join(HADOOP_DIR, 'start-hadoop.sh')
master.RenderTemplate(data.ResourcePath(START_HADOOP_SCRIPT),
script_path, context=context)
master.RemoteCommand('bash {0}'.format(script_path), should_log=True)
logging.info('Sleeping 10s for Hadoop nodes to join.')
time.sleep(10)
logging.info('Checking HDFS status.')
hdfs_online_count = _GetHDFSOnlineNodeCount(master)
if hdfs_online_count != len(workers):
raise ValueError('Not all nodes running HDFS: {0} < {1}'.format(
hdfs_online_count, len(workers)))
else:
logging.info('HDFS running on all %d workers', len(workers))
if start_yarn:
logging.info('Checking YARN status.')
yarn_online_count = _GetYARNOnlineNodeCount(master)
if yarn_online_count != len(workers):
raise ValueError('Not all nodes running YARN: {0} < {1}'.format(
yarn_online_count, len(workers)))
else:
logging.info('YARN running on all %d workers', len(workers))
def _PostCreate(self):
"""Acquire cluster authentication."""
cmd = util.GcloudCommand(
self, 'container', 'clusters', 'get-credentials', self.name)
env = self._GetRequiredGkeEnv()
env['KUBECONFIG'] = FLAGS.kubeconfig
cmd.IssueRetryable(env=env)
if self.gpu_count:
kubernetes_helper.CreateFromFile(NVIDIA_DRIVER_SETUP_DAEMON_SET_SCRIPT)
kubernetes_helper.CreateFromFile(
data.ResourcePath(NVIDIA_UNRESTRICTED_PERMISSIONS_DAEMON_SET))
def _PrepareBucket(benchmark_spec):
"""Prepares the GCS bucket for object storage test.
First creates a bucket in the specified bucket region. Then populates the test
bucket with contents of https://github.com/cloudharmony/web-probe as well as
larger generated test files. The new bucket is saved to later be set as the
test endpoint for the object storage test.
Args:
benchmark_spec: the benchmark specification. Contains all data that is
required to run the benchmark.
"""
# set up Google Cloud Service
service = gcs.GoogleCloudStorageService()
location = cloud_harmony_util.GetRegionFromZone(ENDPOINT_ZONE.value)
service.PrepareService(location)
# create bucket in specified bucket region
bucket = f'ch-{location}-{FLAGS.run_uri}'
bucket_uri = f'gs://{bucket}'
service.MakeBucket(bucket)
# set default permissions to allow cloudharmony test file access
perm_cmd = ['gsutil', 'defacl', 'set', 'public-read', bucket_uri]
vm_util.IssueCommand(perm_cmd)
# set bucket lifecyle to ensure bucket deletion after 30 days
lifecyle_config_file = data.ResourcePath(
'cloudharmony_network_gcp_lifecycle.json')
lc_cmd = ['gsutil', 'lifecycle', 'set', lifecyle_config_file, bucket_uri]
vm_util.IssueCommand(lc_cmd)
# prepare preprovisioned test data
tmp_dir = vm_util.GetTempDir()
tmp_probe_file = posixpath.join(tmp_dir,
cloud_harmony_network.WEB_PROBE_TAR)
wget_install_cmd = ['sudo', 'apt-get', 'install', 'wget']
vm_util.IssueCommand(wget_install_cmd)
wget_cmd = ['wget', '-O', tmp_probe_file, cloud_harmony_network.WEB_PROBE]
vm_util.IssueCommand(wget_cmd)
tar_cmd = ['tar', 'zxf', tmp_probe_file, '-C', tmp_dir]
vm_util.IssueCommand(tar_cmd)
remote_probe_dir = posixpath.join(tmp_dir, 'probe')
dd_cmd = [
'dd', 'if=/dev/urandom', f'of={remote_probe_dir}/test10gb.bin',
'bs=10240', 'count=1048576'
]
vm_util.IssueCommand(dd_cmd)
# copy preprovisioned test data to test bucket
src_path = posixpath.join(remote_probe_dir, '*')
dst_url = f'{bucket_uri}/probe'
cp_cmd = ['gsutil', 'cp', '-r', src_path, dst_url]
vm_util.IssueCommand(cp_cmd, raise_on_timeout=False)
# save the service and the bucket name
benchmark_spec.service = service
benchmark_spec.bucket = bucket
def Prepare(self, benchmark_name: str) -> None:
"""Prepares the client vm to execute query.
Installs the sql server tool dependencies.
Args:
benchmark_name: String name of the benchmark, to allow extraction and
usage of benchmark specific artifacts (certificates, etc.) during client
vm preparation.
"""
self.client_vm.Install('pip')
self.client_vm.RemoteCommand('sudo pip install absl-py')
self.client_vm.Install('mssql_tools')
self.whitelist_ip = self.client_vm.ip_address
cmd = [
azure.AZURE_PATH, 'sql', 'server', 'firewall-rule', 'create',
'--name', self.whitelist_ip, '--resource-group',
self.resource_group, '--server', self.server_name,
'--end-ip-address', self.whitelist_ip, '--start-ip-address',
self.whitelist_ip
]
vm_util.IssueCommand(cmd)
# Push the framework to execute a sql query and gather performance details
service_specific_dir = os.path.join('edw',
Azuresqldatawarehouse.SERVICE_TYPE)
self.client_vm.PushFile(
data.ResourcePath(
os.path.join(service_specific_dir, 'script_runner.sh')))
runner_permission_update_cmd = 'chmod 755 {}'.format(
'script_runner.sh')
self.client_vm.RemoteCommand(runner_permission_update_cmd)
self.client_vm.PushFile(
data.ResourcePath(os.path.join('edw', 'script_driver.py')))
self.client_vm.PushFile(
data.ResourcePath(
os.path.join(service_specific_dir,
'provider_specific_script_driver.py')))
def PrepareBinaries(vms):
"""Prepare binaries on all vms."""
master_vm = vms[0]
if FLAGS.hpcc_binary:
master_vm.PushFile(data.ResourcePath(FLAGS.hpcc_binary), './hpcc')
else:
master_vm.RemoteCommand('cp %s/hpcc hpcc' % hpcc.HPCC_DIR)
for vm in vms[1:]:
vm.Install('fortran')
master_vm.MoveFile(vm, 'hpcc', 'hpcc')
master_vm.MoveFile(vm, '/usr/bin/orted', 'orted')
vm.RemoteCommand('sudo mv orted /usr/bin/orted')
def PrepareVM(vm, service):
vm.Install('pip')
vm.RemoteCommand('sudo pip install python-gflags==2.0')
vm.RemoteCommand('sudo pip install pyyaml')
vm.Install('openssl')
# Prepare data on vm, create a run directory in temporary directory, and add
# permission.
vm.RemoteCommand('sudo mkdir -p /tmp/run/')
vm.RemoteCommand('sudo chmod 777 /tmp/run/')
vm.RemoteCommand('sudo mkdir -p /tmp/run/temp/')
vm.RemoteCommand('sudo chmod 777 /tmp/run/temp/')
file_path = data.ResourcePath(DATA_FILE)
vm.PushFile(file_path, '/tmp/run/')
for file_name in API_TEST_SCRIPT_FILES + service.APIScriptFiles():
path = data.ResourcePath(os.path.join(API_TEST_SCRIPTS_DIR, file_name))
logging.info('Uploading %s to %s', path, vm)
vm.PushFile(path, '/tmp/run/')
