from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import netperf
flags.DEFINE_integer('netperf_max_iter', None,
'Maximum number of iterations to run during '
'confidence interval estimation. If unset, '
'a single iteration will be run.',
lower_bound=3, upper_bound=30)
flags.DEFINE_integer('netperf_test_length', 60,
'netperf test length, in seconds',
lower_bound=1)
flags.DEFINE_bool('netperf_enable_histograms', True,
'Determines whether latency histograms are '
'collected/reported. Only for *RR benchmarks')
flag_util.DEFINE_integerlist('netperf_num_streams', flag_util.IntegerList([1]),
'Number of netperf processes to run. Netperf '
'will run once for each value in the list.',
module_name=__name__)
flags.DEFINE_integer('netperf_thinktime', 0,
'Time in nanoseconds to do work for each request.')
flags.DEFINE_integer('netperf_thinktime_array_size', 0,
'The size of the array to traverse for thinktime.')
flags.DEFINE_integer('netperf_thinktime_run_length', 0,
'The number of contiguous numbers to sum at a time in the '
'thinktime array.')
ALL_BENCHMARKS = ['TCP_RR', 'TCP_CRR', 'TCP_STREAM', 'UDP_RR']
flags.DEFINE_list('netperf_benchmarks', ALL_BENCHMARKS,
flags.DEFINE_integer('terasort_num_rows', 10000,
'Number of 100-byte rows to generate.')
flags.DEFINE_string(
'terasort_unsorted_dir', 'tera_gen_data', 'Location of '
'the unsorted data. TeraGen writes here, and TeraSort '
'reads from here.')
flags.DEFINE_string(
'terasort_data_base', 'terasort_data/',
'The benchmark will append to this to create three '
'directories: one for the generated, unsorted data, '
'one for the sorted data, and one for the validate '
'data. If using a static cluster or if using object '
'storage buckets, you must cleanup.')
flags.DEFINE_bool(
'terasort_append_timestamp', True, 'Append a timestamp to '
'the directories given by terasort_unsorted_dir, '
'terasort_sorted_dir, and terasort_validate_dir')
FLAGS = flags.FLAGS
def GetConfig(user_config):
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
def Prepare(benchmark_spec):
pass
def Run(benchmark_spec):
"""Executes the given jar on the specified Spark cluster.
from perfkitbenchmarker import disk
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import os_types
from perfkitbenchmarker import virtual_machine
from perfkitbenchmarker import vm_util
from perfkitbenchmarker import windows_packages
import timeout_decorator
import winrm
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'log_windows_password', False,
'Whether to log passwords for Windows machines. This can be useful in '
'the event of needing to manually RDP to the instance.')
SMB_PORT = 445
WINRM_PORT = 5986
RDP_PORT = 3389
# This startup script enables remote mangement of the instance. It does so
# by creating a WinRM listener (using a self-signed cert) and opening
# the WinRM port in the Windows firewall.
# It also sets up RDP for manual debugging.
_STARTUP_SCRIPT = """
Enable-PSRemoting -Force
$cert = New-SelfSignedCertificate -DnsName hostname -CertStoreLocation `
Cert:\\LocalMachine\\My\\
New-Item WSMan:\\localhost\\Listener -Transport HTTPS -Address * `
-CertificateThumbPrint $cert.Thumbprint -Force
CONTAINER_MOUNT_DIR = '/mnt'
CONTAINER_WORK_DIR = '/root'
# This pair of scripts used for executing long-running commands, which will be
# resilient in the face of SSH connection errors.
# EXECUTE_COMMAND runs a command, streaming stdout / stderr to a file, then
# writing the return code to a file. An exclusive lock is acquired on the return
# code file, so that other processes may wait for completion.
EXECUTE_COMMAND = 'execute_command.py'
# WAIT_FOR_COMMAND waits on the file lock created by EXECUTE_COMMAND,
# then copies the stdout and stderr, exiting with the status of the command run
# by EXECUTE_COMMAND.
WAIT_FOR_COMMAND = 'wait_for_command.py'
flags.DEFINE_bool(
'setup_remote_firewall', False,
'Whether PKB should configure the firewall of each remote'
'VM to make sure it accepts all internal connections.')
flags.DEFINE_list(
'sysctl', [],
'Sysctl values to set. This flag should be a comma-separated '
'list of path=value pairs. Each value will be written to the '
'corresponding path. For example, if you pass '
'--sysctls=vm.dirty_background_ratio=10,vm.dirty_ratio=25, '
'PKB will run "sysctl vm.dirty_background_ratio=10 '
'vm.dirty_ratio=25" before starting the benchmark.')
flags.DEFINE_list(
'set_files',
[],
'Arbitrary filesystem configuration. This flag should be a '
'Size, in gb, for all data disks.')
flags.DEFINE_integer('scratch_disk_iops', None,
'IOPS for Provisioned IOPS (SSD) volumes in AWS.')
flags.DEFINE_integer('num_striped_disks',
None,
'The number of data disks to stripe together to form one '
'"logical" data disk. This defaults to 1 '
'(except with local disks), which means no striping. '
'When using local disks, they default to striping '
'all disks together. The striped disks will appear as '
'one disk (data_disk_0) in the metadata.',
lower_bound=1)
flags.DEFINE_bool(
'install_packages', None,
'Override for determining whether packages should be '
'installed. If this is false, no packages will be installed '
'on any VMs. This option should probably only ever be used '
'if you have already created an image with all relevant '
'packages installed.')
flags.DEFINE_bool(
'stop_after_benchmark_failure', False,
'Determines response when running multiple benchmarks serially and a '
'benchmark run fails. When True, no further benchmarks are scheduled, and '
'execution ends. When False, benchmarks continue to be scheduled. Does not '
'apply to keyboard interrupts, which will always prevent further '
'benchmarks from being scheduled.')
flags.DEFINE_boolean('ignore_package_requirements', False,
'Disables Python package requirement runtime checks.')
flags.DEFINE_enum('spark_service_type', None,
[spark_service.PKB_MANAGED, spark_service.PROVIDER_MANAGED],
'Type of spark service to use')
'Defaults: K.')
flags.DEFINE_integer('diskspd_stride_or_alignment', 64,
'If the access pattern is sequential, then this value'
'means the stride for the access'
'If the access pattern is random, then this value means'
'the specified number of bytes that random I/O aligns to.'
'Defaults: 64K. Unit: KB, can be set')
flags.DEFINE_enum('diskspd_stride_or_alignment_unit', 'K', ['K', 'M', 'G', 'b'],
'The unit of the stride_or_alignment,'
'available option: K|M|G|b'
'Defaults: K.')
flags.DEFINE_bool('diskspd_large_page', False,
'Whether use large page for IO buffers. '
'Defaults: False')
flags.DEFINE_bool('diskspd_latency_stats', False,
'Whether measure the latency statistics'
'Defaults: False')
flags.DEFINE_bool('diskspd_disable_affinity', False,
'Whether to diable the group affinity,'
'group affinity is to round robin tasks. '
'across processor group. '
'Defaults: False')
flags.DEFINE_bool('diskspd_write_through', True,
'Whether to enable write through IO. '
'Defaults: True')
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing abstract class for a capacity reservation for VMs."""
from perfkitbenchmarker import flags
from perfkitbenchmarker import resource
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'use_capacity_reservations', False,
'Whether to use capacity reservations for virtual '
'machines. Only supported on AWS.')
def GetResourceClass(cloud):
"""Get the CapacityReservation class corresponding to 'cloud'.
Args:
cloud: name of cloud to get the class for.
Returns:
Cloud-specific implementation of BaseCapacityReservation.
"""
return resource.GetResourceClass(BaseCapacityReservation, CLOUD=cloud)
flags.DEFINE_integer('udp_stream_seconds', 3,
'The amount of time to run the UDP stream test.')
flags.DEFINE_integer('udp_client_threads', 1,
'Number of parallel client threads to run.')
flags.DEFINE_integer('udp_buffer_len', 100, 'UDP packet size in bytes.')
flags.DEFINE_integer('tcp_stream_seconds', 3,
'The amount of time to run the TCP stream test.')
flags.DEFINE_integer('tcp_number_of_streams', 10,
'The number of parrallel streams to run in the TCP test')
flags.DEFINE_bool('run_tcp', True,
'setting to false will disable the run of the TCP test')
flags.DEFINE_bool('run_udp', False,
'setting to true will enable the run of the UDP test')
IPERF3_DIR = 'iperf-3.1.3-win64'
IPERF3_ZIP = IPERF3_DIR + '.zip'
IPERF3_URL = 'http://iperf.fr/download/windows/' + IPERF3_ZIP
IPERF3_OUT_FILE = 'iperf_results'
IPERF3_UDP_PORT = 5201
IPERF3_TCP_PORT = IPERF3_UDP_PORT
def Install(vm):
zip_path = ntpath.join(vm.temp_dir, IPERF3_ZIP)
'nuttcp_min_bandwidth_mb', 100,
'The minimum bandwidth, in megabytes, to test in a '
'UDP stream.')
flags.DEFINE_integer(
'nuttcp_bandwidth_step_mb', 1000,
'The amount of megabytes to increase bandwidth in each '
'UDP stream test.')
flags.DEFINE_integer('nuttcp_udp_stream_seconds', 10,
'The amount of time to run the UDP stream test.')
flags.DEFINE_integer('nuttcp_udp_packet_size', 1420,
'The size of each UDP packet sent in the UDP stream.')
flags.DEFINE_bool('nuttcp_udp_run_both_directions', False,
'Run the test twice, using each VM as a source.')
flags.DEFINE_integer('nuttcp_udp_iterations', 1,
'The number of consecutive tests to run.')
NUTTCP_DIR = 'nuttcp-8.1.4.win64'
NUTTCP_ZIP = NUTTCP_DIR + '.zip'
NUTTCP_URL = 'http://nuttcp.net/nuttcp/nuttcp-8.1.4/binaries/' + NUTTCP_ZIP
def Install(vm):
"""Installs the nuttcp package on the VM."""
zip_path = ntpath.join(vm.temp_dir, NUTTCP_ZIP)
vm.DownloadFile(NUTTCP_URL, zip_path)
vm.UnzipFile(zip_path, vm.temp_dir)
# https://github.com/NVIDIA/DeepLearningExamples/pull/386 is merged
GITHUB_MODELS_URL = 'https://github.com/changlan/DeepLearningExamples.git'
BERT_BASE_URL = 'https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip'
BERT_LARGE_URL = 'https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-24_H-1024_A-16.zip'
flags.DEFINE_enum(
'horovod_model', 'resnet-50',
['resnet-50', 'bert-base', 'bert-large', 'maskrcnn', 'resnext-101'],
'name of the model to run.')
flags.DEFINE_integer('horovod_batch_size', 64, 'Batch size per compute device.')
flags.DEFINE_integer('horovod_num_steps', 10,
'Number of steps (epochs for BERT) to train for. ')
flags.DEFINE_bool('horovod_synthetic', False,
'Whether to train with synthetic data.')
flags.DEFINE_enum('horovod_max_seq_len', '128', ['128', '384'],
'Max sequence length for BERT.')
flags.DEFINE_enum('horovod_precision', 'fp16', ['fp16', 'fp32'], 'Precision.')
flags.DEFINE_string(
'horovod_cuda_visible_devices', None,
'GPU identifiers are given as integer indices or as UUID strings.')
flags.DEFINE_bool('horovod_bert_finetune', True,
'Pretrain or finetune a BERT model.')
flags.DEFINE_bool('horovod_timelime', False, 'Enable timeline in Horovod.')
'tpch': 'TPC-H'
}
SPARK_SQL = dpb_service.BaseDpbService.SPARKSQL_JOB_TYPE
PYSPARK = dpb_service.BaseDpbService.PYSPARK_JOB_TYPE
flags.DEFINE_string(
'dpb_sparksql_data', None,
'The HCFS based dataset to run Spark SQL query '
'against')
flags.DEFINE_enum(
'dpb_sparksql_job_type', PYSPARK, [PYSPARK, SPARK_SQL],
'How to trigger the query. Either with the spark-sql CLI '
'or with a PySpark harness inside PKB.')
flags.DEFINE_bool('dpb_sparksql_create_hive_tables', False,
'Whether to load dpb_sparksql_data into external hive tables '
'or not.')
flags.DEFINE_string(
'dpb_sparksql_data_format', None,
"Format of data to load. Assumed to be 'parquet' for HCFS "
"and 'bigquery' for bigquery if unspecified.")
flags.DEFINE_enum('dpb_sparksql_query', 'tpcds_2_4', BENCHMARK_NAMES.keys(),
'A list of query to run on dpb_sparksql_data')
flags.DEFINE_list('dpb_sparksql_order', [],
'The names (numbers) of the queries to run in order. '
'If omitted all queries are run in lexographic order.')
flags.DEFINE_string(
'spark_bigquery_connector',
'gs://spark-lib/bigquery/spark-bigquery-latest.jar',
'The Spark BigQuery Connector jar to pass to the Spark Job')
flags.DEFINE_list(
'The number of VMs per dedicated host. If None, VMs will be packed on a '
'single host until no more can be packed at which point a new host will '
'be created.')
flags.DEFINE_integer(
'num_cpus_override', None,
'Rather than detecting the number of CPUs present on the machine, use this '
'value if set. Some benchmarks will use this number to automatically '
'scale their configurations; this can be used as a method to control '
'benchmark scaling. It will also change the num_cpus metadata '
'published along with the benchmark data.')
flags.DEFINE_list(VM_METADATA, [], 'Metadata to add to the vm. It expects'
'key:value pairs.')
flags.register_validator(VM_METADATA, ValidateVmMetadataFlag)
flags.DEFINE_bool(
'skip_firewall_rules', False,
'If set, this run will not create firewall rules. This is useful if the '
'user project already has all of the firewall rules in place and/or '
'creating new ones is expensive')
flags.DEFINE_bool(
'preprovision_ignore_checksum', False,
'Ignore checksum verification for preprovisioned data. '
'Not recommended, please use with caution')
# Note: If adding a gpu type here, be sure to add it to
# the flag definition in pkb.py too.
VALID_GPU_TYPES = ['k80', 'p100', 'v100', 'p4', 'p4-vws', 't4']
def GetVmSpecClass(cloud):
"""Returns the VmSpec class corresponding to 'cloud'."""
return spec.GetSpecClass(BaseVmSpec, CLOUD=cloud)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing OpenMPI installation and cleanup functions."""
from perfkitbenchmarker import flags
from perfkitbenchmarker.linux_packages import INSTALL_DIR
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'openmpi_enable_shared', False,
'Whether openmpi should build shared libraries '
'in addition to static ones.')
MPI_DIR = '%s/openmpi-3.1.2' % INSTALL_DIR
MPI_TAR = 'openmpi-3.1.2.tar.gz'
MPI_URL = 'https://download.open-mpi.org/release/open-mpi/v3.1/' + MPI_TAR
MPI_VERSION = '3.1.2'
def _Install(vm):
"""Installs the OpenMPI package on the VM."""
vm.Install('build_tools')
vm.Install('wget')
vm.RemoteCommand('wget %s -P %s' % (MPI_URL, INSTALL_DIR))
vm.RemoteCommand('cd %s && tar xvfz %s' % (INSTALL_DIR, MPI_TAR))
make_jobs = vm.num_cpus
'Mode to run: train, eval, train_and_eval')
flags.DEFINE_integer('resnet_train_epochs', 90,
'The Number of epochs to use for training.', lower_bound=1)
flags.DEFINE_integer('resnet_train_batch_size', 1024,
'Global (not per-shard) batch size for training')
flags.DEFINE_integer('resnet_eval_batch_size', 1024,
'Global (not per-shard) batch size for evaluation')
flags.DEFINE_enum('resnet_data_format', 'channels_last',
['channels_first', 'channels_last'],
'A flag to override the data format used in the model. The '
'value is either channels_first or channels_last. To run the '
'network on CPU or TPU, channels_last should be used. For GPU'
', channels_first will improve performance.')
flags.DEFINE_bool('resnet_skip_host_call', False, 'Skip the host_call which is '
'executed every training step. This is generally used for '
'generating training summaries (train loss, learning rate, '
'etc...). When --skip_host_call=false, there could be a '
'performance drop if host_call function is slow and cannot '
'keep up with the TPU-side computation.')
flags.DEFINE_integer(
'resnet_epochs_per_eval', 2, 'Controls how often evaluation is performed.'
' Since evaluation is fairly expensive, it is advised to evaluate as '
'infrequently as possible (i.e. up to --train_steps, which evaluates the '
'model only after finishing the entire training regime).', lower_bound=2)
def GetConfig(user_config):
"""Load and return benchmark config.
Args:
user_config: user supplied configuration (flags and config file)
GCP:
machine_type: n1-standard-4
boot_disk_size: 500
AWS:
machine_type: m4.xlarge
vm_count: 2
"""
# This points to a file on the spark cluster.
DEFAULT_CLASSNAME = 'org.apache.spark.examples.SparkPi'
flags.DEFINE_string('spark_jarfile', None,
'If none, use the spark sample jar.')
flags.DEFINE_string('spark_classname', DEFAULT_CLASSNAME,
'Classname to be used')
flags.DEFINE_bool('spark_print_stdout', True, 'Print the standard '
'output of the job')
flags.DEFINE_list(
'spark_job_arguments', [], 'Arguments to be passed '
'to the class given by spark_classname')
flags.DEFINE_enum(
'spark_job_type', spark_service.SPARK_JOB_TYPE,
[spark_service.SPARK_JOB_TYPE, spark_service.HADOOP_JOB_TYPE],
'Type of the job to submit.')
FLAGS = flags.FLAGS
def GetConfig(user_config):
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
flags.DEFINE_integer('nuttcp_min_bandwidth_mb', 100,
'The minimum bandwidth, in megabytes, to test in a '
'UDP stream.')
flags.DEFINE_integer('nuttcp_bandwidth_step_mb', 1000,
'The amount of megabytes to increase bandwidth in each '
'UDP stream test.')
flags.DEFINE_integer('nuttcp_udp_stream_seconds', 60,
'The amount of time to run the UDP stream test.')
flags.DEFINE_integer('nuttcp_udp_packet_size', 1420,
'The size of each UDP packet sent in the UDP stream.')
flags.DEFINE_bool('nuttcp_udp_run_both_directions', False,
'Run the test twice, using each VM as a source.')
flags.DEFINE_integer('nuttcp_udp_iterations', 1,
'The number of consecutive tests to run.')
flags.DEFINE_bool('nuttcp_udp_unlimited_bandwidth', False,
'Run an "unlimited bandwidth" test')
flags.DEFINE_integer('nuttcp_cpu_sample_time', 3,
'Time, in seconds, to take the CPU usage sample.')
NUTTCP_DIR = 'nuttcp-8.1.4.win64'
NUTTCP_ZIP = NUTTCP_DIR + '.zip'
NUTTCP_URL = 'https://nuttcp.net/nuttcp/nuttcp-8.1.4/binaries/' + NUTTCP_ZIP
_COMMAND_TIMEOUT_BUFFER = 60
from perfkitbenchmarker import background_tasks
from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
FLAGS = flags.FLAGS
flags.DEFINE_integer(
'ntttcp_threads', 1, 'The number of client and server threads for NTttcp '
'to run with.')
flags.DEFINE_integer('ntttcp_time', 60,
'The number of seconds for NTttcp to run.')
flags.DEFINE_bool('ntttcp_udp', False, 'Whether to run a UDP test.')
flags.DEFINE_integer('ntttcp_packet_size', None,
'The size of the packet being used in the test.')
flags.DEFINE_integer(
'ntttcp_sender_sb', -1,
'The size of the send buffer, in Kilo Bytes, on the '
'sending VM. The default is the OS default.')
flags.DEFINE_integer(
'ntttcp_sender_rb', -1,
'The size of the receive buffer, in Kilo Bytes, on the '
'sending VM. The default is the OS default.')
flags.DEFINE_integer(
_FS_TYPE_PERSISTENT = 'persistent'
flags.DEFINE_enum('dpb_terasort_fs_type', _FS_TYPE_EPHEMERAL,
[_FS_TYPE_EPHEMERAL, _FS_TYPE_PERSISTENT],
'The type of File System to use in the Terasort benchmark')
flags.DEFINE_enum(
'dpb_terasort_fs', BaseDpbService.GCS_FS,
[BaseDpbService.GCS_FS, BaseDpbService.S3_FS, BaseDpbService.HDFS_FS],
'File System to use in the Terasort benchmark')
flags.DEFINE_integer('dpb_terasort_num_records', 10000,
'Number of 100-byte rows to generate.')
flags.DEFINE_bool(
'dpb_terasort_pre_cleanup', False,
'Cleanup the terasort directories on the specified filesystem.')
FLAGS = flags.FLAGS
SUPPORTED_DPB_BACKENDS = [dpb_service.DATAPROC, dpb_service.EMR]
JOB_CATEGORY = BaseDpbService.HADOOP_JOB_TYPE
JOB_TYPE = 'terasort'
def GetConfig(user_config):
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
def CheckPrerequisites(benchmark_config):
"""Verifies that the required resources are present.
image: debian-9-tf-v20171009
image_project: ml-images
machine_type: n1-standard-8
zone: us-central1-c
cloud_tpu:
tpu_tf_version: nightly
tpu_zone: us-central1-c
tpu_cidr_range: 10.240.0.0/29
flags:
gcloud_scopes: https://www.googleapis.com/auth/cloud-platform
"""
flags.DEFINE_string('mnist_train_file',
'gs://tfrc-test-bucket/mnist-records/train.tfrecords',
'mnist train file for tensorflow')
flags.DEFINE_bool('mnist_use_tpu', True, 'Use TPUs rather than plain CPUs')
def GetConfig(user_config):
"""Load and return benchmark config.
Args:
user_config: user supplied configuration (flags and config file)
Returns:
loaded benchmark configuration
"""
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
def _UpdateBenchmarkSpecWithFlags(benchmark_spec):
flags.DEFINE_integer('scratch_disk_size', None, 'Size, in gb, for all scratch '
'disks.')
flags.DEFINE_integer('data_disk_size', None, 'Size, in gb, for all data disks.')
flags.DEFINE_integer('scratch_disk_iops', None,
'IOPS for Provisioned IOPS (SSD) volumes in AWS.')
flags.DEFINE_integer('num_striped_disks', None,
'The number of data disks to stripe together to form one '
'"logical" data disk. This defaults to 1 '
'(except with local disks), which means no striping. '
'When using local disks, they default to striping '
'all disks together. The striped disks will appear as '
'one disk (data_disk_0) in the metadata.',
lower_bound=1)
flags.DEFINE_bool('install_packages', None,
'Override for determining whether packages should be '
'installed. If this is false, no packages will be installed '
'on any VMs. This option should probably only ever be used '
'if you have already created an image with all relevant '
'packages installed.')
flags.DEFINE_bool(
'stop_after_benchmark_failure', False,
'Determines response when running multiple benchmarks serially and a '
'benchmark run fails. When True, no further benchmarks are scheduled, and '
'execution ends. When False, benchmarks continue to be scheduled. Does not '
'apply to keyboard interrupts, which will always prevent further '
'benchmarks from being scheduled.')
flags.DEFINE_boolean(
'ignore_package_requirements', False,
'Disables Python package requirement runtime checks.')
flags.DEFINE_enum('spark_service_type', None,
[spark_service.PKB_MANAGED, spark_service.PROVIDER_MANAGED],
'Type of spark service to use')
from __future__ import print_function
import posixpath
import re
from perfkitbenchmarker import flags
from perfkitbenchmarker.linux_packages import INSTALL_DIR
FLAGS = flags.FLAGS
flags.DEFINE_string(
'openmpi_version', '3.1.2',
'OpenMPI version to install, such as 3.1.2 and 4.0.2.'
'Set to empty to ignore the intallation of OpenMPI.')
flags.DEFINE_bool(
'openmpi_enable_shared', False,
'Whether openmpi should build shared libraries '
'in addition to static ones.')
flags.DEFINE_bool('openmpi_with_cuda_support', False,
'Compile with CUDA support')
flags.DEFINE_string(
'openmpi_configs', None,
'command line options to be provided to ./configure for'
'OpenMPI compilation')
MPI_URL_BASE = 'https://download.open-mpi.org/release/open-mpi'
REMOVE_MPI_CMD = 'autoremove -y libopenmpi-dev openmpi-bin openmpi-common'
class MpirunParseOutputError(Exception):
pass
'The number of VMs per dedicated host. If None, VMs will be packed on a '
'single host until no more can be packed at which point a new host will '
'be created.')
flags.DEFINE_integer(
'num_cpus_override', None,
'Rather than detecting the number of CPUs present on the machine, use this '
'value if set. Some benchmarks will use this number to automatically '
'scale their configurations; this can be used as a method to control '
'benchmark scaling. It will also change the num_cpus metadata '
'published along with the benchmark data.')
flags.DEFINE_list(VM_METADATA, [], 'Metadata to add to the vm. It expects'
'key:value pairs.')
flags.register_validator(VM_METADATA, ValidateVmMetadataFlag)
flags.DEFINE_bool(
'skip_firewall_rules', False,
'If set, this run will not create firewall rules. This is useful if the '
'user project already has all of the firewall rules in place and/or '
'creating new ones is expensive')
# Note: If adding a gpu type here, be sure to add it to
# the flag definition in pkb.py too.
VALID_GPU_TYPES = ['k80', 'p100', 'v100', 'p4', 'p4-vws', 't4']
def GetVmSpecClass(cloud):
"""Returns the VmSpec class corresponding to 'cloud'."""
return spec.GetSpecClass(BaseVmSpec, CLOUD=cloud)
def GetVmClass(cloud, os_type):
"""Returns the VM class corresponding to 'cloud' and 'os_type'."""
from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
WRK2_URL = ('https://github.com/giltene/wrk2/archive/'
'c4250acb6921c13f8dccfc162d894bd7135a2979.tar.gz')
WRK2_DIR = posixpath.join(vm_util.VM_TMP_DIR, 'wrk2')
WRK2_PATH = posixpath.join(WRK2_DIR, 'wrk')
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'wrk2_corrected_latency', True,
'Whether or not response latency is corrected.\n'
'If True, wrk2 measure response latency from the time the '
'transmission should have occurred according to the constant '
'throughput configured for the run.\n'
'If False, response latency is the time that actual '
'transmission of a request occured.')
def _Install(vm):
vm.Install('curl')
vm.Install('build_tools')
vm.Install('openssl')
vm.RemoteCommand(
('mkdir -p {0} && '
'curl -L {1} | tar -xzf - -C {0} --strip-components 1').format(
WRK2_DIR, WRK2_URL))
vm.RemoteCommand('make -C {}'.format(WRK2_DIR))
from perfkitbenchmarker import disk
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import os_types
from perfkitbenchmarker import virtual_machine
from perfkitbenchmarker import vm_util
from perfkitbenchmarker import windows_packages
import six
import timeout_decorator
import winrm
FLAGS = flags.FLAGS
flags.DEFINE_bool(
'log_windows_password', False,
'Whether to log passwords for Windows machines. This can be useful in '
'the event of needing to manually RDP to the instance.')
flags.DEFINE_bool(
'set_cpu_priority_high', False,
'Allows executables to be set to High (up from Normal) CPU priority '
'through the SetProcessPriorityToHigh function.')
SMB_PORT = 445
WINRM_PORT = 5986
RDP_PORT = 3389
# This startup script enables remote mangement of the instance. It does so
# by creating a WinRM listener (using a self-signed cert) and opening
# the WinRM port in the Windows firewall.
# It also sets up RDP for manual debugging.
_STARTUP_SCRIPT = """
flags.DEFINE_integer('horovod_batch_size', 64,
'Batch size per compute device.')
flags.DEFINE_integer('horovod_num_epochs', 10,
'Number of epochs to train for. ')
flags.DEFINE_enum('horovod_max_seq_len', '128', ['128', '384'],
'Max sequence length for BERT.')
flags.DEFINE_enum('horovod_precision', 'fp16', ['fp16', 'fp32'], 'Precision.')
flags.DEFINE_string(
'horovod_cuda_visible_devices', None,
'GPU identifiers are given as integer indices or as UUID strings.')
flags.DEFINE_bool('horovod_bert_finetune', True,
'Pretrain or finetune a BERT model.')
flags.DEFINE_bool('horovod_timelime', False, 'Enable timeline in Horovod.')
class HorovodParseOutputError(errors.Benchmarks.RunError):
pass
def GetConfig(user_config):
"""Load and return benchmark config.
Args:
user_config: user supplied configuration (flags and config file)
Returns:
flags.DEFINE_enum('scratch_disk_type', disk.STANDARD,
[disk.STANDARD, disk.REMOTE_SSD, disk.PIOPS, disk.LOCAL],
'Type for all scratch disks. The default is standard')
flags.DEFINE_integer('scratch_disk_iops', 1500,
'IOPS for Provisioned IOPS (SSD) volumes in AWS.')
flags.DEFINE_integer('num_striped_disks',
1, 'The number of disks to stripe together to form one '
'"logical" scratch disk. This defaults to 1 '
'(except with local disks), which means no striping. '
'When using local disks, they default to striping '
'all disks together.',
lower_bound=1)
flags.DEFINE_bool(
'install_packages', True,
'Override for determining whether packages should be '
'installed. If this is false, no packages will be installed '
'on any VMs. This option should probably only ever be used '
'if you have already created an image with all relevant '
'packages installed.')
# Support for using a proxy in the cloud environment.
flags.DEFINE_string(
'http_proxy', '', 'Specify a proxy for HTTP in the form '
'[user:passwd@]proxy.server:port.')
flags.DEFINE_string(
'https_proxy', '', 'Specify a proxy for HTTPS in the form '
'[user:passwd@]proxy.server:port.')
flags.DEFINE_string(
'ftp_proxy', '', 'Specify a proxy for FTP in the form '
'[user:passwd@]proxy.server:port.')
'hammerdb_tpcc_warehouse', 1,
'The number of warehouse used in tpcc benchmarking. '
'Default: 1')
flags.DEFINE_integer(
'hammerdb_tpcc_schema_virtual_user', 1,
'The number of virtual user used when '
'building the schema. '
'Default: 1')
flags.DEFINE_integer(
'hammerdb_tpcc_runtime', 60, 'The running time for tpcc benchmark test'
'Default: 60. Unit: second')
flags.DEFINE_bool(
'hammerdb_run_tpcc', True,
'tpcc is a sql benchmark to measure transaction speed.'
'Default: True')
flags.DEFINE_bool(
'hammerdb_run_tpch', True,
'tpch is a sql benchmark to calculate the query per hour'
'performance metrics.'
'Default: True')
flags.DEFINE_integer(
'hammerdb_tpch_scale_fact', 1, 'The running time for tpcc benchmark test. '
'Default: 60. Unit: second')
flags.DEFINE_integer(
'hammerdb_tpch_virtual_user', 4,
'The virtual user number to run tpch test. '
from perfkitbenchmarker import flags
from perfkitbenchmarker import network
from perfkitbenchmarker import placement_group
from perfkitbenchmarker import providers
from perfkitbenchmarker import resource
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.providers.aws import aws_placement_group
from perfkitbenchmarker.providers.aws import aws_vpc_endpoint
from perfkitbenchmarker.providers.aws import util
flags.DEFINE_string('aws_vpc', None,
'The static AWS VPC id to use. Default creates a new one')
flags.DEFINE_string(
'aws_subnet', None,
'The static AWS subnet id to use. Default creates a new one')
flags.DEFINE_bool('aws_efa', False, 'Whether to use an Elastic Fiber Adapter.')
flags.DEFINE_string('aws_efa_version', '1.8.3',
'Version of AWS EFA to use (must also pass in --aws_efa).')
flags.DEFINE_multi_enum('aws_endpoint', [], ['s3'],
'List of AWS endpoints to create')
FLAGS = flags.FLAGS
REGION = 'region'
ZONE = 'zone'
class AwsFirewall(network.BaseFirewall):
"""An object representing the AWS Firewall."""
f. Raw geo mean performance for each iteration
g. Aggregated geo mean performance using the aggregated query performances
"""
import copy
import enum
import functools
import json
import time
from typing import Any, Dict, List, Text
import numpy as np
from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
flags.DEFINE_bool(
'edw_generate_aggregated_metrics', True,
'Whether the benchmark generates aggregated_metrics such as '
'geomean. Query performance metrics are still generated.')
FLAGS = flags.FLAGS
class EdwPerformanceAggregationError(Exception):
"""Error encountered during aggregation of performance results."""
def geometric_mean(iterable: List[float]) -> float:
"""Function to compute the geo mean for a list of numeric values.
Args:
iterable: A List of Float performance values
