'failed with status "NR". Available results will be saved, and PKB samples '
'will be marked with a metadata value of partial=true. If unset, partial '
'failures are treated as errors.')
flags.DEFINE_boolean(
'runspec_estimate_spec', False,
'Used by the PKB speccpu benchmarks. If set, the benchmark will report '
'an estimated aggregate score even if SPEC CPU did not compute one. '
'This usually occurs when --runspec_iterations is less than 3. '
'--runspec_keep_partial_results is also required to be set. Samples will be'
'created as estimated_SPECint(R)_rate_base and '
'estimated_SPECfp(R)_rate_base. Available results will be saved, '
'and PKB samples will be marked with a metadata value of partial=true. If '
'unset, SPECint(R)_rate_base20** and SPECfp(R)_rate_base20** are listed '
'in the metadata under missing_results.')
flags.DEFINE_enum(
'spec_runmode', BASE_MODE,
[BASE_MODE, PEAK_MODE, ALL_MODE],
'Run mode to use. Defaults to base. ')
VM_STATE_ATTR = 'speccpu_vm_state'
def _CheckTarFile(vm, runspec_config, examine_members, speccpu_vm_state):
"""Performs preliminary checks on the format of tar file downloaded on vm.
Args:
vm: virtual machine
runspec_config: String. User-specified name of the config file that is
expected to be in the tar file.
examine_members: Boolean. If True, this function will examine the tar file's
members to verify that certain required members are present.
BENCHMARK_CONFIG = """
dpb_sparksql_benchmark:
description: Run Spark SQL on dataproc and emr
dpb_service:
service_type: dataproc
worker_group:
vm_spec:
GCP:
machine_type: n1-standard-1
AWS:
machine_type: m5.xlarge
worker_count: 2
"""
flags.DEFINE_string('dpb_sparksql_data', None,
'The dataset to run Spark SQL query')
flags.DEFINE_enum('dpb_sparksql_query', 'tpcds_2_4', ['tpcds_2_4', 'tpch'],
'A list of query to run on dpb_sparksql_data')
flags.DEFINE_list('dpb_sparksql_order', [],
'The order of query templates in each query stream.')
FLAGS = flags.FLAGS
SUPPORTED_DPB_BACKENDS = [dpb_service.DATAPROC, dpb_service.EMR]
JOB_CATEGORY = BaseDpbService.SPARK_JOB_TYPE
JOB_TYPE = 'spark-sql'
# Creates spark table using pyspark by loading the parquet data.
# Args:
# argv[1]: string, The table name in the dataset that this script will create.
# argv[2]: string, The data path of the table.
SPARK_TABLE_SCRIPT = 'spark_table.py'
SPARK_SQL_PERF_GIT = 'https://github.com/databricks/spark-sql-perf.git'
SPARK_SQL_PERF_GIT_COMMIT = '6b2bf9f9ad6f6c2f620062fda78cded203f619c8'
from perfkitbenchmarker import providers
from perfkitbenchmarker import configs
from perfkitbenchmarker import data
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import flag_util
from perfkitbenchmarker import object_storage_service
from perfkitbenchmarker import sample
from perfkitbenchmarker import units
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.providers.gcp import gcs
from perfkitbenchmarker.sample import PercentileCalculator # noqa
flags.DEFINE_enum(
'storage', providers.GCP,
[providers.GCP, providers.AWS, providers.AZURE, providers.OPENSTACK],
'storage provider (GCP/AZURE/AWS/OPENSTACK) to use.')
flags.DEFINE_string('object_storage_region', None,
'Storage region for object storage benchmark.')
flags.DEFINE_string(
'object_storage_gcs_multiregion', None,
'Storage multiregion for GCS in object storage benchmark.')
flags.DEFINE_string('object_storage_storage_class', None,
'Storage class to use in object storage benchmark.')
flags.DEFINE_enum(
'object_storage_scenario', 'all', [
'all', 'cli', 'api_data', 'api_namespace', 'api_multistream',
lock = threading.Lock()
if locked:
if not lock.acquire(False):
raise pickle.UnpicklingError('Cannot acquire lock')
return lock
copy_reg.pickle(thread.LockType, PickleLock)
SUPPORTED = 'strict'
NOT_EXCLUDED = 'permissive'
SKIP_CHECK = 'none'
FLAGS = flags.FLAGS
flags.DEFINE_enum('cloud', providers.GCP, providers.VALID_CLOUDS,
'Name of the cloud to use.')
flags.DEFINE_string(
'scratch_dir', None,
'Base name for all scratch disk directories in the VM.'
'Upon creation, these directories will have numbers'
'appended to them (for example /scratch0, /scratch1, etc).')
flags.DEFINE_enum(
'benchmark_compatibility_checking', SUPPORTED,
[SUPPORTED, NOT_EXCLUDED, SKIP_CHECK],
'Method used to check compatibility between the benchmark '
' and the cloud. ' + SUPPORTED + ' runs the benchmark only'
' if the cloud provider has declared it supported. ' + NOT_EXCLUDED +
' runs the benchmark unless it has been '
' declared not supported by the could provider. ' + SKIP_CHECK +
' does not do the compatibility'
' check. The default is ' + SUPPORTED)
CPU = 'cpu'
MODELS = [
'alexnet', 'googlenet', 'inception-bn', 'inception-resnet-v2',
'inception-v3', 'inception-v4', 'lenet', 'mlp', 'mobilenet', 'resnet-v1',
'resnet', 'resnext', 'vgg'
]
flags.DEFINE_list('mx_models', ['inception-v3', 'vgg', 'alexnet', 'resnet'],
'The network to train')
flags.register_validator(
'mx_models', lambda models: models and set(models).issubset(MODELS),
'Invalid models list. mx_models must be a subset of ' + ', '.join(MODELS))
flags.DEFINE_integer('mx_batch_size', None, 'The batch size for SGD training.')
flags.DEFINE_integer('mx_num_epochs', 80,
'The maximal number of epochs to train.')
flags.DEFINE_enum('mx_device', GPU, [CPU, GPU],
'Device to use for computation: cpu or gpu')
flags.DEFINE_integer(
'mx_num_layers', None, 'Number of layers in the neural '
'network, required by some networks such as resnet')
flags.DEFINE_enum('mx_precision', 'float32', ['float16', 'float32'],
'Precision')
flags.DEFINE_enum(
'mx_key_value_store', 'device',
['local', 'device', 'nccl', 'dist_sync', 'dist_device_sync', 'dist_async'],
'Key-Value store types.')
flags.DEFINE_string('mx_image_shape', None,
'The image shape that feeds into the network.')
DEFAULT_BATCH_SIZE = 64
DEFAULT = 'default'
DEFAULT_BATCH_SIZES_BY_MODEL = {
import posixpath
import re
import tarfile
from perfkitbenchmarker import configs
from perfkitbenchmarker import data
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker import stages
FLAGS = flags.FLAGS
flags.DEFINE_enum(
'benchmark_subset', 'int', ['int', 'fp', 'all'],
'Used by the PKB speccpu2006 benchmark. Specifies a subset of SPEC CPU2006 '
'benchmarks to run.')
flags.DEFINE_string(
'runspec_config', 'linux64-x64-gcc47.cfg',
'Used by the PKB speccpu2006 benchmark. Name of the cfg file to use as the '
'SPEC CPU2006 config file provided to the runspec binary via its --config '
'flag. If the benchmark is run using the cpu2006-1.2.iso file, then the '
'cfg file must be placed in the local PKB data directory and will be '
'copied to the remote machine prior to executing runspec. See README.md '
'for instructions if running with a repackaged cpu2006v1.2.tgz file.')
flags.DEFINE_integer(
'runspec_iterations', 3,
'Used by the PKB speccpu2006 benchmark. The number of benchmark iterations '
'to execute, provided to the runspec binary via its --iterations flag.')
flags.DEFINE_string(
'runspec_define', '',
import logging
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import regex_util
from perfkitbenchmarker import sample
from perfkitbenchmarker.linux_packages import INSTALL_DIR
GIT_REPO = 'https://github.com/leverich/mutilate'
MUTILATE_DIR = '%s/mutilate_benchmark' % INSTALL_DIR
MUTILATE_BIN = '%s/mutilate' % MUTILATE_DIR
APT_PACKAGES = 'scons libevent-dev gengetopt libzmq-dev'
FLAGS = flags.FLAGS
flags.DEFINE_enum(
'mutilate_protocol', 'binary', ['binary', 'ascii'],
'Protocol to use. Supported protocols are binary and ascii.')
flags.DEFINE_list('mutilate_qps', [],
'Target aggregate QPS. If not set, target for peak qps.')
flags.DEFINE_integer('mutilate_time', 300, 'Maximum time to run (seconds).')
flags.DEFINE_string('mutilate_keysize', '16',
'Length of memcached keys (distribution).')
flags.DEFINE_string('mutilate_valuesize', '128',
'Length of memcached values (distribution).')
flags.DEFINE_integer('mutilate_records', 10000,
'Number of memcached records to use.')
flags.DEFINE_float('mutilate_ratio', 0.0,
'Ratio of set:get. By default, read only.')
flags.DEFINE_list(
'mutilate_options', [],
'Additional mutilate long-form options (--) in comma separated form. e.g.'
'extra_zones', [],
'Zones that will be appended to the "zones" list. This is functionally '
'the same, but allows flag matrices to have two zone axes.')
# TODO(user): note that this is currently very GCE specific. Need to create a
# module which can translate from some generic types to provider specific
# nomenclature.
flags.DEFINE_string(
'machine_type', None, 'Machine '
'types that will be created for benchmarks that don\'t '
'require a particular type.')
flags.DEFINE_integer(
'gpu_count', None,
'Number of gpus to attach to the VM. Requires gpu_type to be '
'specified.')
flags.DEFINE_enum(
'gpu_type', None, ['k80', 'p100', 'v100', 'p4', 'p4-vws', 't4'],
'Type of gpus to attach to the VM. Requires gpu_count to be '
'specified.')
flags.DEFINE_integer(
'num_vms', 1, 'For benchmarks which can make use of a '
'variable number of machines, the number of VMs to use.')
flags.DEFINE_string('image', None, 'Default image that will be '
'linked to the VM')
flags.DEFINE_string(
'run_uri', None, 'Name of the Run. If provided, this '
'should be alphanumeric and less than or equal to %d '
'characters in length.' % MAX_RUN_URI_LENGTH)
flags.DEFINE_boolean(
'use_pkb_logging', True, 'Whether to use PKB-specific '
'logging handlers. Disabling this will use the standard '
'ABSL logging directly.')
flags.DEFINE_boolean(
'machine_type', None, 'Machine '
'types that will be created for benchmarks that don\'t '
'require a particular type.')
flags.DEFINE_integer(
'num_vms', 1, 'For benchmarks which can make use of a '
'variable number of machines, the number of VMs to use.')
flags.DEFINE_string('image', None, 'Default image that will be '
'linked to the VM')
flags.DEFINE_string(
'run_uri', None, 'Name of the Run. If provided, this '
'should be alphanumeric and less than or equal to 10 '
'characters in length.')
flags.DEFINE_string(
'owner', getpass.getuser(), 'Owner name. '
'Used to tag created resources and performance records.')
flags.DEFINE_enum('log_level', log_util.INFO, [log_util.DEBUG, log_util.INFO],
'The log level to run at.')
flags.DEFINE_enum(
'file_log_level', log_util.DEBUG, [log_util.DEBUG, log_util.INFO],
'Anything logged at this level or higher will be written to the log file.')
flags.DEFINE_integer(
'duration_in_seconds', None, 'duration of benchmarks. '
'(only valid for mesh_benchmark)')
flags.DEFINE_string(
'static_vm_file', None, 'The file path for the Static Machine file. See '
'static_virtual_machine.py for a description of this file.')
flags.DEFINE_boolean('version', False, 'Display the version and exit.')
flags.DEFINE_enum('scratch_disk_type', None,
[disk.STANDARD, disk.REMOTE_SSD, disk.PIOPS, disk.LOCAL],
'Type for all scratch disks. The default is standard')
flags.DEFINE_string(
'data_disk_type', None,
flags.DEFINE_string('edw_service_endpoint', None,
'If set, the preprovisioned edw cluster endpoint.')
flags.DEFINE_string(
'edw_service_cluster_db', None,
'If set, the db on cluster to use during the benchmark ('
'only applicable when using snapshots).')
flags.DEFINE_string(
'edw_service_cluster_user', None,
'If set, the user authorized on cluster (only applicable '
'when using snapshots).')
flags.DEFINE_string(
'edw_service_cluster_password', None,
'If set, the password authorized on cluster (only '
'applicable when using snapshots).')
flags.DEFINE_enum('edw_query_execution_mode', 'sequential',
['sequential', 'concurrent'],
'The mode for executing the queries on the edw cluster.')
flags.DEFINE_string('edw_service_resource_group', None,
'Needed to manage Azure clusters.')
FLAGS = flags.FLAGS
TYPE_2_PROVIDER = dict([('redshift', 'aws'), ('spectrum', 'aws'),
('bigquery', 'gcp'),
('azuresqldatawarehouse', 'azure')])
TYPE_2_MODULE = dict([
('redshift', 'perfkitbenchmarker.providers.aws.redshift'),
('spectrum', 'perfkitbenchmarker.providers.aws.spectrum'),
('bigquery', 'perfkitbenchmarker.providers.gcp.bigquery'),
('azuresqldatawarehouse', 'perfkitbenchmarker.providers.azure.'
'azure_sql_data_warehouse')
'ceph_monitors', [], 'IP addresses and ports of Ceph Monitors. '
'Must be provided when Ceph scratch disk is required. '
'Example: "127.0.0.1:6789,192.168.1.1:6789"')
flags.DEFINE_string(
'username', 'root',
'User name that Perfkit will attempt to use in order to '
'SSH into Docker instance.')
flags.DEFINE_boolean(
'docker_in_privileged_mode', True,
'If set to True, will attempt to create Docker containers '
'in a privileged mode. Note that some benchmarks execute '
'commands which are only allowed in privileged mode.')
flags.DEFINE_boolean(
'kubernetes_anti_affinity', True,
'If set to True, PKB pods will not be scheduled on the '
'same nodes as other PKB pods.')
flags.DEFINE_multi_string(
'k8s_volume_parameters', None,
'A colon separated key-value pair that will be '
'added to Kubernetes storage class parameters.')
_K8S_PROVISIONERS = [
'kubernetes.io/azure-disk', 'kubernetes.io/gce-pd',
'kubernetes.io/aws-ebs', 'kubernetes.io/glusterfs'
]
flags.DEFINE_enum(
'k8s_volume_provisioner', None, _K8S_PROVISIONERS,
'The name of the provisioner to use for K8s storage '
'classes.')
from perfkitbenchmarker import flags
from perfkitbenchmarker import hpc_util
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import openfoam
from perfkitbenchmarker.linux_packages import openmpi
_DEFAULT_CASE = 'motorbike'
_CASE_PATHS = {
'motorbike': 'tutorials/incompressible/simpleFoam/motorBike',
}
assert _DEFAULT_CASE in _CASE_PATHS
FLAGS = flags.FLAGS
flags.DEFINE_enum('openfoam_case', _DEFAULT_CASE,
sorted(list(_CASE_PATHS.keys())),
'Name of the OpenFOAM case to run.')
flags.DEFINE_list('openfoam_dimensions', ['20_8_8'], 'Dimensions of the case.')
flags.DEFINE_integer('openfoam_num_threads', None,
'The number of threads to run OpenFOAM with.')
flags.DEFINE_string(
'openfoam_mpi_mapping', 'core:SPAN',
'Mpirun process mapping to use as arguments to "mpirun --map-by".')
flags.DEFINE_enum(
'openfoam_decomp_method', 'scotch', ['scotch', 'hierarchical', 'simple'],
'Decomposition method to use in decomposePar. See: '
'https://cfd.direct/openfoam/user-guide/v7-running-applications-parallel/')
flags.DEFINE_integer(
'openfoam_max_global_cells', 200 * 1000 * 1000,
'The maximum number of refinement cells to use in snappHexMeshDict. See: '
'https://cfd.direct/openfoam/user-guide/v6-snappyhexmesh/')
'HPL_nprow',
'HPL_npcol',
'HPL_depth',
'HPL_nbdiv',
'HPL_nbmin',
'HPL_ctop',
},
}
# The names of the benchmarks.
HPCC_BENCHMARKS = sorted(HPCC_METRIC_MAP)
flags.DEFINE_enum(
'hpcc_math_library', HPCC_MATH_LIBRARY_OPEN_BLAS, [
HPCC_MATH_LIBRARY_OPEN_BLAS, HPCC_MATH_LIBRARY_MKL,
HPCC_MATH_LIBRARY_AMD_BLIS
], 'The math library to use when compiling hpcc: openblas, mkl, or '
'amdblis. The default is openblas.')
flags.DEFINE_list(
'hpcc_benchmarks', [], 'A list of benchmarks in HPCC to run. If none are '
'specified (the default), then all of the benchmarks are run. In 1.5.0, '
'the benchmarks may include the following: %s' % ', '.join(HPCC_BENCHMARKS))
flags.register_validator(
'hpcc_benchmarks',
lambda hpcc_benchmarks: set(hpcc_benchmarks).issubset(set(HPCC_BENCHMARKS)))
FLAGS = flags.FLAGS
def _LimitBenchmarksToRun(vm, selected_hpcc_benchmarks):
"""Limits the benchmarks to run.
'network traffic to generate during the run phase '
'of the benchmark')
class IpAddressSubset(object):
"""Enum of options for --ip_addresses."""
REACHABLE = 'REACHABLE'
BOTH = 'BOTH'
INTERNAL = 'INTERNAL'
EXTERNAL = 'EXTERNAL'
ALL = (REACHABLE, BOTH, INTERNAL, EXTERNAL)
flags.DEFINE_enum('ip_addresses', IpAddressSubset.REACHABLE,
IpAddressSubset.ALL,
'For networking tests: use both internal and external '
'IP addresses (BOTH), external and internal only if '
'the receiving VM is reachable by internal IP (REACHABLE), '
'external IP only (EXTERNAL) or internal IP only (INTERNAL)')
flags.DEFINE_enum('background_network_ip_type', IpAddressSubset.EXTERNAL,
(IpAddressSubset.INTERNAL, IpAddressSubset.EXTERNAL),
'IP address type to use when generating background network '
'traffic')
def GetTempDir():
"""Returns the tmp dir of the current run."""
return temp_dir.GetRunDirPath()
def PrependTempDir(file_name):
NCHW = 'NCHW'
NHWC = 'NHWC'
PID_PREFIX = 'TF_PS_PID'
MODELS = ['vgg11', 'vgg16', 'vgg19', 'lenet', 'googlenet', 'overfeat',
'alexnet', 'trivial', 'inception3', 'inception4', 'resnet50',
'resnet101', 'resnet152']
flags.DEFINE_boolean('tf_forward_only', False, '''whether use forward-only or
training for benchmarking''')
flags.DEFINE_list('tf_models', ['inception3', 'vgg16', 'alexnet', 'resnet50'],
'name of the models to run')
flags.register_validator('tf_models',
lambda models: models and set(models).issubset(MODELS),
'Invalid models list. tf_models must be a subset of '
+ ', '.join(MODELS))
flags.DEFINE_enum('tf_data_name', 'imagenet', ['imagenet', 'flowers'],
'Name of dataset: imagenet or flowers.')
flags.DEFINE_integer('tf_batch_size', None, 'batch size per compute device. '
'If not provided, the suggested batch size is used for '
'the given model')
flags.DEFINE_enum('tf_variable_update', 'parameter_server',
['parameter_server', 'replicated',
'distributed_replicated', 'independent'],
'''The method for managing variables: parameter_server,
replicated, distributed_replicated, independent''')
flags.DEFINE_enum('tf_local_parameter_device', CPU, [CPU, GPU],
'''Device to use as parameter server: cpu or gpu. For
distributed training, it can affect where caching of
variables happens.''')
flags.DEFINE_enum('tf_device', GPU, [CPU, GPU],
'Device to use for computation: cpu or gpu')
flags.DEFINE_enum('tf_data_format', NCHW, [NCHW, NHWC], '''Data layout to
NUM_KEYS_PER_CORE = 2000000
PROPAGATION_WAIT_TIME = 30
# cassandra-stress command
WRITE_COMMAND = 'write'
COUNTER_WRITE_COMMAND = 'counter_write'
USER_COMMAND = 'user'
READ_COMMAND = 'read'
COUNTER_READ_COMMAND = 'counter_read'
MIXED_COMMAND = 'mixed'
PRELOAD_REQUIRED = (READ_COMMAND, COUNTER_READ_COMMAND, MIXED_COMMAND)
# cassandra-stress command [options]
flags.DEFINE_enum('cassandra_stress_command', WRITE_COMMAND, [
WRITE_COMMAND, COUNTER_WRITE_COMMAND, USER_COMMAND, READ_COMMAND,
COUNTER_READ_COMMAND, MIXED_COMMAND
], 'cassandra-stress command to use.')
flags.DEFINE_integer(
'cassandra_stress_preload_num_keys', None,
'Number of keys to preload into cassandra database. '
'Read/counter_read/mixed modes require preloading '
'cassandra database. If not set, the number of the keys '
'preloaded will be the same as --num_keys for '
'read/counter_read/mixed mode, the same as the number of '
'loaders for write/counter_write/user mode.')
# Options for cassandra-stress
flags.DEFINE_integer(
'num_keys', 0, 'Number of keys used in cassandra-stress tool across '
'all loader vms. If unset, this benchmark will use '
EBS-GP and PIOPS.
"""
import json
import logging
from perfkitbenchmarker import configs
from perfkitbenchmarker import flags
from perfkitbenchmarker.linux_packages import fio
LOGGING = 'logging'
DATABASE = 'database'
STREAMING = 'streaming'
flags.DEFINE_enum('workload_mode', LOGGING,
[LOGGING, DATABASE, STREAMING],
'Simulate a logging, database or streaming scenario.')
flags.DEFINE_list('iodepth_list', None, 'A list of iodepth parameter used by '
'fio command in simulated database and streaming scenarios '
'only.')
flags.DEFINE_integer('maxjobs', 0,
'The maximum allowed number of jobs to support.')
FLAGS = flags.FLAGS
BENCHMARK_NAME = 'block_storage_workload'
BENCHMARK_CONFIG = """
block_storage_workload:
description: >
'Can be absolute or relative to the executable.')
flags.DEFINE_list(
'queries_to_execute', [], 'List of all queries to execute, as strings. '
'E.g., "1","5","6","21"')
flags.DEFINE_string('snowflake_warehouse', None,
'A virtual warehouse, often referred to simply as a - '
'warehouse, is a cluster of compute in Snowflake. '
'https://docs.snowflake.com/en/user-guide/warehouses.html') # pylint: disable=line-too-long
flags.DEFINE_string(
'snowflake_database', None,
'The hosted snowflake database to use during the benchmark.')
flags.DEFINE_string(
'snowflake_schema', None,
'The schema of the hosted snowflake database to use during the benchmark.')
flags.DEFINE_enum(
'snowflake_client_interface', 'JDBC', ['JDBC'],
'The Runtime Interface used when interacting with Snowflake.')
FLAGS = flags.FLAGS
TYPE_2_PROVIDER = dict([('athena', 'aws'), ('redshift', 'aws'),
('spectrum', 'aws'), ('snowflake_aws', 'aws'),
('bigquery', 'gcp'),
('azuresqldatawarehouse', 'azure')])
TYPE_2_MODULE = dict([
('athena', 'perfkitbenchmarker.providers.aws.athena'),
('redshift', 'perfkitbenchmarker.providers.aws.redshift'),
('spectrum', 'perfkitbenchmarker.providers.aws.spectrum'),
('snowflake_aws', 'perfkitbenchmarker.providers.aws.snowflake'),
('bigquery', 'perfkitbenchmarker.providers.gcp.bigquery'),
('azuresqldatawarehouse', 'perfkitbenchmarker.providers.azure.'
NUM_SERVERS = 1
CLASSPATH = ('$FABAN_HOME/lib/fabanagents.jar:$FABAN_HOME/lib/fabancommon.jar:'
'$FABAN_HOME/lib/fabandriver.jar:$JAVA_HOME/lib/tools.jar:'
'$FABAN_HOME/search/build/lib/search.jar')
flags.DEFINE_string(
'cs_websearch_client_heap_size', '2g',
'Java heap size for Faban client in the usual java format.'
' Default: 2g.')
flags.DEFINE_string(
'cs_websearch_server_heap_size', '3g',
'Java heap size for Solr server in the usual java format.'
' Default: 3g.')
flags.DEFINE_enum(
'cs_websearch_query_distr', 'Random', ['Random', 'Ziphian'],
'Distribution of query terms. '
'Random and Ziphian distributions are available. '
'Default: Random.')
flags.DEFINE_integer('cs_websearch_num_clients',
1,
'Number of client machines.',
lower_bound=1)
flags.DEFINE_integer('cs_websearch_ramp_up',
90,
'Benchmark ramp up time in seconds.',
lower_bound=1)
flags.DEFINE_integer('cs_websearch_steady_state',
60,
'Benchmark steady state time in seconds.',
lower_bound=1)
flags.DEFINE_integer('cs_websearch_scale',
CENTOS8,
CLEAR,
CORE_OS,
COS,
DEBIAN9,
DEBIAN10,
JUJU,
RHEL7,
RHEL8,
UBUNTU_CONTAINER,
UBUNTU1604,
UBUNTU1604_CUDA9,
UBUNTU1710,
UBUNTU1804,
]
WINDOWS_OS_TYPES = [
WINDOWS2012_CORE,
WINDOWS2016_CORE,
WINDOWS2019_CORE,
WINDOWS2012_DESKTOP,
WINDOWS2016_DESKTOP,
WINDOWS2019_DESKTOP,
]
ALL = LINUX_OS_TYPES + WINDOWS_OS_TYPES
BASE_OS_TYPES = [CLEAR, CORE_OS, DEBIAN, RHEL, WINDOWS]
# May change from time to time.
DEFAULT = UBUNTU1604
flags.DEFINE_enum('os_type', DEFAULT, ALL, 'The VM\'s OS type.')
"""
WORD_COUNT_CONFIGURATION = dict([
(dpb_service.DATAPROC, (gcp_dpb_dataproc.SPARK_SAMPLE_LOCATION,
'org.apache.spark.examples.JavaWordCount',
BaseDpbService.SPARK_JOB_TYPE)),
(dpb_service.DATAFLOW, (None, 'org.example.WordCount',
BaseDpbService.DATAFLOW_JOB_TYPE)),
(dpb_service.EMR, (aws_dpb_emr.SPARK_SAMPLE_LOCATION,
'org.apache.spark.examples.JavaWordCount',
BaseDpbService.SPARK_JOB_TYPE))
])
flags.DEFINE_string('dpb_wordcount_input', None, 'Input for word count')
flags.DEFINE_enum('dpb_wordcount_fs', BaseDpbService.GCS_FS,
[BaseDpbService.GCS_FS, BaseDpbService.S3_FS],
'File System to use for the job output')
flags.DEFINE_string('dpb_wordcount_out_base', None,
'Base directory for word count output')
FLAGS = flags.FLAGS
def GetConfig(user_config):
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
def CheckPrerequisites(benchmark_config):
"""Verifies that the required resources are present.
Raises:
from perfkitbenchmarker.providers.aws import aws_network
from perfkitbenchmarker.providers.aws import util
from perfkitbenchmarker.linux_packages import sysbench05plus
MYSQL_SVC_OLTP_TABLES_COUNT = 'mysql_svc_oltp_tables_count'
MYSQL_SVC_OLTP_TABLE_SIZE = 'mysql_svc_oltp_table_size'
MYSQL_SVC_DB_INSTANCE_CORES = 'mysql_svc_db_instance_cores'
SYSBENCH_WARMUP_SECONDS = 'sysbench_warmup_seconds'
SYSBENCH_RUN_SECONDS = 'sysbench_run_seconds'
SYSBENCH_THREAD_COUNT = 'sysbench_thread_count'
SYSBENCH_LATENCY_PERCENTILE = 'sysbench_latency_percentile'
SYSBENCH_REPORT_INTERVAL = 'sysbench_report_interval'
FLAGS = flags.FLAGS
flags.DEFINE_enum(
'mysql_svc_db_instance_cores', '4', ['1', '4', '8', '16'],
'The number of cores to be provisioned for the DB instance.')
flags.DEFINE_integer(MYSQL_SVC_OLTP_TABLES_COUNT, 4,
'The number of tables used in sysbench oltp.lua tests')
flags.DEFINE_integer(
MYSQL_SVC_OLTP_TABLE_SIZE, 100000,
'The number of rows of each table used in the oltp tests')
flags.DEFINE_integer(
SYSBENCH_WARMUP_SECONDS, 120,
'The duration of the warmup run in which results are '
'discarded, in seconds.')
flags.DEFINE_integer(
SYSBENCH_RUN_SECONDS, 480,
'The duration of the actual run in which results are '
'collected, in seconds.')
flags.DEFINE_string(
'beam_runner_option', None,
'Overrides any pipeline options by the dpb service to'
' specify the runner.')
flags.DEFINE_boolean(
'beam_prebuilt', False, 'Set this to indicate that the'
' repo in beam_location does not'
' need to be rebuilt before'
' being used')
flags.DEFINE_integer('beam_it_timeout', 600, 'Integration Test Timeout.')
flags.DEFINE_string('git_binary', 'git', 'Path to git binary.')
flags.DEFINE_string(
'beam_version', None, 'Version of Beam to download. Use'
' tag from Github as value. If not'
' specified, will use HEAD.')
flags.DEFINE_enum('beam_sdk', None, [BEAM_JAVA_SDK, BEAM_PYTHON_SDK],
'Which BEAM SDK is used to build the benchmark pipeline.')
flags.DEFINE_string(
'beam_python_attr', 'IT',
'Test decorator that is used in Beam Python to filter a '
'specific category.')
FLAGS = flags.FLAGS
SUPPORTED_RUNNERS = [
dpb_service.DATAFLOW,
]
BEAM_REPO_LOCATION = 'https://github.com/apache/beam.git'
INSTALL_COMMAND_ARGS = [
"clean", "install", "-DskipTests", "-Dcheckstyle.skip=true"
]
boot_disk_size: 105
boot_disk_type: pd-ssd
min_cpu_platform: skylake
AWS:
machine_type: p3dn.24xlarge
zone: us-east-1a
boot_disk_size: 105
image: ami-0a4a0d42e3b855a2c
Azure:
machine_type: Standard_ND40s_v2
zone: eastus
boot_disk_size: 105
"""
flags.DEFINE_enum('mlperf_benchmark', 'resnet',
['resnet', 'transformer', 'mask', 'gnmt', 'ssd', 'minigo'],
'MLPerf benchmark test to run.')
flags.DEFINE_string(
'mlperf_gcs_resnet_checkpoint',
'gs://cloud-tpu-artifacts/resnet/resnet-nhwc-2018-02-07/model.ckpt-112603',
'A ResNet backbone trained on the ImageNet dataset.')
flags.DEFINE_string('mlperf_transformer_decode_dir', '',
'Transformer decode directory')
flags.DEFINE_string('wmt_data_dir',
'gs://pkb-sgpyc-us-west1/mlperf_v0.6_nv_transformer/',
'Directory where the wmt dataset is stored')
flags.DEFINE_string('coco_data_dir', 'gs://pkb-sgpyc-us-west1/coco2017/',
'Directory where the coco dataset is stored')
flags.DEFINE_string('gnmt_data_dir',
'gs://pkb-sgpyc-us-west1/mlperf_v0.6_nv_gnmt/',
'Directory where the nv v0.6 WMT dataset is stored')
},
DIGITALOCEAN: {
VIRTUAL_MACHINE: {
DEBIAN:
digitalocean_virtual_machine.DebianBasedDigitalOceanVirtualMachine,
RHEL:
digitalocean_virtual_machine.RhelBasedDigitalOceanVirtualMachine,
},
NETWORK: digitalocean_network.DigitalOceanNetwork,
FIREWALL: digitalocean_network.DigitalOceanFirewall
},
}
FLAGS = flags.FLAGS
flags.DEFINE_enum('cloud', GCP, [GCP, AZURE, AWS, DIGITALOCEAN],
'Name of the cloud to use.')
class BenchmarkSpec(object):
"""Contains the various data required to make a benchmark run."""
def __init__(self, benchmark_info):
if (FLAGS.benchmark_config_pair and benchmark_info['name']
in FLAGS.benchmark_config_pair.keys()):
# TODO(user): Unify naming between config_reader and
# perfkitbenchmarker.
self.config = config_reader.ConfigLoader(
FLAGS.benchmark_config_pair[benchmark_info['name']])
self.vms = []
self.vm_dict = {'default': []}
self.networks = {}
self.benchmark_name = benchmark_info['name']
flags.DEFINE_string(
'fio_jobfile', None,
'Job file that fio will use. If not given, use a job file '
'bundled with PKB. Cannot use with '
'--fio_generate_scenarios.')
flags.DEFINE_list(
'fio_generate_scenarios', [],
'Generate a job file with the given scenarios. Special '
'scenario \'all\' generates all scenarios. Available '
'scenarios are sequential_write, sequential_read, '
'random_write, and random_read. Cannot use with '
'--fio_jobfile.')
flags.DEFINE_enum(
'fio_target_mode', AGAINST_FILE_WITHOUT_FILL_MODE, [
AGAINST_DEVICE_WITH_FILL_MODE, AGAINST_DEVICE_WITHOUT_FILL_MODE,
AGAINST_FILE_WITH_FILL_MODE, AGAINST_FILE_WITHOUT_FILL_MODE
], 'Whether to run against a raw device or a file, and whether '
'to prefill.')
flags.DEFINE_string(
'fio_fill_size', '100%', 'The amount of device to fill in prepare stage. '
'The valid value can either be an integer, which '
'represents the number of bytes to fill or a '
'percentage, which represents the percentage '
'of the device. A filesystem will be unmounted before '
'filling and remounted afterwards. Only valid when '
'--fio_target_mode is against_device_with_fill or '
'against_file_with_fill.')
flag_util.DEFINE_integerlist('fio_io_depths',
flag_util.IntegerList([1]),
'IO queue depths to run on. Can specify a single '
'number, like --fio_io_depths=1, a range, like '
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import INSTALL_DIR
FLAGS = flags.FLAGS
GIT_REPO = 'https://github.com/aerospike/aerospike-server.git'
GIT_TAG = '3.7.5'
AEROSPIKE_DIR = '%s/aerospike-server' % INSTALL_DIR
AEROSPIKE_CONF_PATH = '%s/as/etc/aerospike_dev.conf' % AEROSPIKE_DIR
AEROSPIKE_DEFAULT_TELNET_PORT = 3003
MEMORY = 'memory'
DISK = 'disk'
flags.DEFINE_enum(
'aerospike_storage_type', MEMORY, [MEMORY, DISK],
'The type of storage to use for Aerospike data. The type of '
'disk is controlled by the "data_disk_type" flag.')
flags.DEFINE_integer('aerospike_replication_factor', 1,
'Replication factor for aerospike server.')
flags.DEFINE_integer('aerospike_transaction_threads_per_queue', 4,
'Number of threads per transaction queue.')
def _Install(vm):
"""Installs the Aerospike server on the VM."""
vm.Install('build_tools')
vm.Install('lua5_1')
vm.Install('openssl')
vm.RemoteCommand('git clone {0} {1}'.format(GIT_REPO, AEROSPIKE_DIR))
vm.RemoteCommand(
'cd {0} && git checkout {1} && git submodule update --init '
flags.DEFINE_boolean(
'official', False,
'A boolean indicating whether results are official or not. The '
'default is False. Official test results are treated and queried '
'differently from non-official test results.')
flags.DEFINE_boolean(
'hostname_metadata', False,
'A boolean indicating whether to publish VM hostnames as part of sample '
'metadata.')
flags.DEFINE_string(
'json_path', None, 'A path to write newline-delimited JSON results '
'Default: write to a run-specific temporary directory')
flags.DEFINE_enum(
'json_write_mode', 'wb', ['wb', 'ab'],
'Open mode for file specified by --json_path. Default: overwrite file')
flags.DEFINE_boolean('collapse_labels', True,
'Collapse entries in labels in JSON output.')
flags.DEFINE_string('csv_path', None, 'A path to write CSV-format results')
flags.DEFINE_string(
'bigquery_table', None,
'The BigQuery table to publish results to. This should be of the form '
'"[project_id:]dataset_name.table_name".')
flags.DEFINE_string('bq_path', 'bq', 'Path to the "bq" executable.')
flags.DEFINE_string('bq_project', None,
'Project to use for authenticating with BigQuery.')
flags.DEFINE_string('service_account', None,
'Service account to use to authenticate with BQ.')
flags.DEFINE_string('service_account_private_key', None,
GIT_TAG = '1.2.15'
LIBEVENT_TAR = 'libevent-2.0.21-stable.tar.gz'
LIBEVENT_URL = 'https://github.com/downloads/libevent/libevent/' + LIBEVENT_TAR
LIBEVENT_DIR = '%s/libevent-2.0.21-stable' % INSTALL_DIR
MEMTIER_DIR = '%s/memtier_benchmark' % INSTALL_DIR
APT_PACKAGES = ('autoconf automake libpcre3-dev '
'libevent-dev pkg-config zlib1g-dev')
YUM_PACKAGES = 'zlib-devel pcre-devel libmemcached-devel'
MEMTIER_RESULTS = 'memtier_results'
FLAGS = flags.FLAGS
flags.DEFINE_enum('memtier_protocol', 'memcache_binary',
['memcache_binary', 'redis', 'memcache_text'],
'Protocol to use. Supported protocols are redis, '
'memcache_text, and memcache_binary. '
'Defaults to memcache_binary.')
flags.DEFINE_integer('memtier_run_count', 1,
'Number of full-test iterations to perform. '
'Defaults to 1.')
flags.DEFINE_integer('memtier_run_duration', None,
'Duration for each client count in seconds. '
'By default, test length is set '
'by memtier_requests, the number of requests sent by each '
'client. By specifying run_duration, key space remains '
'the same (from 1 to memtier_requests), but test stops '
'once run_duration is passed. '
'Total test duration = run_duration * runs * '
'len(memtier_clients).')
flags.DEFINE_integer('memtier_requests', 10000,
lmbench:
description: Runs Lmbench Microbenchmark.
vm_groups:
default:
vm_spec: *default_dual_core
vm_count: null
"""
_LMBENCH_HARDWARE_DEFAULT = 'NO'
flags.DEFINE_integer(
'lmbench_mem_size', None,
'The range of memory on which several benchmarks operate. If not provided, '
'the memory size should be 8MB as default')
flags.DEFINE_enum(
'lmbench_hardware', _LMBENCH_HARDWARE_DEFAULT, ['YES', 'NO'],
'The decision to run BENCHMARK_HARDWARE tests: YES or NO. The default is NO'
)
def GetConfig(user_config):
return configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
def _PrepareLmbench(vm):
"""Builds Lmbench on a single vm."""
logging.info('Installing Lmbench on %s', vm)
vm.Install('lmbench')
def _ConfigureRun(vm):
"""Configure Lmbench tests."""
