'network to use instead of creating a new one.')
flags.DEFINE_string(
'gce_subnet_region', None, 'Region to create subnet in '
'instead of automatically creating one in every region.')
flags.DEFINE_string(
'gce_subnet_addr', '10.128.0.0/20', 'Address range to the '
'subnet, given in CDR notation. Not used unless '
'--gce_subnet_region is given.')
flags.DEFINE_string(
'gce_remote_access_firewall_rule', None, 'The name of an '
'already created firewall rule which allows remote access '
'instead of creating a new one.')
flags.DEFINE_multi_string(
'gcp_instance_metadata_from_file', [],
'A colon separated key-value pair that will be added to the '
'"--metadata-from-file" flag of the gcloud cli (with the colon replaced by '
'the equal sign). Multiple key-value pairs may be specified by separating '
'each pair by commas. This option can be repeated multiple times. For '
'information about GCP instance metadata, see: --metadata-from-file from '
'`gcloud help compute instances create`.')
flags.DEFINE_multi_string(
'gcp_instance_metadata', [],
'A colon separated key-value pair that will be added to the '
'"--metadata" flag of the gcloud cli (with the colon replaced by the equal '
'sign). Multiple key-value pairs may be specified by separating each pair '
'by commas. This option can be repeated multiple times. For information '
'about GCP instance metadata, see: --metadata from '
'`gcloud help compute instances create`.')
flags.DEFINE_integer('gce_boot_disk_size', None,
'The boot disk size in GB for GCP VMs.')
flags.DEFINE_enum('gce_boot_disk_type', None, ['pd-standard', 'pd-ssd'],
'The boot disk type for GCP VMs.')
'subnet to use instead of creating a new one.')
flags.DEFINE_string(
'gce_subnet_region', None, 'Region to create subnet in '
'instead of automatically creating one in every region.')
flags.DEFINE_string(
'gce_subnet_addr', '10.128.0.0/20', 'Address range to the '
'subnet, given in CDR notation. Not used unless '
'--gce_subnet_region is given.')
flags.DEFINE_string(
'gce_remote_access_firewall_rule', None, 'The name of an '
'already created firewall rule which allows remote access '
'instead of creating a new one.')
flags.DEFINE_multi_string(
'gcp_instance_metadata_from_file', [],
'A colon separated key-value pair that will be added to the '
'"--metadata-from-file" flag of the gcloud cli (with the colon replaced by '
'the equal sign). Multiple key-value pairs may be specified by separating '
'each pair by commas. This option can be repeated multiple times. For '
'information about GCP instance metadata, see: --metadata-from-file from '
'`gcloud help compute instances create`.')
flags.DEFINE_multi_string(
'gcp_instance_metadata', [],
'A colon separated key-value pair that will be added to the '
'"--metadata" flag of the gcloud cli (with the colon replaced by the equal '
'sign). Multiple key-value pairs may be specified by separating each pair '
'by commas. This option can be repeated multiple times. For information '
'about GCP instance metadata, see: --metadata from '
'`gcloud help compute instances create`.')
flags.DEFINE_integer('gce_boot_disk_size', None,
'The boot disk size in GB for GCP VMs.')
flags.DEFINE_enum('gce_boot_disk_type', None, ['pd-standard', 'pd-ssd'],
'The boot disk type for GCP VMs.')
'ceph_monitors', [], 'IP addresses and ports of Ceph Monitors. '
'Must be provided when Ceph scratch disk is required. '
'Example: "127.0.0.1:6789,192.168.1.1:6789"')
flags.DEFINE_string(
'username', 'root',
'User name that Perfkit will attempt to use in order to '
'SSH into Docker instance.')
flags.DEFINE_boolean(
'docker_in_privileged_mode', True,
'If set to True, will attempt to create Docker containers '
'in a privileged mode. Note that some benchmarks execute '
'commands which are only allowed in privileged mode.')
flags.DEFINE_boolean(
'kubernetes_anti_affinity', True,
'If set to True, PKB pods will not be scheduled on the '
'same nodes as other PKB pods.')
flags.DEFINE_multi_string(
'k8s_volume_parameters', None,
'A colon separated key-value pair that will be '
'added to Kubernetes storage class parameters.')
_K8S_PROVISIONERS = [
'kubernetes.io/azure-disk', 'kubernetes.io/gce-pd',
'kubernetes.io/aws-ebs', 'kubernetes.io/glusterfs'
]
flags.DEFINE_enum(
'k8s_volume_provisioner', None, _K8S_PROVISIONERS,
'The name of the provisioner to use for K8s storage '
'classes.')
'cloud_storage_bucket',
None,
'GCS bucket to upload records to. Bucket must exist.')
flags.DEFINE_string(
'es_uri', None,
'The Elasticsearch address and port. e.g. http://localhost:9200')
flags.DEFINE_string(
'es_index', 'perfkit', 'Elasticsearch index name to store documents')
flags.DEFINE_string('es_type', 'result', 'Elasticsearch document type')
flags.DEFINE_multi_string(
'metadata',
[],
'A colon separated key-value pair that will be added to the labels field '
'of all samples as metadata. Multiple key-value pairs may be specified '
'by separating each pair by commas.')
flags.DEFINE_string(
'influx_uri', None,
'The Influx DB address and port. Expects the format hostname:port'
'If port is not passed in it assumes port 80. e.g. localhost:8086')
flags.DEFINE_string(
'influx_db_name', 'perfkit',
'Name of Influx DB database that you wish to publish to or create')
DEFAULT_JSON_OUTPUT_NAME = 'perfkitbenchmarker_results.json'
DEFAULT_CREDENTIALS_JSON = 'credentials.json'
GCS_OBJECT_NAME_LENGTH = 20
FLAGS = flags.FLAGS
CONFIG_CONSTANTS = 'default_config_constants.yaml'
FLAGS_KEY = 'flags'
IMPORT_REGEX = re.compile('^#import (.*)')
flags.DEFINE_string(
'benchmark_config_file', None,
'The file path to the user config file which will '
'override benchmark defaults. This should either be '
'a path relative to the current working directory, '
'an absolute path, or just the name of a file in the '
'configs/ directory.')
flags.DEFINE_multi_string(
'config_override', None,
'This flag can be used to override any config value. It is applied after '
'the user config (specified via --benchmark_config_file_path), so it has '
'a higher priority than that config. The value of the flag should be '
'fully.qualified.key=value (e.g. --config_override=cluster_boot.vm_groups.'
'default.vm_count=4).')
class _ConcatenatedFiles(object):
"""Class that presents several files as a single object.
The class exposes a single method (read) which is all that yaml
needs to interact with a stream.
Attributes:
files: A list of opened file objects.
current_file_index: The index of the current file that is being read from.
"""
flags.DEFINE_list('ssh_options', [], 'Additional options to pass to ssh.')
flags.DEFINE_boolean('use_ipv6', False, 'Whether to use ipv6 for ssh/scp.')
flags.DEFINE_list(
'benchmarks', [benchmark_sets.STANDARD_SET],
'Benchmarks and/or benchmark sets that should be run. The '
'default is the standard set. For more information about '
'benchmarks and benchmark sets, see the README and '
'benchmark_sets.py.')
flags.DEFINE_string('archive_bucket', None,
'Archive results to the given S3/GCS bucket.')
flags.DEFINE_string(
'project', None, 'GCP project ID under which '
'to create the virtual machines')
flags.DEFINE_multi_string(
'zone', [],
'Similar to the --zones flag, but allows the flag to be specified '
'multiple times on the commandline. For example, --zone=a --zone=b is '
'equivalent to --zones=a,b. Furthermore, any values specified by --zone '
'will be appended to those specfied by --zones.')
flags.DEFINE_list(
'zones', [], 'A list of zones within which to run PerfKitBenchmarker. '
'This is specific to the cloud provider you are running on. '
'If multiple zones are given, PerfKitBenchmarker will create 1 VM in '
'zone, until enough VMs are created as specified in each '
'benchmark. The order in which this flag is applied to VMs is '
'undefined.')
flags.DEFINE_list(
'extra_zones', [],
'Zones that will be appended to the "zones" list. This is functionally '
'the same, but allows flag matrices to have two zone axes.')
# TODO(user): note that this is currently very GCE specific. Need to create a
# module which can translate from some generic types to provider specific
'edw_service_cluster_password', None,
'If set, the password authorized on cluster (only '
'applicable when using snapshots).')
flags.DEFINE_string('snowflake_snowsql_config_override_file', None,
'The SnowSQL configuration to use.'
'https://docs.snowflake.net/manuals/user-guide/snowsql-config.html#snowsql-config-file') # pylint: disable=line-too-long
flags.DEFINE_string('snowflake_connection', None,
'Named Snowflake connection defined in SnowSQL config file.'
'https://docs.snowflake.net/manuals/user-guide/snowsql-start.html#using-named-connections') # pylint: disable=line-too-long
flags.DEFINE_integer('edw_suite_iterations', 1,
'Number of suite iterations to perform.')
# TODO(user): Revisit flags for accepting query lists.
flags.DEFINE_string('edw_simultaneous_queries', None,
'CSV list of simultaneous queries to benchmark.')
flags.DEFINE_multi_string(
'concurrency_streams', [], 'List of all query streams to execute. Each '
'stream should be passed in separately and the queries should be comma '
'separated, e.g. --concurrency_streams=1,2,3 --concurrency_streams=3,2,1')
flags.DEFINE_string('snowflake_warehouse', None,
'A virtual warehouse, often referred to simply as a - '
'warehouse, is a cluster of compute in Snowflake. '
'https://docs.snowflake.com/en/user-guide/warehouses.html') # pylint: disable=line-too-long
flags.DEFINE_string(
'snowflake_database', None,
'The hosted snowflake database to use during the benchmark.')
flags.DEFINE_string(
'snowflake_schema', None,
'The schema of the hosted snowflake database to use during the benchmark.')
flags.DEFINE_enum(
'snowflake_client_interface', 'JDBC', ['JDBC'],
'The Runtime Interface used when interacting with Snowflake.')
`--data_search_paths` flag.
"""
import abc
import logging
import os
import shutil
import perfkitbenchmarker
from perfkitbenchmarker import flags
from perfkitbenchmarker import temp_dir
import pkg_resources
FLAGS = flags.FLAGS
flags.DEFINE_multi_string(
'data_search_paths', ['.'], 'Additional paths to search for data files. '
'These paths will be searched prior to using files '
'bundled with PerfKitBenchmarker.')
_RESOURCES = 'resources'
class ResourceNotFound(ValueError):
"""Error raised when a resource could not be found on the search path."""
pass
class ResourceLoader(object):
"""An interface for loading named resources."""
__metaclass__ = abc.ABCMeta
