def Prepare(benchmark_spec: bm_spec.BenchmarkSpec) -> None:
"""Prepare the virtual machines to run cloud bigtable.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
vms = benchmark_spec.vms
if _TABLE_OBJECT_SHARING.value:
ycsb.SetYcsbTarUrl(YCSB_BIGTABLE_TABLE_SHARING_TAR_URL)
instance: _Bigtable = benchmark_spec.non_relational_db
args = [((vm, instance), {}) for vm in vms]
vm_util.RunThreaded(_Install, args)
table_name = _GetTableName()
# If the table already exists, it will be an no-op.
hbase_ycsb.CreateYCSBTable(vms[0], table_name=table_name, use_snappy=False,
limit_filesize=False)
# Add hbase conf dir to the classpath.
ycsb_memory = min(vms[0].total_memory_kb // 1024, 4096)
jvm_args = pipes.quote(f' -Xmx{ycsb_memory}m')
executor_flags = {
'cp': hbase.HBASE_CONF_DIR,
'jvm-args': jvm_args,
'table': table_name}
benchmark_spec.executor = ycsb.YCSBExecutor(FLAGS.hbase_binding,
**executor_flags)
def Prepare(benchmark_spec):
benchmark_spec.always_call_cleanup = True
vms = benchmark_spec.vms
# Install required packages and copy credential files.
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud spanner benchmarks.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
benchmark_spec.spanner_instance = gcp_spanner.GcpSpannerInstance(
name=BENCHMARK_INSTANCE_PREFIX + FLAGS.run_uri,
description=BENCHMARK_DESCRIPTION,
database=BENCHMARK_DATABASE,
ddl=BENCHMARK_SCHEMA)
if benchmark_spec.spanner_instance._Exists(instance_only=True):
logging.warning('Cloud Spanner instance %s exists, delete it first.' %
FLAGS.cloud_spanner_ycsb_instance)
benchmark_spec.spanner_instance.Delete()
benchmark_spec.spanner_instance.Create()
if not benchmark_spec.spanner_instance._Exists():
logging.warning(
'Failed to create Cloud Spanner instance and database.')
benchmark_spec.spanner_instance.Delete()
if FLAGS.cloud_spanner_ycsb_client_type != 'java':
ycsb.SetYcsbTarUrl(
CLIENT_TAR_URL[FLAGS.cloud_spanner_ycsb_client_type])
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('cloudspanner')
def Run(benchmark_spec):
"""Spawn YCSB and gather the results.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample objects.
"""
by_role = _GetVMsByRole(benchmark_spec.vm_groups)
loaders = by_role['clients']
logging.info('Loaders: %s', loaders)
executor = ycsb.YCSBExecutor('hbase-10')
metadata = {'ycsb_client_vms': len(loaders),
'hbase_cluster_size': len(by_role['hbase_vms']),
'hbase_zookeeper_nodes': FLAGS.hbase_zookeeper_nodes}
# By default YCSB uses a BufferedMutator for Puts / Deletes.
# This leads to incorrect update latencies, since since the call returns
# before the request is acked by the server.
# Disable this behavior during the benchmark run.
run_kwargs = {'columnfamily': COLUMN_FAMILY,
'clientbuffering': 'false'}
load_kwargs = run_kwargs.copy()
load_kwargs['clientbuffering'] = 'true'
samples = list(executor.LoadAndRun(loaders,
load_kwargs=load_kwargs,
run_kwargs=run_kwargs))
for sample in samples:
sample.metadata.update(metadata)
return samples
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run YCSB against memcached.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
clients = benchmark_spec.vm_groups['clients']
assert clients, benchmark_spec.vm_groups
hosts = []
if FLAGS.memcached_scenario == 'managed':
# We need to delete the managed memcached backend when we're done
benchmark_spec.always_call_cleanup = True
if FLAGS.memcached_managed == providers.GCP:
raise NotImplementedError(
"GCP managed memcached backend not implemented "
"yet")
elif FLAGS.memcached_managed == providers.AWS:
cluster_id = 'pkb%s' % FLAGS.run_uri
service = providers.aws.elasticache.ElastiCacheMemcacheService(
aws_network.AwsNetwork.GetNetwork(clients[0]), cluster_id,
FLAGS.memcached_elasticache_region,
FLAGS.memcached_elasticache_node_type,
FLAGS.memcached_elasticache_num_servers)
service.Create()
hosts = service.GetHosts()
benchmark_spec.service = service
benchmark_spec.metadata = service.GetMetadata()
else:
# custom scenario
# Install memcached on all the servers
servers = benchmark_spec.vm_groups['servers']
assert servers, 'No memcached servers: {0}'.format(
benchmark_spec.vm_groups)
memcached_install_fns = \
[functools.partial(memcached_server.ConfigureAndStart, vm)
for vm in servers]
vm_util.RunThreaded(lambda f: f(), memcached_install_fns)
hosts = [
'%s:%s' % (vm.internal_ip, memcached_server.MEMCACHED_PORT)
for vm in servers
]
benchmark_spec.metadata = {
'ycsb_client_vms': FLAGS.ycsb_client_vms,
'ycsb_server_vms': FLAGS.ycsb_server_vms,
'num_vms': len(servers),
'cache_size': FLAGS.memcached_size_mb
}
assert len(hosts) > 0
ycsb_install_fns = [
functools.partial(vm.Install, 'ycsb') for vm in clients
]
vm_util.RunThreaded(lambda f: f(), ycsb_install_fns)
benchmark_spec.executor = ycsb.YCSBExecutor(
'memcached', **{'memcached.hosts': ','.join(hosts)})
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run YCSB against Cassandra.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
vms = benchmark_spec.vms
by_role = _GetVMsByRole(benchmark_spec)
loaders = by_role['clients']
assert loaders, vms
# Cassandra cluster
cassandra_vms = by_role['cassandra_vms']
assert cassandra_vms, 'No Cassandra VMs: {0}'.format(by_role)
seed_vm = by_role['seed_vm']
assert seed_vm, 'No seed VM: {0}'.format(by_role)
cassandra_install_fns = [functools.partial(_InstallCassandra,
vm, seed_vms=[seed_vm])
for vm in cassandra_vms]
ycsb_install_fns = [functools.partial(vm.Install, 'ycsb')
for vm in loaders]
vm_util.RunThreaded(lambda f: f(), cassandra_install_fns + ycsb_install_fns)
cassandra.StartCluster(seed_vm, by_role['non_seed_cassandra_vms'])
_CreateYCSBTable(seed_vm)
benchmark_spec.executor = ycsb.YCSBExecutor(
'cassandra-10',
hosts=','.join(vm.internal_ip for vm in cassandra_vms))
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run YCSB against memcached.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
loaders = benchmark_spec.vm_groups['clients']
assert loaders, benchmark_spec.vm_groups
# Memcached cluster
memcached_vms = benchmark_spec.vm_groups['servers']
assert memcached_vms, 'No memcached VMs: {0}'.format(
benchmark_spec.vm_groups)
memcached_install_fns = [
functools.partial(memcached_server.ConfigureAndStart, vm)
for vm in memcached_vms
]
ycsb_install_fns = [
functools.partial(vm.Install, 'ycsb') for vm in loaders
]
vm_util.RunThreaded(lambda f: f(),
memcached_install_fns + ycsb_install_fns)
benchmark_spec.executor = ycsb.YCSBExecutor(
'memcached', **{
'memcached.hosts':
','.join([vm.internal_ip for vm in memcached_vms])
})
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud datastore.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
# Check that the database is empty before running
if FLAGS.google_datastore_deletion_keyfile:
dataset_id = FLAGS.google_datastore_datasetId
credentials = GetDatastoreDeleteCredentials()
client = datastore.Client(project=dataset_id, credentials=credentials)
for kind in _YCSB_COLLECTIONS:
# TODO(user): Allow a small number of leftover entities until we
# figure out why these are not getting deleted.
if len(list(client.query(kind=kind).fetch(limit=200))) > 100:
raise errors.Benchmarks.PrepareException(
'Database is non-empty. Stopping test.')
else:
logging.warning('Test could be executed on a non-empty database.')
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
# Restore YCSB_TAR_URL
benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run YCSB against Aerospike.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
loaders = benchmark_spec.vm_groups['clients']
assert loaders, benchmark_spec.vm_groups
# Aerospike cluster
aerospike_vms = benchmark_spec.vm_groups['workers']
assert aerospike_vms, 'No aerospike VMs: {0}'.format(
benchmark_spec.vm_groups)
seed_ips = [vm.internal_ip for vm in aerospike_vms]
aerospike_install_fns = [
functools.partial(aerospike_server.ConfigureAndStart,
vm,
seed_node_ips=seed_ips) for vm in aerospike_vms
]
ycsb_install_fns = [
functools.partial(vm.Install, 'ycsb') for vm in loaders
]
vm_util.RunThreaded(lambda f: f(),
aerospike_install_fns + ycsb_install_fns)
benchmark_spec.executor = ycsb.YCSBExecutor(
'aerospike', **{
'as.host': aerospike_vms[0].internal_ip,
'as.namespace': 'test'
})
def Prepare(benchmark_spec):
"""Prepare the cloud redis instance to YCSB tasks.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
ycsb_vms = benchmark_spec.vm_groups['clients']
vm_util.RunThreaded(_Install, ycsb_vms)
cloud_redis_class = (managed_memory_store.GetManagedMemoryStoreClass(
FLAGS.cloud, managed_memory_store.REDIS))
benchmark_spec.cloud_redis_instance = (cloud_redis_class(benchmark_spec))
benchmark_spec.cloud_redis_instance.Create()
instance_details = benchmark_spec.cloud_redis_instance.GetInstanceDetails()
redis_args = {
'shardkeyspace': True,
'redis.host': instance_details['host'],
'redis.port': instance_details['port']
}
if 'password' in instance_details:
redis_args['redis.password'] = instance_details['password']
benchmark_spec.executor = ycsb.YCSBExecutor('redis', **redis_args)
def Run(benchmark_spec):
"""Spawn YCSB and gather the results.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample instances.
"""
loaders = benchmark_spec.vm_groups['clients']
aerospike_vms = benchmark_spec.vm_groups['workers']
executor = ycsb.YCSBExecutor(
'aerospike', **{
'as.host': aerospike_vms[0].internal_ip,
'as.namespace': 'test'
})
metadata = {
'ycsb_client_vms': FLAGS.ycsb_client_vms,
'num_vms': len(aerospike_vms)
}
samples = list(executor.LoadAndRun(loaders))
for sample in samples:
sample.metadata.update(metadata)
return samples
def Prepare(benchmark_spec):
"""Install Redis on one VM and memtier_benchmark on another.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
groups = benchmark_spec.vm_groups
redis_vm = groups['workers'][0]
ycsb_vms = groups['clients']
prepare_fns = ([functools.partial(PrepareServer, redis_vm)] +
[functools.partial(vm.Install, 'ycsb') for vm in ycsb_vms])
vm_util.RunThreaded(lambda f: f(), prepare_fns)
num_ycsb = FLAGS.redis_ycsb_processes
num_server = FLAGS.redis_total_num_processes
# Each redis process use different ports, number of ycsb processes should
# be at least as large as number of server processes, use round-robin
# to assign target server process to each ycsb process
server_metadata = [{
'redis.port':
redis_server.REDIS_FIRST_PORT + i % num_server
} for i in range(num_ycsb)]
benchmark_spec.executor = ycsb.YCSBExecutor(
'redis', **{
'shardkeyspace': True,
'redis.host': redis_vm.internal_ip,
'perclientparam': server_metadata
})
vm_util.SetupSimulatedMaintenance(redis_vm)
def Run(benchmark_spec):
"""Spawn YCSB and gather the results.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample instances.
"""
vms = benchmark_spec.vms
table_name = _GetTableName()
# Add hbase conf dir to the classpath.
ycsb_memory = ycsb_memory = min(vms[0].total_memory_kb // 1024, 4096)
jvm_args = pipes.quote(' -Xmx{0}m'.format(ycsb_memory))
executor_flags = {
'cp': hbase.HBASE_CONF_DIR,
'jvm-args': jvm_args,
'table': table_name
}
executor = ycsb.YCSBExecutor('hbase-10', **executor_flags)
cluster_name = (FLAGS.google_bigtable_cluster_name
or 'pkb-bigtable-{0}'.format(FLAGS.run_uri))
cluster_info = _GetClusterDescription(
FLAGS.project or _GetDefaultProject(), FLAGS.google_bigtable_zone_name,
cluster_name)
metadata = {
'ycsb_client_vms': len(vms),
'bigtable_nodes': cluster_info.get('serveNodes')
}
# By default YCSB uses a BufferedMutator for Puts / Deletes.
# This leads to incorrect update latencies, since since the call returns
# before the request is acked by the server.
# Disable this behavior during the benchmark run.
run_kwargs = {'columnfamily': COLUMN_FAMILY, 'clientbuffering': 'false'}
load_kwargs = run_kwargs.copy()
# During the load stage, use a buffered mutator with a single thread.
# The BufferedMutator will handle multiplexing RPCs.
load_kwargs['clientbuffering'] = 'true'
if not FLAGS['ycsb_preload_threads'].present:
load_kwargs['threads'] = 1
samples = list(
executor.LoadAndRun(vms,
load_kwargs=load_kwargs,
run_kwargs=run_kwargs))
for sample in samples:
sample.metadata.update(metadata)
return samples
def Prepare(benchmark_spec):
benchmark_spec.always_call_cleanup = True
ycsb.SetYcsbTarUrl(YCSB_BINDING_TAR_URL)
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
# Restore YCSB_TAR_URL
ycsb.SetYcsbTarUrl(None)
benchmark_spec.executor = ycsb.YCSBExecutor('googlefirestore')
def Prepare(benchmark_spec):
benchmark_spec.always_call_cleanup = True
vms = benchmark_spec.vms
# Install required packages and copy credential files.
vm_util.RunThreaded(_Install, vms)
# Create benchmark table.
ExecuteSql(vms[0], DROP_TABLE_SQL)
ExecuteSql(vms[0], CREATE_TABLE_SQL)
benchmark_spec.executor = ycsb.YCSBExecutor('jdbc')
def Prepare(benchmark_spec):
"""Install YCSB on the target vm.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
vms = benchmark_spec.vms
# Install required packages.
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')
def testRunCalledWithCorrectTarget(self):
# Arrange
FLAGS.ycsb_workload_files = ['workloadc']
test_executor = ycsb.YCSBExecutor('test_database')
test_vm = mock.Mock()
test_vm.RobustRemoteCommand.return_value = ['', '']
self.enter_context(mock.patch.object(ycsb, 'ParseResults'))
# Act
test_executor.Run([test_vm], run_kwargs={'target': 1000})
# Assert
self.assertIn('-target 1000',
test_vm.RobustRemoteCommand.call_args[0][0])
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud spanner benchmarks.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True if not FLAGS.cloud_spanner_instance_name else False
instance_name = FLAGS.cloud_spanner_instance_name or BENCHMARK_INSTANCE_PREFIX + FLAGS.run_uri
instance_description = FLAGS.cloud_spanner_instance_description or BENCHMARK_DESCRIPTION
benchmark_spec.spanner_instance = gcp_spanner.GcpSpannerInstance(
name=instance_name,
description=instance_description,
database=BENCHMARK_DATABASE,
ddl=BENCHMARK_SCHEMA)
# If instance name is provided, we might re-use an existing instance
if FLAGS.cloud_spanner_instance_name:
benchmark_spec.always_call_cleanup = False
if benchmark_spec.spanner_instance._Exists(instance_only=True):
logging.info("Re-using existing instance %s", instance_name)
else:
logging.info("Creating new instance %s", instance_name)
benchmark_spec.spanner_instance.Create()
# If instance name is not provided, we delete the existing instance before creating
elif benchmark_spec.spanner_instance._Exists(instance_only=True):
logging.warning('Cloud Spanner instance %s exists, delete it first.' %
instance_name)
benchmark_spec.spanner_instance.Delete()
logging.info("Creating new instance %s", instance_name)
benchmark_spec.spanner_instance.Create()
if not benchmark_spec.spanner_instance._Exists():
logging.warning(
'Failed to create Cloud Spanner instance and database.')
benchmark_spec.spanner_instance.Delete()
if FLAGS.cloud_spanner_ycsb_client_type != 'java':
ycsb.SetYcsbTarUrl(
CLIENT_TAR_URL[FLAGS.cloud_spanner_ycsb_client_type])
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('cloudspanner')
def Prepare(benchmark_spec):
"""Install MongoDB on one VM and YCSB on another.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
server_partials = [functools.partial(_PrepareServer, mongo_vm)
for mongo_vm in benchmark_spec.vm_groups['workers']]
client_partials = [functools.partial(_PrepareClient, client)
for client in benchmark_spec.vm_groups['clients']]
vm_util.RunThreaded((lambda f: f()), server_partials + client_partials)
benchmark_spec.executor = ycsb.YCSBExecutor('mongodb', cp=ycsb.YCSB_DIR)
def Prepare(benchmark_spec):
benchmark_spec.always_call_cleanup = True
default_ycsb_tar_url = ycsb.YCSB_TAR_URL
vms = benchmark_spec.vms
# TODO: figure out a less hacky way to override.
# Override so that we only need to download the required binding.
ycsb.YCSB_TAR_URL = YCSB_BINDING_TAR_URL
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
# Restore YCSB_TAR_URL
ycsb.YCSB_TAR_URL = default_ycsb_tar_url
benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud datastore.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
# Restore YCSB_TAR_URL
benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Prepare(benchmark_spec):
"""Install YCSB on the target vm.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
benchmark_spec.firestore_instance = gcp_firestore.GcpFirestoreInstance(
'pkb-{0}'.format(FLAGS.run_uri), FLAGS.zones[0])
vms = benchmark_spec.vms
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('googlefirestore')
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run hadoop.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
by_role = _GetVMsByRole(benchmark_spec.vm_groups)
loaders = by_role['clients']
assert loaders, 'No loader VMs: {0}'.format(by_role)
# HBase cluster
hbase_vms = by_role['hbase_vms']
assert hbase_vms, 'No HBase VMs: {0}'.format(by_role)
master = by_role['master']
zk_quorum = by_role['zk_quorum']
assert zk_quorum, 'No zookeeper quorum: {0}'.format(by_role)
workers = by_role['workers']
assert workers, 'No workers: {0}'.format(by_role)
hbase_install_fns = [
functools.partial(vm.Install, 'hbase') for vm in hbase_vms
]
ycsb_install_fns = [
functools.partial(vm.Install, 'ycsb') for vm in loaders
]
vm_util.RunThreaded(lambda f: f(), hbase_install_fns + ycsb_install_fns)
hadoop.ConfigureAndStart(master, workers, start_yarn=False)
hbase.ConfigureAndStart(master, workers, zk_quorum)
CreateYCSBTable(master, use_snappy=FLAGS.hbase_use_snappy)
# Populate hbase-site.xml on the loaders.
master.PullFile(vm_util.GetTempDir(),
posixpath.join(hbase.HBASE_CONF_DIR, HBASE_SITE))
def PushHBaseSite(vm):
conf_dir = posixpath.join(ycsb.YCSB_DIR,
FLAGS.hbase_binding + '-binding', 'conf')
vm.RemoteCommand('mkdir -p {}'.format(conf_dir))
vm.PushFile(os.path.join(vm_util.GetTempDir(), HBASE_SITE),
posixpath.join(conf_dir, HBASE_SITE))
vm_util.RunThreaded(PushHBaseSite, loaders)
benchmark_spec.executor = ycsb.YCSBExecutor(FLAGS.hbase_binding)
def Prepare(benchmark_spec):
"""Install YCSB on the target vm.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
benchmark_spec.dynamodb_instance = aws_dynamodb.AwsDynamoDBInstance(
table_name='pkb-{0}'.format(FLAGS.run_uri))
benchmark_spec.dynamodb_instance._Create()
vms = benchmark_spec.vms
# Install required packages.
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud bigtable.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
vms = benchmark_spec.vms
if FLAGS.google_bigtable_enable_table_object_sharing:
ycsb.SetYcsbTarUrl(YCSB_BIGTABLE_TABLE_SHARING_TAR_URL)
# TODO: in the future, it might be nice to change this so that
# a gcp_bigtable.GcpBigtableInstance can be created with an
# flag that says don't create/delete the instance. That would
# reduce the code paths here.
if FLAGS.google_bigtable_instance_name is None:
instance_name = 'pkb-bigtable-{0}'.format(FLAGS.run_uri)
project = FLAGS.project or _GetDefaultProject()
logging.info('Creating bigtable instance %s', instance_name)
zone = FLAGS.google_bigtable_zone
benchmark_spec.bigtable_instance = gcp_bigtable.GcpBigtableInstance(
instance_name, project, zone)
benchmark_spec.bigtable_instance.Create()
instance = _GetInstanceDescription(project, instance_name)
logging.info('Instance %s created successfully', instance)
vm_util.RunThreaded(_Install, vms)
table_name = _GetTableName()
# If the table already exists, it will be an no-op.
hbase_ycsb.CreateYCSBTable(vms[0],
table_name=table_name,
use_snappy=False,
limit_filesize=False)
# Add hbase conf dir to the classpath.
ycsb_memory = min(vms[0].total_memory_kb // 1024, 4096)
jvm_args = pipes.quote(' -Xmx{0}m'.format(ycsb_memory))
executor_flags = {
'cp': hbase.HBASE_CONF_DIR,
'jvm-args': jvm_args,
'table': table_name
}
benchmark_spec.executor = ycsb.YCSBExecutor('hbase10', **executor_flags)
def Prepare(benchmark_spec):
"""Prepare the cloud redis instance to YCSB tasks.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
ycsb_vms = benchmark_spec.vm_groups['clients']
vm_util.RunThreaded(_Install, ycsb_vms)
instance_details = benchmark_spec.cloud_redis.GetInstanceDetails()
benchmark_spec.executor = ycsb.YCSBExecutor(
'redis', **{
'shardkeyspace': True,
'redis.host': instance_details['host'],
'redis.port': instance_details['port']})
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud datastore.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
if FLAGS.google_datastore_repopulate:
EmptyDatabase()
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
# Restore YCSB_TAR_URL
benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Run(benchmark_spec):
vms = benchmark_spec.vms
executor = ycsb.YCSBExecutor('googledatastore')
run_kwargs = {
'googledatastore.datasetId': FLAGS.google_datastore_datasetId,
'googledatastore.privateKeyFile': PRIVATE_KEYFILE_DIR,
'googledatastore.serviceAccountEmail':
FLAGS.google_datastore_serviceAccount,
'googledatastore.debug': FLAGS.google_datastore_debug,
}
load_kwargs = run_kwargs.copy()
if FLAGS['ycsb_preload_threads'].present:
load_kwargs['threads'] = FLAGS['ycsb_preload_threads']
samples = list(
executor.LoadAndRun(vms,
load_kwargs=load_kwargs,
run_kwargs=run_kwargs))
return samples
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run YCSB against Cassandra.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
vms = benchmark_spec.vms
by_role = _GetVMsByRole(benchmark_spec)
loaders = by_role['clients']
assert loaders, vms
# Cassandra cluster
cassandra_vms = by_role['cassandra_vms']
assert cassandra_vms, 'No Cassandra VMs: {0}'.format(by_role)
seed_vm = by_role['seed_vm']
assert seed_vm, 'No seed VM: {0}'.format(by_role)
cassandra_install_fns = [
functools.partial(_InstallCassandra, vm, seed_vms=[seed_vm])
for vm in cassandra_vms
]
ycsb_install_fns = [
functools.partial(vm.Install, 'ycsb') for vm in loaders
]
if FLAGS.ycsb_client_vms:
vm_util.RunThreaded(lambda f: f(),
cassandra_install_fns + ycsb_install_fns)
else:
# If putting server and client on same vm, prepare packages one by one to
# avoid race condition.
vm_util.RunThreaded(lambda f: f(), cassandra_install_fns)
vm_util.RunThreaded(lambda f: f(), ycsb_install_fns)
cassandra.StartCluster(seed_vm, by_role['non_seed_cassandra_vms'])
_CreateYCSBTable(seed_vm,
replication_factor=FLAGS.cassandra_replication_factor)
benchmark_spec.executor = ycsb.YCSBExecutor(
'cassandra2-cql',
hosts=','.join(vm.internal_ip for vm in cassandra_vms))
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run cloud spanner benchmarks.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
benchmark_spec.always_call_cleanup = True
if FLAGS.cloud_spanner_ycsb_client_type != 'java':
ycsb.SetYcsbTarUrl(
CLIENT_TAR_URL[FLAGS.cloud_spanner_ycsb_client_type])
vms = benchmark_spec.vms
# Install required packages and copy credential files
vm_util.RunThreaded(_Install, vms)
benchmark_spec.executor = ycsb.YCSBExecutor('cloudspanner')
