def Prepare(benchmark_spec: bm_spec.BenchmarkSpec) -> None: """Prepare the virtual machines to run cloud bigtable. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True vms = benchmark_spec.vms if _TABLE_OBJECT_SHARING.value: ycsb.SetYcsbTarUrl(YCSB_BIGTABLE_TABLE_SHARING_TAR_URL) instance: _Bigtable = benchmark_spec.non_relational_db args = [((vm, instance), {}) for vm in vms] vm_util.RunThreaded(_Install, args) table_name = _GetTableName() # If the table already exists, it will be an no-op. hbase_ycsb.CreateYCSBTable(vms[0], table_name=table_name, use_snappy=False, limit_filesize=False) # Add hbase conf dir to the classpath. ycsb_memory = min(vms[0].total_memory_kb // 1024, 4096) jvm_args = pipes.quote(f' -Xmx{ycsb_memory}m') executor_flags = { 'cp': hbase.HBASE_CONF_DIR, 'jvm-args': jvm_args, 'table': table_name} benchmark_spec.executor = ycsb.YCSBExecutor(FLAGS.hbase_binding, **executor_flags)
def Prepare(benchmark_spec): benchmark_spec.always_call_cleanup = True vms = benchmark_spec.vms # Install required packages and copy credential files. vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud spanner benchmarks. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True benchmark_spec.spanner_instance = gcp_spanner.GcpSpannerInstance( name=BENCHMARK_INSTANCE_PREFIX + FLAGS.run_uri, description=BENCHMARK_DESCRIPTION, database=BENCHMARK_DATABASE, ddl=BENCHMARK_SCHEMA) if benchmark_spec.spanner_instance._Exists(instance_only=True): logging.warning('Cloud Spanner instance %s exists, delete it first.' % FLAGS.cloud_spanner_ycsb_instance) benchmark_spec.spanner_instance.Delete() benchmark_spec.spanner_instance.Create() if not benchmark_spec.spanner_instance._Exists(): logging.warning( 'Failed to create Cloud Spanner instance and database.') benchmark_spec.spanner_instance.Delete() if FLAGS.cloud_spanner_ycsb_client_type != 'java': ycsb.SetYcsbTarUrl( CLIENT_TAR_URL[FLAGS.cloud_spanner_ycsb_client_type]) vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('cloudspanner')
def Run(benchmark_spec): """Spawn YCSB and gather the results. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. Returns: A list of sample.Sample objects. """ by_role = _GetVMsByRole(benchmark_spec.vm_groups) loaders = by_role['clients'] logging.info('Loaders: %s', loaders) executor = ycsb.YCSBExecutor('hbase-10') metadata = {'ycsb_client_vms': len(loaders), 'hbase_cluster_size': len(by_role['hbase_vms']), 'hbase_zookeeper_nodes': FLAGS.hbase_zookeeper_nodes} # By default YCSB uses a BufferedMutator for Puts / Deletes. # This leads to incorrect update latencies, since since the call returns # before the request is acked by the server. # Disable this behavior during the benchmark run. run_kwargs = {'columnfamily': COLUMN_FAMILY, 'clientbuffering': 'false'} load_kwargs = run_kwargs.copy() load_kwargs['clientbuffering'] = 'true' samples = list(executor.LoadAndRun(loaders, load_kwargs=load_kwargs, run_kwargs=run_kwargs)) for sample in samples: sample.metadata.update(metadata) return samples
def Prepare(benchmark_spec): """Prepare the virtual machines to run YCSB against memcached. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ clients = benchmark_spec.vm_groups['clients'] assert clients, benchmark_spec.vm_groups hosts = [] if FLAGS.memcached_scenario == 'managed': # We need to delete the managed memcached backend when we're done benchmark_spec.always_call_cleanup = True if FLAGS.memcached_managed == providers.GCP: raise NotImplementedError( "GCP managed memcached backend not implemented " "yet") elif FLAGS.memcached_managed == providers.AWS: cluster_id = 'pkb%s' % FLAGS.run_uri service = providers.aws.elasticache.ElastiCacheMemcacheService( aws_network.AwsNetwork.GetNetwork(clients[0]), cluster_id, FLAGS.memcached_elasticache_region, FLAGS.memcached_elasticache_node_type, FLAGS.memcached_elasticache_num_servers) service.Create() hosts = service.GetHosts() benchmark_spec.service = service benchmark_spec.metadata = service.GetMetadata() else: # custom scenario # Install memcached on all the servers servers = benchmark_spec.vm_groups['servers'] assert servers, 'No memcached servers: {0}'.format( benchmark_spec.vm_groups) memcached_install_fns = \ [functools.partial(memcached_server.ConfigureAndStart, vm) for vm in servers] vm_util.RunThreaded(lambda f: f(), memcached_install_fns) hosts = [ '%s:%s' % (vm.internal_ip, memcached_server.MEMCACHED_PORT) for vm in servers ] benchmark_spec.metadata = { 'ycsb_client_vms': FLAGS.ycsb_client_vms, 'ycsb_server_vms': FLAGS.ycsb_server_vms, 'num_vms': len(servers), 'cache_size': FLAGS.memcached_size_mb } assert len(hosts) > 0 ycsb_install_fns = [ functools.partial(vm.Install, 'ycsb') for vm in clients ] vm_util.RunThreaded(lambda f: f(), ycsb_install_fns) benchmark_spec.executor = ycsb.YCSBExecutor( 'memcached', **{'memcached.hosts': ','.join(hosts)})
def Prepare(benchmark_spec): """Prepare the virtual machines to run YCSB against Cassandra. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ vms = benchmark_spec.vms by_role = _GetVMsByRole(benchmark_spec) loaders = by_role['clients'] assert loaders, vms # Cassandra cluster cassandra_vms = by_role['cassandra_vms'] assert cassandra_vms, 'No Cassandra VMs: {0}'.format(by_role) seed_vm = by_role['seed_vm'] assert seed_vm, 'No seed VM: {0}'.format(by_role) cassandra_install_fns = [functools.partial(_InstallCassandra, vm, seed_vms=[seed_vm]) for vm in cassandra_vms] ycsb_install_fns = [functools.partial(vm.Install, 'ycsb') for vm in loaders] vm_util.RunThreaded(lambda f: f(), cassandra_install_fns + ycsb_install_fns) cassandra.StartCluster(seed_vm, by_role['non_seed_cassandra_vms']) _CreateYCSBTable(seed_vm) benchmark_spec.executor = ycsb.YCSBExecutor( 'cassandra-10', hosts=','.join(vm.internal_ip for vm in cassandra_vms))
def Prepare(benchmark_spec): """Prepare the virtual machines to run YCSB against memcached. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ loaders = benchmark_spec.vm_groups['clients'] assert loaders, benchmark_spec.vm_groups # Memcached cluster memcached_vms = benchmark_spec.vm_groups['servers'] assert memcached_vms, 'No memcached VMs: {0}'.format( benchmark_spec.vm_groups) memcached_install_fns = [ functools.partial(memcached_server.ConfigureAndStart, vm) for vm in memcached_vms ] ycsb_install_fns = [ functools.partial(vm.Install, 'ycsb') for vm in loaders ] vm_util.RunThreaded(lambda f: f(), memcached_install_fns + ycsb_install_fns) benchmark_spec.executor = ycsb.YCSBExecutor( 'memcached', **{ 'memcached.hosts': ','.join([vm.internal_ip for vm in memcached_vms]) })
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud datastore. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True # Check that the database is empty before running if FLAGS.google_datastore_deletion_keyfile: dataset_id = FLAGS.google_datastore_datasetId credentials = GetDatastoreDeleteCredentials() client = datastore.Client(project=dataset_id, credentials=credentials) for kind in _YCSB_COLLECTIONS: # TODO(user): Allow a small number of leftover entities until we # figure out why these are not getting deleted. if len(list(client.query(kind=kind).fetch(limit=200))) > 100: raise errors.Benchmarks.PrepareException( 'Database is non-empty. Stopping test.') else: logging.warning('Test could be executed on a non-empty database.') vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) # Restore YCSB_TAR_URL benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Prepare(benchmark_spec): """Prepare the virtual machines to run YCSB against Aerospike. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ loaders = benchmark_spec.vm_groups['clients'] assert loaders, benchmark_spec.vm_groups # Aerospike cluster aerospike_vms = benchmark_spec.vm_groups['workers'] assert aerospike_vms, 'No aerospike VMs: {0}'.format( benchmark_spec.vm_groups) seed_ips = [vm.internal_ip for vm in aerospike_vms] aerospike_install_fns = [ functools.partial(aerospike_server.ConfigureAndStart, vm, seed_node_ips=seed_ips) for vm in aerospike_vms ] ycsb_install_fns = [ functools.partial(vm.Install, 'ycsb') for vm in loaders ] vm_util.RunThreaded(lambda f: f(), aerospike_install_fns + ycsb_install_fns) benchmark_spec.executor = ycsb.YCSBExecutor( 'aerospike', **{ 'as.host': aerospike_vms[0].internal_ip, 'as.namespace': 'test' })
def Prepare(benchmark_spec): """Prepare the cloud redis instance to YCSB tasks. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True ycsb_vms = benchmark_spec.vm_groups['clients'] vm_util.RunThreaded(_Install, ycsb_vms) cloud_redis_class = (managed_memory_store.GetManagedMemoryStoreClass( FLAGS.cloud, managed_memory_store.REDIS)) benchmark_spec.cloud_redis_instance = (cloud_redis_class(benchmark_spec)) benchmark_spec.cloud_redis_instance.Create() instance_details = benchmark_spec.cloud_redis_instance.GetInstanceDetails() redis_args = { 'shardkeyspace': True, 'redis.host': instance_details['host'], 'redis.port': instance_details['port'] } if 'password' in instance_details: redis_args['redis.password'] = instance_details['password'] benchmark_spec.executor = ycsb.YCSBExecutor('redis', **redis_args)
def Run(benchmark_spec): """Spawn YCSB and gather the results. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. Returns: A list of sample.Sample instances. """ loaders = benchmark_spec.vm_groups['clients'] aerospike_vms = benchmark_spec.vm_groups['workers'] executor = ycsb.YCSBExecutor( 'aerospike', **{ 'as.host': aerospike_vms[0].internal_ip, 'as.namespace': 'test' }) metadata = { 'ycsb_client_vms': FLAGS.ycsb_client_vms, 'num_vms': len(aerospike_vms) } samples = list(executor.LoadAndRun(loaders)) for sample in samples: sample.metadata.update(metadata) return samples
def Prepare(benchmark_spec): """Install Redis on one VM and memtier_benchmark on another. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ groups = benchmark_spec.vm_groups redis_vm = groups['workers'][0] ycsb_vms = groups['clients'] prepare_fns = ([functools.partial(PrepareServer, redis_vm)] + [functools.partial(vm.Install, 'ycsb') for vm in ycsb_vms]) vm_util.RunThreaded(lambda f: f(), prepare_fns) num_ycsb = FLAGS.redis_ycsb_processes num_server = FLAGS.redis_total_num_processes # Each redis process use different ports, number of ycsb processes should # be at least as large as number of server processes, use round-robin # to assign target server process to each ycsb process server_metadata = [{ 'redis.port': redis_server.REDIS_FIRST_PORT + i % num_server } for i in range(num_ycsb)] benchmark_spec.executor = ycsb.YCSBExecutor( 'redis', **{ 'shardkeyspace': True, 'redis.host': redis_vm.internal_ip, 'perclientparam': server_metadata }) vm_util.SetupSimulatedMaintenance(redis_vm)
def Run(benchmark_spec): """Spawn YCSB and gather the results. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. Returns: A list of sample.Sample instances. """ vms = benchmark_spec.vms table_name = _GetTableName() # Add hbase conf dir to the classpath. ycsb_memory = ycsb_memory = min(vms[0].total_memory_kb // 1024, 4096) jvm_args = pipes.quote(' -Xmx{0}m'.format(ycsb_memory)) executor_flags = { 'cp': hbase.HBASE_CONF_DIR, 'jvm-args': jvm_args, 'table': table_name } executor = ycsb.YCSBExecutor('hbase-10', **executor_flags) cluster_name = (FLAGS.google_bigtable_cluster_name or 'pkb-bigtable-{0}'.format(FLAGS.run_uri)) cluster_info = _GetClusterDescription( FLAGS.project or _GetDefaultProject(), FLAGS.google_bigtable_zone_name, cluster_name) metadata = { 'ycsb_client_vms': len(vms), 'bigtable_nodes': cluster_info.get('serveNodes') } # By default YCSB uses a BufferedMutator for Puts / Deletes. # This leads to incorrect update latencies, since since the call returns # before the request is acked by the server. # Disable this behavior during the benchmark run. run_kwargs = {'columnfamily': COLUMN_FAMILY, 'clientbuffering': 'false'} load_kwargs = run_kwargs.copy() # During the load stage, use a buffered mutator with a single thread. # The BufferedMutator will handle multiplexing RPCs. load_kwargs['clientbuffering'] = 'true' if not FLAGS['ycsb_preload_threads'].present: load_kwargs['threads'] = 1 samples = list( executor.LoadAndRun(vms, load_kwargs=load_kwargs, run_kwargs=run_kwargs)) for sample in samples: sample.metadata.update(metadata) return samples
def Prepare(benchmark_spec): benchmark_spec.always_call_cleanup = True ycsb.SetYcsbTarUrl(YCSB_BINDING_TAR_URL) vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) # Restore YCSB_TAR_URL ycsb.SetYcsbTarUrl(None) benchmark_spec.executor = ycsb.YCSBExecutor('googlefirestore')
def Prepare(benchmark_spec): benchmark_spec.always_call_cleanup = True vms = benchmark_spec.vms # Install required packages and copy credential files. vm_util.RunThreaded(_Install, vms) # Create benchmark table. ExecuteSql(vms[0], DROP_TABLE_SQL) ExecuteSql(vms[0], CREATE_TABLE_SQL) benchmark_spec.executor = ycsb.YCSBExecutor('jdbc')
def Prepare(benchmark_spec): """Install YCSB on the target vm. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True vms = benchmark_spec.vms # Install required packages. vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')
def testRunCalledWithCorrectTarget(self): # Arrange FLAGS.ycsb_workload_files = ['workloadc'] test_executor = ycsb.YCSBExecutor('test_database') test_vm = mock.Mock() test_vm.RobustRemoteCommand.return_value = ['', ''] self.enter_context(mock.patch.object(ycsb, 'ParseResults')) # Act test_executor.Run([test_vm], run_kwargs={'target': 1000}) # Assert self.assertIn('-target 1000', test_vm.RobustRemoteCommand.call_args[0][0])
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud spanner benchmarks. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True if not FLAGS.cloud_spanner_instance_name else False instance_name = FLAGS.cloud_spanner_instance_name or BENCHMARK_INSTANCE_PREFIX + FLAGS.run_uri instance_description = FLAGS.cloud_spanner_instance_description or BENCHMARK_DESCRIPTION benchmark_spec.spanner_instance = gcp_spanner.GcpSpannerInstance( name=instance_name, description=instance_description, database=BENCHMARK_DATABASE, ddl=BENCHMARK_SCHEMA) # If instance name is provided, we might re-use an existing instance if FLAGS.cloud_spanner_instance_name: benchmark_spec.always_call_cleanup = False if benchmark_spec.spanner_instance._Exists(instance_only=True): logging.info("Re-using existing instance %s", instance_name) else: logging.info("Creating new instance %s", instance_name) benchmark_spec.spanner_instance.Create() # If instance name is not provided, we delete the existing instance before creating elif benchmark_spec.spanner_instance._Exists(instance_only=True): logging.warning('Cloud Spanner instance %s exists, delete it first.' % instance_name) benchmark_spec.spanner_instance.Delete() logging.info("Creating new instance %s", instance_name) benchmark_spec.spanner_instance.Create() if not benchmark_spec.spanner_instance._Exists(): logging.warning( 'Failed to create Cloud Spanner instance and database.') benchmark_spec.spanner_instance.Delete() if FLAGS.cloud_spanner_ycsb_client_type != 'java': ycsb.SetYcsbTarUrl( CLIENT_TAR_URL[FLAGS.cloud_spanner_ycsb_client_type]) vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('cloudspanner')
def Prepare(benchmark_spec): """Install MongoDB on one VM and YCSB on another. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ server_partials = [functools.partial(_PrepareServer, mongo_vm) for mongo_vm in benchmark_spec.vm_groups['workers']] client_partials = [functools.partial(_PrepareClient, client) for client in benchmark_spec.vm_groups['clients']] vm_util.RunThreaded((lambda f: f()), server_partials + client_partials) benchmark_spec.executor = ycsb.YCSBExecutor('mongodb', cp=ycsb.YCSB_DIR)
def Prepare(benchmark_spec): benchmark_spec.always_call_cleanup = True default_ycsb_tar_url = ycsb.YCSB_TAR_URL vms = benchmark_spec.vms # TODO: figure out a less hacky way to override. # Override so that we only need to download the required binding. ycsb.YCSB_TAR_URL = YCSB_BINDING_TAR_URL # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) # Restore YCSB_TAR_URL ycsb.YCSB_TAR_URL = default_ycsb_tar_url benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud datastore. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) # Restore YCSB_TAR_URL benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Prepare(benchmark_spec): """Install YCSB on the target vm. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True benchmark_spec.firestore_instance = gcp_firestore.GcpFirestoreInstance( 'pkb-{0}'.format(FLAGS.run_uri), FLAGS.zones[0]) vms = benchmark_spec.vms vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('googlefirestore')
def Prepare(benchmark_spec): """Prepare the virtual machines to run hadoop. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ by_role = _GetVMsByRole(benchmark_spec.vm_groups) loaders = by_role['clients'] assert loaders, 'No loader VMs: {0}'.format(by_role) # HBase cluster hbase_vms = by_role['hbase_vms'] assert hbase_vms, 'No HBase VMs: {0}'.format(by_role) master = by_role['master'] zk_quorum = by_role['zk_quorum'] assert zk_quorum, 'No zookeeper quorum: {0}'.format(by_role) workers = by_role['workers'] assert workers, 'No workers: {0}'.format(by_role) hbase_install_fns = [ functools.partial(vm.Install, 'hbase') for vm in hbase_vms ] ycsb_install_fns = [ functools.partial(vm.Install, 'ycsb') for vm in loaders ] vm_util.RunThreaded(lambda f: f(), hbase_install_fns + ycsb_install_fns) hadoop.ConfigureAndStart(master, workers, start_yarn=False) hbase.ConfigureAndStart(master, workers, zk_quorum) CreateYCSBTable(master, use_snappy=FLAGS.hbase_use_snappy) # Populate hbase-site.xml on the loaders. master.PullFile(vm_util.GetTempDir(), posixpath.join(hbase.HBASE_CONF_DIR, HBASE_SITE)) def PushHBaseSite(vm): conf_dir = posixpath.join(ycsb.YCSB_DIR, FLAGS.hbase_binding + '-binding', 'conf') vm.RemoteCommand('mkdir -p {}'.format(conf_dir)) vm.PushFile(os.path.join(vm_util.GetTempDir(), HBASE_SITE), posixpath.join(conf_dir, HBASE_SITE)) vm_util.RunThreaded(PushHBaseSite, loaders) benchmark_spec.executor = ycsb.YCSBExecutor(FLAGS.hbase_binding)
def Prepare(benchmark_spec): """Install YCSB on the target vm. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True benchmark_spec.dynamodb_instance = aws_dynamodb.AwsDynamoDBInstance( table_name='pkb-{0}'.format(FLAGS.run_uri)) benchmark_spec.dynamodb_instance._Create() vms = benchmark_spec.vms # Install required packages. vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('dynamodb')
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud bigtable. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True vms = benchmark_spec.vms if FLAGS.google_bigtable_enable_table_object_sharing: ycsb.SetYcsbTarUrl(YCSB_BIGTABLE_TABLE_SHARING_TAR_URL) # TODO: in the future, it might be nice to change this so that # a gcp_bigtable.GcpBigtableInstance can be created with an # flag that says don't create/delete the instance. That would # reduce the code paths here. if FLAGS.google_bigtable_instance_name is None: instance_name = 'pkb-bigtable-{0}'.format(FLAGS.run_uri) project = FLAGS.project or _GetDefaultProject() logging.info('Creating bigtable instance %s', instance_name) zone = FLAGS.google_bigtable_zone benchmark_spec.bigtable_instance = gcp_bigtable.GcpBigtableInstance( instance_name, project, zone) benchmark_spec.bigtable_instance.Create() instance = _GetInstanceDescription(project, instance_name) logging.info('Instance %s created successfully', instance) vm_util.RunThreaded(_Install, vms) table_name = _GetTableName() # If the table already exists, it will be an no-op. hbase_ycsb.CreateYCSBTable(vms[0], table_name=table_name, use_snappy=False, limit_filesize=False) # Add hbase conf dir to the classpath. ycsb_memory = min(vms[0].total_memory_kb // 1024, 4096) jvm_args = pipes.quote(' -Xmx{0}m'.format(ycsb_memory)) executor_flags = { 'cp': hbase.HBASE_CONF_DIR, 'jvm-args': jvm_args, 'table': table_name } benchmark_spec.executor = ycsb.YCSBExecutor('hbase10', **executor_flags)
def Prepare(benchmark_spec): """Prepare the cloud redis instance to YCSB tasks. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True ycsb_vms = benchmark_spec.vm_groups['clients'] vm_util.RunThreaded(_Install, ycsb_vms) instance_details = benchmark_spec.cloud_redis.GetInstanceDetails() benchmark_spec.executor = ycsb.YCSBExecutor( 'redis', **{ 'shardkeyspace': True, 'redis.host': instance_details['host'], 'redis.port': instance_details['port']})
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud datastore. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ if FLAGS.google_datastore_repopulate: EmptyDatabase() vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) # Restore YCSB_TAR_URL benchmark_spec.executor = ycsb.YCSBExecutor('googledatastore')
def Run(benchmark_spec): vms = benchmark_spec.vms executor = ycsb.YCSBExecutor('googledatastore') run_kwargs = { 'googledatastore.datasetId': FLAGS.google_datastore_datasetId, 'googledatastore.privateKeyFile': PRIVATE_KEYFILE_DIR, 'googledatastore.serviceAccountEmail': FLAGS.google_datastore_serviceAccount, 'googledatastore.debug': FLAGS.google_datastore_debug, } load_kwargs = run_kwargs.copy() if FLAGS['ycsb_preload_threads'].present: load_kwargs['threads'] = FLAGS['ycsb_preload_threads'] samples = list( executor.LoadAndRun(vms, load_kwargs=load_kwargs, run_kwargs=run_kwargs)) return samples
def Prepare(benchmark_spec): """Prepare the virtual machines to run YCSB against Cassandra. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ vms = benchmark_spec.vms by_role = _GetVMsByRole(benchmark_spec) loaders = by_role['clients'] assert loaders, vms # Cassandra cluster cassandra_vms = by_role['cassandra_vms'] assert cassandra_vms, 'No Cassandra VMs: {0}'.format(by_role) seed_vm = by_role['seed_vm'] assert seed_vm, 'No seed VM: {0}'.format(by_role) cassandra_install_fns = [ functools.partial(_InstallCassandra, vm, seed_vms=[seed_vm]) for vm in cassandra_vms ] ycsb_install_fns = [ functools.partial(vm.Install, 'ycsb') for vm in loaders ] if FLAGS.ycsb_client_vms: vm_util.RunThreaded(lambda f: f(), cassandra_install_fns + ycsb_install_fns) else: # If putting server and client on same vm, prepare packages one by one to # avoid race condition. vm_util.RunThreaded(lambda f: f(), cassandra_install_fns) vm_util.RunThreaded(lambda f: f(), ycsb_install_fns) cassandra.StartCluster(seed_vm, by_role['non_seed_cassandra_vms']) _CreateYCSBTable(seed_vm, replication_factor=FLAGS.cassandra_replication_factor) benchmark_spec.executor = ycsb.YCSBExecutor( 'cassandra2-cql', hosts=','.join(vm.internal_ip for vm in cassandra_vms))
def Prepare(benchmark_spec): """Prepare the virtual machines to run cloud spanner benchmarks. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ benchmark_spec.always_call_cleanup = True if FLAGS.cloud_spanner_ycsb_client_type != 'java': ycsb.SetYcsbTarUrl( CLIENT_TAR_URL[FLAGS.cloud_spanner_ycsb_client_type]) vms = benchmark_spec.vms # Install required packages and copy credential files vm_util.RunThreaded(_Install, vms) benchmark_spec.executor = ycsb.YCSBExecutor('cloudspanner')