def Prepare(benchmark_spec):
"""Prepare the virtual machines to run hadoop.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
vms = benchmark_spec.vms
master = vms[0]
workers = vms[1:]
def InstallHadoop(vm):
vm.Install('hadoop')
vm_util.RunThreaded(InstallHadoop, vms)
hadoop.ConfigureAndStart(master, workers)
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run hadoop.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
by_role = _GetVMsByRole(benchmark_spec.vms)
loaders = by_role['loaders']
assert loaders, 'No loader VMs: {0}'.format(by_role)
# HBase cluster
hbase_vms = by_role['hbase_vms']
assert hbase_vms, 'No HBase VMs: {0}'.format(by_role)
master = by_role['master']
zk_quorum = by_role['zk_quorum']
assert zk_quorum, 'No zookeeper quorum: {0}'.format(by_role)
workers = by_role['workers']
assert workers, 'No workers: {0}'.format(by_role)
hbase_install_fns = [
functools.partial(vm.Install, 'hbase') for vm in hbase_vms
]
ycsb_install_fns = [
functools.partial(vm.Install, 'ycsb') for vm in loaders
]
vm_util.RunThreaded(lambda f: f(), hbase_install_fns + ycsb_install_fns)
hadoop.ConfigureAndStart(master, workers, start_yarn=False)
hbase.ConfigureAndStart(master, workers, zk_quorum)
_CreateYCSBTable(master)
# Populate hbase-site.xml on the loaders.
master.PullFile(vm_util.GetTempDir(),
posixpath.join(hbase.HBASE_CONF_DIR, HBASE_SITE))
def PushHBaseSite(vm):
conf_dir = posixpath.join(ycsb.YCSB_DIR, 'hbase-binding', 'conf')
vm.RemoteCommand('mkdir -p {}'.format(conf_dir))
vm.PushFile(os.path.join(vm_util.GetTempDir(), HBASE_SITE),
posixpath.join(conf_dir, HBASE_SITE))
vm_util.RunThreaded(PushHBaseSite, loaders)
def Prepare(benchmark_spec):
"""Prepare the virtual machines to run hadoop.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
"""
vms = benchmark_spec.vms
by_role = _GetVMsByRole(benchmark_spec.vms)
loaders = by_role['loaders']
assert loaders, 'No loader VMs: {0}'.format(by_role)
# HBase cluster
hbase_vms = by_role['hbase_vms']
assert hbase_vms, 'No HBase VMs: {0}'.format(by_role)
master = by_role['master']
zk_quorum = by_role['zk_quorum']
assert zk_quorum, 'No zookeeper quorum: {0}'.format(by_role)
workers = by_role['workers']
assert workers, 'No workers: {0}'.format(by_role)
vms = benchmark_spec.vms
def InstallHBase(vm):
vm.Install('hbase')
vm_util.RunThreaded(InstallHBase, vms)
hadoop.ConfigureAndStart(master, workers, start_yarn=False)
hbase.ConfigureAndStart(master, workers, zk_quorum)
_CreateYCSBTable(master)
def InstallYCSB(vm):
vm.Install('ycsb')
vm_util.RunThreaded(InstallYCSB, loaders)
# Populate hbase-site.xml on the loaders.
master.PullFile(
vm_util.GetTempDir(),
posixpath.join(hbase.HBASE_CONF_DIR, HBASE_SITE))
for loader_vm in loaders:
loader_vm.PushFile(
os.path.join(vm_util.GetTempDir(), HBASE_SITE),
posixpath.join(ycsb.YCSB_DIR, 'hbase-binding', 'conf', HBASE_SITE))