def exitGracefully(storageClients, signal, frame):
print ""
print "\033[1;31mShutting down Storage\033[0m"
for client in storageClients:
client.kill()
unmount_memfs()
exit(0)
def startStorage():
# to be on the safe side, first unmount the drives if they are still mounted for whatever reason
unmount_memfs()
javaHome = ""
masterObs = ""
storageObs = ""
numa0Args = ""
numa1Args = ""
if Storage.storage == Kudu:
confKudu()
master_cmd = '/mnt/local/tell/kudu_install/bin/kudu-master --fs_data_dirs={0} --fs_wal_dir={0} --block_manager=file'.format(Kudu.master_dir)
server_cmd = '/mnt/local/tell/kudu_install/bin/kudu-tserver'
numa0Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1}".format(Kudu.tserver_dir, Storage.master)
numa1Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1} --rpc_bind_addresses=0.0.0.0:7049 --webserver_port=8049".format(Kudu.tserver_dir1, Storage.master)
elif Storage.storage == TellStore:
TellStore.rsyncBuild()
master_cmd = "{0}/commitmanager/server/commitmanagerd".format(TellStore.builddir)
server_cmd = "{0}/tellstore/server/tellstored-{1} -l INFO --scan-threads {2} --network-threads 1 --gc-interval {5} -m {3} -c {4}".format(TellStore.builddir, TellStore.approach, TellStore.scanThreads, TellStore.memorysize, TellStore.hashmapsize, TellStore.gcInterval)
numa1Args = '-p 7240'
storageObs = "Storage ready"
elif Storage.storage == Hadoop:
confHdfs()
return startHdfs()
elif Storage.storage == Hbase:
confHbaseCluster()
return startHbaseThreads()
elif Storage.storage == Cassandra:
confCassandraCluster()
return startCassandra()
elif Storage.storage == Ramcloud:
confRamcloud()
return startRamcloud()
return startStorageThreads(master_cmd, server_cmd, numa0Args, numa1Args, masterObs, storageObs, javaHome)
def exitGracefully(storageClients, signal, frame):
print ""
print "\033[1;31mShutting down Storage\033[0m"
if Storage.storage == Memsql:
stopMemsql()
for client in storageClients:
client.kill()
unmount_memfs()
exit(0)
def runMBench(outdir, onlyPopulation=False):
# do as for many experiments we have to run
## start storages
storageClients = startStorage()
signal.signal(signal.SIGINT, partial(exitGracefully, storageClients))
print "Storage started"
## start microbenchmark server
mbObserver = Observer("Started mbench server")
serverClients = startMBServer([mbObserver])
mbObserver.waitFor(len(Microbench.servers0) + len(Microbench.servers1))
time.sleep(2)
print "Server started... Continue with population"
clients = Microbench.clients
Microbench.clients = 10 * (len(Microbench.servers0) +
len(Microbench.servers1)) - 1
res = startMBClient(True, "{0}/{1}_population".format(outdir, sqliteOut()))
Microbench.clients = clients
if res != 0:
print "Population failed"
exit(res)
print "Population done"
if not onlyPopulation:
res = startMBClient(False, "{0}/{1}".format(outdir, sqliteOut()))
if res != 0:
print "Benchmark failed"
exit(res)
for client in serverClients:
client.kill()
for client in storageClients:
client.kill()
for client in serverClients:
client.join()
for client in storageClients:
client.join()
unmount_memfs()
def runMBench(outdir, onlyPopulation = False):
# do as for many experiments we have to run
## start storages
storageClients = startStorage()
signal.signal(signal.SIGINT, partial(exitGracefully, storageClients))
print "Storage started"
## start microbenchmark server
mbObserver = Observer("Started mbench server")
serverClients = startMBServer([mbObserver])
mbObserver.waitFor(len(Microbench.servers0) + len(Microbench.servers1))
time.sleep(2)
print "Server started... Continue with population"
clients = Microbench.clients
Microbench.clients = 10*(len(Microbench.servers0) + len(Microbench.servers1)) - 1
res = startMBClient(True, "{0}/{1}_population".format(outdir, sqliteOut()))
Microbench.clients = clients
if res != 0:
print "Population failed"
exit(res)
print "Population done"
if not onlyPopulation:
res = startMBClient(False, "{0}/{1}".format(outdir, sqliteOut()))
if res != 0:
print "Benchmark failed"
exit(res)
for client in serverClients:
client.kill()
for client in storageClients:
client.kill()
for client in serverClients:
client.join()
for client in storageClients:
client.join()
unmount_memfs()
def startStorage():
# to be on the safe side, first unmount the drives if they are still mounted for whatever reason
unmount_memfs()
javaHome = ""
masterObs = ""
storageObs = ""
numa0Args = ""
numa1Args = ""
if Storage.storage == Kudu:
confKudu()
master_cmd = '/mnt/local/tell/kudu_install/bin/kudu-master --fs_data_dirs={0} --fs_wal_dir={0} --block_manager=file'.format(Kudu.master_dir)
server_cmd = '/mnt/local/tell/kudu_install/bin/kudu-tserver'
numa0Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1}".format(Kudu.tserver_dir, Storage.master)
numa1Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1} --rpc_bind_addresses=0.0.0.0:7049 --webserver_port=8049".format(Kudu.tserver_dir1, Storage.master)
elif Storage.storage == TellStore:
TellStore.rsyncBuild()
master_cmd = "{0}/commitmanager/server/commitmanagerd".format(TellStore.builddir)
server_cmd = "{0}/tellstore/server/tellstored-{1} -l INFO --scan-threads {2} --network-threads 1 --gc-interval {5} -m {3} -c {4}".format(TellStore.builddir, TellStore.approach, TellStore.scanThreads, TellStore.memorysize, TellStore.hashmapsize, TellStore.gcInterval)
numa1Args = '-p 7240'
storageObs = "Storage ready"
elif Storage.storage == Hadoop:
confHdfs()
return startHdfs()
elif Storage.storage == Hbase:
confHbaseCluster()
return startHbaseThreads()
elif Storage.storage == Cassandra:
confCassandraCluster()
return startCassandra()
elif Storage.storage == Ramcloud:
confRamcloud()
return startRamcloud()
elif Storage.storage == Memsql:
return startMemsql()
return startStorageThreads(master_cmd, server_cmd, numa0Args, numa1Args, masterObs, storageObs, javaHome)
