def runOnTell(experiment, outdir, numNodes): Microbench.txBatch = 200 for approach in ["columnmap", "rowstore", "logstructured"]: TellStore.approach = approach TellStore.setDefaultMemorySizeAndScanThreads() for num in numNodes: experiment(outdir, num)
def startStorage(): # to be on the safe side, first unmount the drives if they are still mounted for whatever reason unmount_memfs() javaHome = "" masterObs = "" storageObs = "" numa0Args = "" numa1Args = "" if Storage.storage == Kudu: confKudu() master_cmd = '/mnt/local/tell/kudu_install/bin/kudu-master --fs_data_dirs={0} --fs_wal_dir={0} --block_manager=file'.format(Kudu.master_dir) server_cmd = '/mnt/local/tell/kudu_install/bin/kudu-tserver' numa0Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1}".format(Kudu.tserver_dir, Storage.master) numa1Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1} --rpc_bind_addresses=0.0.0.0:7049 --webserver_port=8049".format(Kudu.tserver_dir1, Storage.master) elif Storage.storage == TellStore: TellStore.rsyncBuild() master_cmd = "{0}/commitmanager/server/commitmanagerd".format(TellStore.builddir) server_cmd = "{0}/tellstore/server/tellstored-{1} -l INFO --scan-threads {2} --network-threads 1 --gc-interval {5} -m {3} -c {4}".format(TellStore.builddir, TellStore.approach, TellStore.scanThreads, TellStore.memorysize, TellStore.hashmapsize, TellStore.gcInterval) numa1Args = '-p 7240' storageObs = "Storage ready" elif Storage.storage == Hadoop: confHdfs() return startHdfs() elif Storage.storage == Hbase: confHbaseCluster() return startHbaseThreads() elif Storage.storage == Cassandra: confCassandraCluster() return startCassandra() elif Storage.storage == Ramcloud: confRamcloud() return startRamcloud() return startStorageThreads(master_cmd, server_cmd, numa0Args, numa1Args, masterObs, storageObs, javaHome)
def startMBServer(observers): Microbench.rsyncBuild() path = "" params = '-t {0} -n {1} -s {2} '.format(Microbench.threads, Microbench.numColumns, Microbench.scaling) if Storage.storage == TellStore: cmd = '{0}/watch/microbench/mbserver_tell {1}'.format( TellStore.builddir, params) cmd += '-c "{0}" --storage "{1}" --network-threads {2} -m {3}'.format( TellStore.getCommitManagerAddress(), TellStore.getServerList(), Microbench.networkThreads, Microbench.infinioBatch) elif Storage.storage == Kudu: cmd = '{0}/watch/microbench/mbserver_kudu {1}'.format( TellStore.builddir, params) cmd += '-c {0}'.format(Storage.master) elif Storage.storage == Cassandra: Microbench.rsyncJars() path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd = 'java -jar {0}/mbserver_cassandra.jar {1}'.format( Microbench.javaDir, params) cmd += getNodes(Storage.servers, " -cn ") elif Storage.storage == Hbase: path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd = 'java -jar {0}/mbserver_hbase.jar {1}'.format( Microbench.javaDir, params) cmd += '-hm {0}'.format(Storage.master) cmd += '-zm {0}'.format(Storage.master) elif Storage.storage == Ramcloud: cmd = '{0}/watch/microbench/mbserver_ramcloud {1}'.format( TellStore.builddir, params) cmd += '-c main -l "infrc:host={0}-infrc,port=11100" -x {1}'.format( Storage.master, len(Storage.servers) + len(Storage.servers1)) client0 = ThreadedClients(Microbench.servers0, "{0}numactl -m 0 -N 0 {1}".format(path, cmd), observers=observers) client1 = ThreadedClients(Microbench.servers1, "{0}numactl -m 1 -N 1 {1} -p 8712".format( path, cmd), observers=observers) client0.start() client1.start() return [client0, client1]
def startMBServer(observers): Microbench.rsyncBuild() path = "" params = '-t {0} -n {1} -s {2} '.format(Microbench.threads, Microbench.numColumns, Microbench.scaling) if Storage.storage == TellStore: cmd = '{0}/watch/microbench/mbserver_tell {1}'.format(TellStore.builddir, params) cmd += '-c "{0}" --storage "{1}" --network-threads {2} -m {3}'.format(TellStore.getCommitManagerAddress(), TellStore.getServerList(), Microbench.networkThreads, Microbench.infinioBatch) elif Storage.storage == Kudu: cmd = '{0}/watch/microbench/mbserver_kudu {1}'.format(TellStore.builddir, params) cmd += '-c {0}'.format(Storage.master) elif Storage.storage == Cassandra: Microbench.rsyncJars() path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd ='java -jar {0}/mbserver_cassandra.jar {1}'.format(Microbench.javaDir, params) cmd += getNodes(Storage.servers, " -cn ") elif Storage.storage == Hbase: path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd = 'java -jar {0}/mbserver_hbase.jar {1}'.format(Microbench.javaDir, params) cmd += '-hm {0}'.format(Storage.master) cmd += '-zm {0}'.format(Storage.master) elif Storage.storage == Ramcloud: cmd = '{0}/watch/microbench/mbserver_ramcloud {1}'.format(TellStore.builddir, params) cmd += '-c main -l "infrc:host={0}-infrc,port=11100" -x {1}'.format(Storage.master, len(Storage.servers) + len(Storage.servers1)) client0 = ThreadedClients(Microbench.servers0, "{0}numactl -m 0 -N 0 {1}".format(path, cmd), observers=observers) client1 = ThreadedClients(Microbench.servers1, "{0}numactl -m 1 -N 1 {1} -p 8712".format(path, cmd), observers=observers) client0.start() client1.start() return [client0, client1]
def startStorage(): # to be on the safe side, first unmount the drives if they are still mounted for whatever reason unmount_memfs() javaHome = "" masterObs = "" storageObs = "" numa0Args = "" numa1Args = "" if Storage.storage == Kudu: confKudu() master_cmd = '/mnt/local/tell/kudu_install/bin/kudu-master --fs_data_dirs={0} --fs_wal_dir={0} --block_manager=file'.format(Kudu.master_dir) server_cmd = '/mnt/local/tell/kudu_install/bin/kudu-tserver' numa0Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1}".format(Kudu.tserver_dir, Storage.master) numa1Args = "--fs_data_dirs={0} --fs_wal_dir={0} --block_cache_capacity_mb 51200 --tserver_master_addrs {1} --rpc_bind_addresses=0.0.0.0:7049 --webserver_port=8049".format(Kudu.tserver_dir1, Storage.master) elif Storage.storage == TellStore: TellStore.rsyncBuild() master_cmd = "{0}/commitmanager/server/commitmanagerd".format(TellStore.builddir) server_cmd = "{0}/tellstore/server/tellstored-{1} -l INFO --scan-threads {2} --network-threads 1 --gc-interval {5} -m {3} -c {4}".format(TellStore.builddir, TellStore.approach, TellStore.scanThreads, TellStore.memorysize, TellStore.hashmapsize, TellStore.gcInterval) numa1Args = '-p 7240' storageObs = "Storage ready" elif Storage.storage == Hadoop: confHdfs() return startHdfs() elif Storage.storage == Hbase: confHbaseCluster() return startHbaseThreads() elif Storage.storage == Cassandra: confCassandraCluster() return startCassandra() elif Storage.storage == Ramcloud: confRamcloud() return startRamcloud() elif Storage.storage == Memsql: return startMemsql() return startStorageThreads(master_cmd, server_cmd, numa0Args, numa1Args, masterObs, storageObs, javaHome)
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import General import time server_cmd = 'cd {0}/watch/tpch; {0}/watch/tpch/tpch_server -s "{1}" -c "{2}"'.format( Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) client_cmd = '{0}/watch/tpch/tpch_client -H "{1}" -s {2} -P'.format( Tpch.builddir, Tpch.server, Tpch.scaling) server = ThreadedClients([Tpch.server], server_cmd) server.start() time.sleep(5) client = ThreadedClients([Tpch.client], client_cmd) client.start() client.join() print "Population done, please hit Ctr+C to finish" server.join()
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import Kudu from ServerConfig import General import time cmd = "" if Tpch.storage == Kudu: cmd = '{0}/watch/tpch/tpch_server -s "{1}" --network-threads 8 -P {2} -k'.format(Tpch.builddir, Kudu.master, len(Kudu.tservers)*4) elif Tpch.storage == TellStore: cmd = '{0}/watch/tpch/tpch_server -s "{1}" --network-threads 4 -c "{2}"'.format(Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) server0 = ThreadedClients(Tpch.servers0, "numactl -m 0 -N 0 {0}".format(cmd)) server1 = ThreadedClients(Tpch.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd)) server0.start() server1.start() server0.join() server1.join()
with open(sparkDefault, 'w+') as f: f.write('spark.driver.extraClassPath {0}\n'.format(classpath)) f.write('spark.executor.extraClassPath {0}\n'.format(classpath)) f.write('spark.serializer org.apache.spark.serializer.KryoSerializer\n') f.write('spark.driver.memory 10g\n') f.write('spark.executor.memory 90g\n') f.write('spark.local.dir {0}\n'.format(Spark.tmpDir)) f.write('spark.executor.cores {0}\n'.format(Spark.numCores)) # TellStore f.write('spark.sql.tell.numPartitions {0}\n'.format(Spark.tellPartitions)) #f.write('spark.sql.tell.chunkSizeSmall 104857600\n') numChunks = (len(TellStore.servers) + len(TellStore.servers1)) * Spark.numCores f.write('spark.sql.tell.chunkSizeBig {0}\n'.format(((TellStore.scanMemory // numChunks) // 8) * 8)) f.write('spark.sql.tell.chunkCount {0}\n'.format(numChunks)) f.write('spark.sql.tell.commitmanager {0}\n'.format(TellStore.getCommitManagerAddress())) f.write('spark.sql.tell.storagemanager {0}\n'.format(TellStore.getServerList())) with open(sparkEnv, 'w+') as f: f.write('export JAVA_HOME={0}\n'.format(Spark.javahome)) f.write('export LD_LIBRARY_PATH={0}\n'.format(Spark.telljava)) tmpDirCommand = lambda host: os.system("ssh root@{0} 'rm -rf {1}; mkdir {1}'".format(host, Spark.tmpDir)) configCopyCommand = lambda host: os.system('scp {0} {1} root@{2}:{3}/conf/'.format(sparkEnv, sparkDefault, host, Spark.sparkdir)) jarCopyCommand = lambda host: os.system('scp {0}/*.jar root@{1}:{0}'.format(Spark.jarsDir, host)) tmpDirCommand(Spark.master) configCopyCommand(Spark.master) jarCopyCommand(Spark.master) for host in Spark.slaves: tmpDirCommand(host) configCopyCommand(host)
#!/usr/bin/env python from argparse import ArgumentParser import os from ServerConfig import Client from ServerConfig import General from ServerConfig import Storage from ServerConfig import Kudu from ServerConfig import Tpch from ServerConfig import TellStore storageAddr = Kudu.master + " -k" if Storage.storage == TellStore: storageAddr = TellStore.getServerList() masterAddr = TellStore.getCommitManagerAddress() cmd = '{0}/watch/tpcc/tpch -S "{1}" -d {2}/{3} -C "{4}"'.format(TellStore.builddir, storageAddr, Tpch.dbgenFiles, Tpch.scaling, masterAddr) print cmd exit(os.system(cmd))
with open(sparkDefault, "w+") as f: f.write("spark.driver.extraClassPath {0}\n".format(classpath)) f.write("spark.executor.extraClassPath {0}\n".format(classpath)) f.write("spark.serializer org.apache.spark.serializer.KryoSerializer\n") f.write("spark.driver.memory 10g\n") f.write("spark.executor.memory 90g\n") f.write("spark.local.dir {0}\n".format(Spark.tmpDir)) f.write("spark.executor.cores {0}\n".format(Spark.numCores)) # TellStore f.write("spark.sql.tell.numPartitions {0}\n".format(Spark.tellPartitions)) # f.write('spark.sql.tell.chunkSizeSmall 104857600\n') numChunks = (len(TellStore.servers) + len(TellStore.servers1)) * Spark.numCores f.write("spark.sql.tell.chunkSizeBig {0}\n".format(((TellStore.scanMemory // numChunks) // 8) * 8)) f.write("spark.sql.tell.chunkCount {0}\n".format(numChunks)) f.write("spark.sql.tell.commitmanager {0}\n".format(TellStore.getCommitManagerAddress())) f.write("spark.sql.tell.storagemanager {0}\n".format(TellStore.getServerList())) with open(sparkEnv, "w+") as f: f.write("export JAVA_HOME={0}\n".format(Spark.javahome)) f.write("export LD_LIBRARY_PATH={0}\n".format(Spark.telljava)) tmpDirCommand = lambda host: os.system("ssh root@{0} 'rm -rf {1}; mkdir {1}'".format(host, Spark.tmpDir)) configCopyCommand = lambda host: os.system( "scp {0} {1} root@{2}:{3}/conf/".format(sparkEnv, sparkDefault, host, Spark.sparkdir) ) jarCopyCommand = lambda host: os.system("scp {0}/*.jar root@{1}:{0}".format(Spark.jarsDir, host)) tmpDirCommand(Spark.master) configCopyCommand(Spark.master) jarCopyCommand(Spark.master) for host in Spark.slaves:
def startAimServers(observers = []): Aim.rsyncBuild() numChunks = (len(Storage.servers) + len(Storage.servers1)) * Aim.numRTAClients * 16 chunkSize = ((TellStore.scanMemory // numChunks) // 8) * 8 serverExec = "" if Storage.storage == Kudu: serverExec = "aim_kudu -P {0} -s {1}".format((len(Storage.servers) + len(Storage.servers1)) * 2, Storage.master) elif Storage.storage == TellStore: serverExec = 'aim_server -M {0} -m {1} -c "{2}" -s "{3}" --processing-threads {4}'.format(numChunks, chunkSize, TellStore.getCommitManagerAddress(), TellStore.getServerList(), Aim.serverthreads) cmd = '{0}/watch/aim-benchmark/{3} -f {1} -b {2}'.format(Aim.builddir, Aim.schemaFile, Aim.batchSize, serverExec) client0 = ThreadedClients(Aim.sepservers0 + Aim.rtaservers0, "numactl -m 0 -N 0 {0}".format(cmd), root=True) client1 = ThreadedClients(Aim.sepservers1 + Aim.rtaservers1, "numactl -m 1 -N 1 {0} -p 8715 -u 8716".format(cmd), root=True) client0.start() client1.start() return [client0, client1]
def confNode(host, coordinator = False): print "\nCONFIGURING {0}".format(host) # node properties nodeProps = "{0}/etc/node.properties".format(Presto.prestodir) with open (nodeProps, 'w+') as f: f.write("node.environment=ethz\n") f.write("node.id=ffffffff-ffff-ffff-ffff-{0}\n".format(host)) f.write("node.data-dir={0}\n".format(Presto.datadir)) copyToHost([host], nodeProps) # jvm config jvmConf = "{0}/etc/jvm.config".format(Presto.prestodir) with open (jvmConf, 'w+') as f: f.write("-server\n") f.write("-Djava.library.path={0}\n".format(Java.telljava)) if Presto.debug: f.write('-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005\n') f.write("-Xmx{0}\n".format(Presto.jvmheap)) f.write("-XX:+UseG1GC\n") f.write("-XX:G1HeapRegionSize={0}\n".format(Presto.jvmheapregion)) f.write("-XX:+UseGCOverheadLimit\n") f.write("-XX:+ExplicitGCInvokesConcurrent\n") f.write("-XX:+HeapDumpOnOutOfMemoryError\n") f.write("-XX:OnOutOfMemoryError=kill -9 %p\n") copyToHost([host], jvmConf) # config properties confProps = "{0}/etc/config.properties".format(Presto.prestodir) with open (confProps, 'w+') as f: if (coordinator): f.write("coordinator=true\n") f.write("node-scheduler.include-coordinator=false\n") f.write("discovery-server.enabled=true\n") else: f.write("coordinator=false\n") f.write("http-server.http.port={0}\n".format(Presto.httpport)) f.write("query.max-memory={0}\n".format(Presto.querymaxmem)) f.write("query.max-memory-per-node={0}\n".format(Presto.querymaxnode)) f.write("discovery.uri=http://{0}:8080\n".format(Presto.coordinator)) f.write("node-scheduler.max-splits-per-node={0}\n".format(Presto.splitsPerMachine)) f.write("node-scheduler.max-pending-splits-per-node-per-task={0}\n".format(0)) copyToHost([host], confProps) # catalog: if Storage.storage == Hadoop: hiveCat = "{0}/etc/catalog/hive.properties".format(Presto.prestodir) with open (hiveCat, 'w+') as f: f.write("connector.name=hive-hadoop2\n") f.write("hive.metastore.uri=thrift://{0}:{1}\n".format(Hive.metastoreuri, Hive.metastoreport)) f.write("hive.metastore-timeout={0}\n".format(Hive.metastoretimeout)) copyToHost([host], hiveCat) elif Storage.storage == TellStore: tellCat = "{0}/etc/catalog/tell.properties".format(Presto.prestodir) numChunks = Presto.splitsPerMachine * TellStore.numServers() with open (tellCat, 'w+') as f: f.write('connector.name=tell\n') f.write('tell.commitManager={0}\n'.format(TellStore.getCommitManagerAddress())) f.write('tell.storages={0}\n'.format(TellStore.getServerList())) f.write('tell.numPartitions={0}\n'.format(Presto.splitsPerMachine * len(Presto.nodes))) f.write('tell.partitionShift={0}\n'.format(TellStore.scanShift)) f.write('tell.chunkCount={0}\n'.format(numChunks)) f.write('tell.chunkSize={0}\n'.format(((TellStore.scanMemory // numChunks) // 8) * 8)) copyToHost([host], tellCat) # log level logProps = "{0}/etc/log.properties".format(Presto.prestodir) f = open(logProps, 'w+') f.write("com.facebook.presto={0}\n".format(Presto.loglevel)) f.close() copyToHost([host], logProps) # tmp files for logging os.system("ssh root@{0} 'rm -rf {1}; mkdir {1}'".format(host, Presto.datadir))
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import General import time server_cmd = 'cd {0}/watch/tpch; {0}/watch/tpch/tpch_server -s "{1}" -c "{2}"'.format(Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) client_cmd = '{0}/watch/tpch/tpch_client -H "{1}" -s {2} -P'.format(Tpch.builddir, Tpch.server, Tpch.scaling) server = ThreadedClients([Tpch.server], server_cmd) server.start() time.sleep(5) client = ThreadedClients([Tpch.client], client_cmd) client.start() client.join() print "Population done, please hit Ctr+C to finish" server.join()