def startMBServer(observers): Microbench.rsyncBuild() path = "" params = '-t {0} -n {1} -s {2} '.format(Microbench.threads, Microbench.numColumns, Microbench.scaling) if Storage.storage == TellStore: cmd = '{0}/watch/microbench/mbserver_tell {1}'.format(TellStore.builddir, params) cmd += '-c "{0}" --storage "{1}" --network-threads {2} -m {3}'.format(TellStore.getCommitManagerAddress(), TellStore.getServerList(), Microbench.networkThreads, Microbench.infinioBatch) elif Storage.storage == Kudu: cmd = '{0}/watch/microbench/mbserver_kudu {1}'.format(TellStore.builddir, params) cmd += '-c {0}'.format(Storage.master) elif Storage.storage == Cassandra: Microbench.rsyncJars() path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd ='java -jar {0}/mbserver_cassandra.jar {1}'.format(Microbench.javaDir, params) cmd += getNodes(Storage.servers, " -cn ") elif Storage.storage == Hbase: path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd = 'java -jar {0}/mbserver_hbase.jar {1}'.format(Microbench.javaDir, params) cmd += '-hm {0}'.format(Storage.master) cmd += '-zm {0}'.format(Storage.master) elif Storage.storage == Ramcloud: cmd = '{0}/watch/microbench/mbserver_ramcloud {1}'.format(TellStore.builddir, params) cmd += '-c main -l "infrc:host={0}-infrc,port=11100" -x {1}'.format(Storage.master, len(Storage.servers) + len(Storage.servers1)) client0 = ThreadedClients(Microbench.servers0, "{0}numactl -m 0 -N 0 {1}".format(path, cmd), observers=observers) client1 = ThreadedClients(Microbench.servers1, "{0}numactl -m 1 -N 1 {1} -p 8712".format(path, cmd), observers=observers) client0.start() client1.start() return [client0, client1]
def startZk(): observer = Observer("binding to port") zk_cmd = 'numactl -m 0 -N 0 {0}/bin/zkServer.sh start-foreground'.format(Zookeeper.zkdir) zkClient = ThreadedClients([Storage.master], zk_cmd, root=True, observers=[observer]) zkClient.start() observer.waitFor(1) return zkClient
def stopMemsql(): # sudo ./memsql-ops/memsql-ops/memsql-ops stop print "Stopping all active Memsql nodes" del_mnodes_cmd = "{0}/memsql-ops/memsql-ops memsql-delete --all --delete-without-prompting".format(Memsql.msqlopsdir) del_memsql_nodes = ThreadedClients([Storage.master], del_mnodes_cmd, root=True) del_memsql_nodes.start() del_memsql_nodes.join()
def createTopic(tName, nParts, ePath, server): # TODO Change zk default port? ct_cmd = '{0}/bin/kafka-topics.sh --create --partitions {1} --zookeeper {2}:2181 --replication-factor 1 --topic {3}' topicClient = ThreadedClients([Config.server], ct_cmd.format(ePath, nParts, server, tName), root=True) topicClient.start() topicClient.join()
def startPresto(): #start_presto_cmd = "'JAVA_HOME={1} PATH={1}/bin:$PATH {0}/bin/launcher run'".format(Presto.prestodir, General.javahome) start_presto_cmd = "PATH={0}/bin:$PATH {1}/bin/launcher run".format(General.javahome, Presto.prestodir) coordinator = ThreadedClients([Presto.coordinator], start_presto_cmd) coordinator.start() time.sleep(5) workers = ThreadedClients(Presto.nodes, start_presto_cmd) workers.start() coordinator.join() workers.join()
def execApp(approach, nMiss): #jParams = "kafka={0}:9092 zk={0}:2181 missing={1} > {2}.{1}.log" jParams = "kafka={0}:9092 zk={0}:2181 missing={1}" # loop for all operations jPa = jParams.format(Config.server, nMiss) print javaCmd.format(Config.jarpath, approach, jPa) jC = ThreadedClients([Config.server], javaCmd.format(Config.jarpath, approach, jPa)) jC.start() jC.join() time.sleep(3) print doneStr.format(approach + "-" + str(nMiss))
def stopSetup(): # stop kafka stopKafkaClient = ThreadedClients([Config.server], "{0}/bin/kafka-server-stop.sh".format( Config.exec_path)) stopKafkaClient.start() stopKafkaClient.join() # stop zk stopZkClient = ThreadedClients([Config.server], "{0}/bin/zookeeper-server-stop.sh".format( Config.exec_path)) stopZkClient.start() stopZkClient.join()
def startKafka(): confKafka() # Start Kafka observer = Observer("started (kafka.server.KafkaServer)") k_cmd = '{0}/bin/kafka-server-start.sh {0}/config/{1}'.format( Config.exec_path, Config.k_config) kClient = ThreadedClients([Config.server], k_cmd, root=True, observers=[observer]) kClient.start() observer.waitFor(1) return kClient
def startZk(): confZk() observer = Observer("binding to port") zk_cmd = '{0}/bin/zookeeper-server-start.sh {0}/config/{1}'.format( Config.exec_path, Config.zk_config) zkClient = ThreadedClients([Config.server], zk_cmd, root=True, observers=[observer]) print zk_cmd zkClient.start() observer.waitFor(1) return zkClient
def cleanDirs(): # Clean zk dirs dClient = ThreadedClients([Config.server], "rm -rf {0}".format(Config.zk_dir), root=True) dClient.start() dClient.join() # Clean kafka dirs dClient2 = ThreadedClients([Config.server], "rm -rf {0}".format(Config.k_dir), root=True) dClient2.start() dClient2.join()
def startMBServer(observers): Microbench.rsyncBuild() path = "" params = '-t {0} -n {1} -s {2} '.format(Microbench.threads, Microbench.numColumns, Microbench.scaling) if Storage.storage == TellStore: cmd = '{0}/watch/microbench/mbserver_tell {1}'.format( TellStore.builddir, params) cmd += '-c "{0}" --storage "{1}" --network-threads {2} -m {3}'.format( TellStore.getCommitManagerAddress(), TellStore.getServerList(), Microbench.networkThreads, Microbench.infinioBatch) elif Storage.storage == Kudu: cmd = '{0}/watch/microbench/mbserver_kudu {1}'.format( TellStore.builddir, params) cmd += '-c {0}'.format(Storage.master) elif Storage.storage == Cassandra: Microbench.rsyncJars() path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd = 'java -jar {0}/mbserver_cassandra.jar {1}'.format( Microbench.javaDir, params) cmd += getNodes(Storage.servers, " -cn ") elif Storage.storage == Hbase: path = "PATH={0}/bin:$PATH ".format(General.javahome) cmd = 'java -jar {0}/mbserver_hbase.jar {1}'.format( Microbench.javaDir, params) cmd += '-hm {0}'.format(Storage.master) cmd += '-zm {0}'.format(Storage.master) elif Storage.storage == Ramcloud: cmd = '{0}/watch/microbench/mbserver_ramcloud {1}'.format( TellStore.builddir, params) cmd += '-c main -l "infrc:host={0}-infrc,port=11100" -x {1}'.format( Storage.master, len(Storage.servers) + len(Storage.servers1)) client0 = ThreadedClients(Microbench.servers0, "{0}numactl -m 0 -N 0 {1}".format(path, cmd), observers=observers) client1 = ThreadedClients(Microbench.servers1, "{0}numactl -m 1 -N 1 {1} -p 8712".format( path, cmd), observers=observers) client0.start() client1.start() return [client0, client1]
def confZk(): zooCfg = '{0}/conf/zoo.cfg'.format(Zookeeper.zkdir) with open(zooCfg, 'w+') as f: f.write("maxClientCnxns={0}\n".format(Zookeeper.maxclients)) f.write("tickTime={0}\n".format(Zookeeper.ticktime)) f.write("dataDir={0}\n".format(Zookeeper.datadir)) f.write("clientPort={0}\n".format(Zookeeper.clientport)) deleteClient = ThreadedClients([Storage.master], "rm -rf {0}".format(Zookeeper.datadir), root=True) deleteClient.start() deleteClient.join() copyClient = ThreadedClients([Storage.master], "mkdir -p {0}".format(Zookeeper.datadir), root=True) copyClient.start() copyClient.join() copyToHost([Storage.master], zooCfg)
def startAimServers(observers = []): Aim.rsyncBuild() numChunks = (len(Storage.servers) + len(Storage.servers1)) * Aim.numRTAClients * 16 chunkSize = ((TellStore.scanMemory // numChunks) // 8) * 8 serverExec = "" if Storage.storage == Kudu: serverExec = "aim_kudu -P {0} -s {1}".format((len(Storage.servers) + len(Storage.servers1)) * 2, Storage.master) elif Storage.storage == TellStore: serverExec = 'aim_server -M {0} -m {1} -c "{2}" -s "{3}" --processing-threads {4}'.format(numChunks, chunkSize, TellStore.getCommitManagerAddress(), TellStore.getServerList(), Aim.serverthreads) cmd = '{0}/watch/aim-benchmark/{3} -f {1} -b {2}'.format(Aim.builddir, Aim.schemaFile, Aim.batchSize, serverExec) client0 = ThreadedClients(Aim.sepservers0 + Aim.rtaservers0, "numactl -m 0 -N 0 {0}".format(cmd), root=True) client1 = ThreadedClients(Aim.sepservers1 + Aim.rtaservers1, "numactl -m 1 -N 1 {0} -p 8715 -u 8716".format(cmd), root=True) client0.start() client1.start() return [client0, client1]
def startCassandra(): observerString = "No host ID found" start_cas_cmd = "{0}/bin/cassandra -f".format(Cassandra.casdir) javaHome = "JAVA_HOME={0}".format(General.javahome) # startup seed node obs = Observer(observerString) seedClient = ThreadedClients([Storage.servers[0]], "{1} numactl -m 0 -N 0 {0}".format(start_cas_cmd, javaHome), observers=[obs]) seedClient.start() obs.waitFor(1) nodeClients = [] # startup remaining nodes on NUMA 0 if len(Storage.servers) > 1: for server in Storage.servers[1:]: obs = Observer(observerString) nodeClient = ThreadedClients([server], "{1} numactl -m 0 -N 0 {0}".format(start_cas_cmd, javaHome), observers=[obs]) nodeClient.start() obs.waitFor(1) nodeClients = nodeClients + [nodeClient] if len(Storage.servers1) > 0: # startup nodes on NUMA 1 for server in Storage.servers1: obs = Observer(observerString) nodeClient = ThreadedClients([server], '{2} CASSANDRA_CONF={1}/conf1 numactl -m 1 -N 1 {0} -Dcassandra.logdir={3}'.format(start_cas_cmd, Cassandra.casdir, javaHome, Cassandra.logdir1), observers=[obs]) nodeClient.start() obs.waitFor(1) nodeClients = nodeClients + [nodeClient] return nodeClients + [seedClient]
def startStorageThreads(master_cmd, server_cmd, numa0Args, numa1Args, masterObserverString="", serverObserverString="", envVars = ""): masterObservers = [] if len(masterObserverString) > 0: masterObservers = [Observer(masterObserverString)] # if Storage.storage == Hbase: # masterObservers.append(Observer("Master has completed initialization")) mclient = ThreadedClients([Storage.master], "{0}numactl -m 0 -N 0 {1}".format(envVars, master_cmd), observers=masterObservers) mclient.start() if len(masterObservers) > 0: masterObservers[0].waitFor(1) else: time.sleep(2) storageObservers = [] if len(serverObserverString) > 0: storageObservers = [Observer(serverObserverString)] tclient = ThreadedClients(Storage.servers, "{0}numactl -m 0 -N 0 {1} {2}".format(envVars, server_cmd, numa0Args), observers=storageObservers) tclient.start() tclient1 = ThreadedClients(Storage.servers1, '{0}numactl -m 1 -N 1 {1} {2}'.format(envVars, server_cmd, numa1Args), observers=storageObservers) tclient1.start() if len(storageObservers) > 0: storageObservers[0].waitFor(len(Storage.servers) + len(Storage.servers1)) else: time.sleep(2) if Storage.storage == Hbase: time.sleep(5) return [tclient, tclient1, mclient]
def populate(nMiss): checkTopics() print doneStr.format("CREATION") time.sleep(3) #params = "kafka={0}:9092 zk={0}:2181 missing={1} sf={2} tuples={3} pcompletion={4} psuccess={5} bid_ratio={6} > {7}.{1}.log" params = "kafka={0}:9092 zk={0}:2181 missing={1} sf={2} tuples={3} pcompletion={4} psuccess={5} bid_ratio={6}" #aPa = params.format(Config.server, Config.missing, Config.sf, Config.tuples, Config.pcompletion, Config.psuccess, Config.bratio, "auctionProducer") aPa = params.format(Config.server, nMiss, Config.sf, Config.tuples, Config.pcompletion, Config.psuccess, Config.bratio) #bPa = params.format(Config.server, Config.missing, Config.sf, Config.tuples, Config.pcompletion, Config.psuccess, Config.bratio, "bidProducer") bPa = params.format(Config.server, nMiss, Config.sf, Config.tuples, Config.pcompletion, Config.psuccess, Config.bratio) # populating auctions aC = ThreadedClients([Config.server], javaCmd.format(Config.jarpath, Config.aproducer, aPa)) aC.start() aC.join() time.sleep(3) # populating bids bC = ThreadedClients([Config.server], javaCmd.format(Config.jarpath, Config.bproducer, bPa)) bC.start() bC.join() print doneStr.format("POPULATION") time.sleep(3)
def confRamcloud(): deleteClient = ThreadedClients(Storage.servers, "rm -rf {0}".format(Ramcloud.backupdir), root=True) deleteClient.start() deleteClient.join() deleteClient = ThreadedClients(Storage.servers1, "rm -rf {0}".format(Ramcloud.backupdir1), root=True) deleteClient.start() deleteClient.join() copyClient = ThreadedClients(Storage.servers, "mkdir -p {0}".format(Ramcloud.backupdir), root=True) copyClient.start() copyClient.join() copyClient = ThreadedClients(Storage.servers1, "mkdir -p {0}".format(Ramcloud.backupdir1), root=True) copyClient.start() copyClient.join() confZk()
def confKudu(): if Kudu.clean: rmcommand = 'rm -rf {0}/*' master_client = ThreadedClients([Storage.master], rmcommand.format(Kudu.master_dir), root=True) master_client.start() tserver_client = ThreadedClients(Storage.servers, rmcommand.format(Kudu.tserver_dir), root=True) tserver_client.start() tserver_client1 = ThreadedClients(Storage.servers1, rmcommand.format(Kudu.tserver_dir1), root=True) tserver_client1.start() master_client.join() tserver_client.join() tserver_client1.join()
def startRamcloud(): zkClient = startZk() master_cmd = "LD_LIBRARY_PATH={3} numactl -m 0 -N 0 {0}/coordinator -C infrc:host={1}-infrc,port=11100 -x zk:{1}:{2} --timeout {4}".format(Ramcloud.ramclouddir, Storage.master, Zookeeper.clientport, Ramcloud.boost_lib, Ramcloud.timeout) masterObs = Observer("Memory usage now") masterClient = ThreadedClients([Storage.master], master_cmd, observers=[masterObs]) masterClient.start() masterObs.waitFor(1) # create observer list storageObs = [] for i in range(len(Storage.servers) + len(Storage.servers1)): storageObs = storageObs + [Observer("Server " + str(i+1) + ".0 is up")] nodeClients = [] storage_cmd = "LD_LIBRARY_PATH={7} numactl -m 0 -N 0 {0}/server -L infrc:host={3}-infrc,port={4} -x zk:{1}:{2} --totalMasterMemory {5} -f {6} --segmentFrames 10000 -r 0 --timeout {8}" storage_cmd = storage_cmd.format(Ramcloud.ramclouddir, Storage.master, Zookeeper.clientport, "{0}", Ramcloud.storageport, Ramcloud.memorysize, Ramcloud.backupfile, Ramcloud.boost_lib, Ramcloud.timeout) # startup nodes on NUMA 0 for server in Storage.servers: nodeClient = ThreadedClients([server], storage_cmd.format(server), observers=storageObs) nodeClient.start() nodeClients = nodeClients + [nodeClient] # startup nodes on NUMA 1 if len(Storage.servers1) > 0: storage_cmd = "LD_LIBRARY_PATH={7} numactl -m 1 -N 1 {0}/server -L infrc:host={3}-infrc,port={4} -x zk:{1}:{2} --totalMasterMemory {5} -f {6} --segmentFrames 10000 -r 0 --timeout {8}" storage_cmd = storage_cmd.format(Ramcloud.ramclouddir, Storage.master, Zookeeper.clientport, "{0}", Ramcloud.storageport1, Ramcloud.memorysize, Ramcloud.backupfile1, Ramcloud.boost_lib, Ramcloud.timeout) for server in Storage.servers1: nodeClient = ThreadedClients([server], storage_cmd.format(server)) nodeClient.start() nodeClients = nodeClients + [nodeClient] # wait until all storages are up (the message is displayed at least once at at least one storage server) for storageOb in storageObs: storageOb.waitFor(1) return nodeClients + [masterClient, zkClient]
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import Kudu from ServerConfig import General import time cmd = "" if Tpch.storage == Kudu: cmd = '{0}/watch/tpch/tpch_server -s "{1}" --network-threads 8 -P {2} -k'.format( Tpch.builddir, Kudu.master, len(Kudu.tservers) * 4) elif Tpch.storage == TellStore: cmd = '{0}/watch/tpch/tpch_server -s "{1}" --network-threads 4 -c "{2}"'.format( Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) server0 = ThreadedClients(Tpch.servers0, "numactl -m 0 -N 0 {0}".format(cmd)) server1 = ThreadedClients(Tpch.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd)) server0.start() server1.start() server0.join() server1.join()
def confCassandraCluster(): os.system("mkdir -p {0}/conf1".format(Cassandra.casdir)) dirClient = ThreadedClients(Storage.servers1, "mkdir -p {0}/conf1".format(Cassandra.casdir)) dirClient.start() dirClient.join() # copy all conf files over, cassandra.yaml will be overwritten later os.system("cp -a {0}/conf/* {0}/conf1/".format(Cassandra.casdir)) copyClient = ThreadedClients(Storage.servers1, "cp -a {0}/conf/* {0}/conf1/".format(Cassandra.casdir)) copyClient.start() copyClient.join() # we also have to change cassandra-env.sh f = open('{0}/conf/cassandra-env.sh'.format(Cassandra.casdir), 'r') templateEnv = f.read() f.close() templateEnv = templateEnv.replace('JMX_PORT="7199"', 'JMX_PORT="7198"') cassandraEnv = '{0}/conf1/cassandra-env.sh'.format(Cassandra.casdir) with open (cassandraEnv, 'w') as f: f.write(templateEnv) f.close() copyToHost(Storage.servers1, cassandraEnv) for numaNode in [0,1]: servers = Storage.servers if numaNode == 0 else Storage.servers1 datadir = Cassandra.datadir if numaNode == 0 else Cassandra.datadir1 logdir = Cassandra.logdir if numaNode == 0 else Cassandra.logdir1 nativeport = Cassandra.nativeport if numaNode == 0 else Cassandra.nativeport1 rpcport = Cassandra.rpcport if numaNode == 0 else Cassandra.rpcport1 storageport = Cassandra.storageport if numaNode == 0 else Cassandra.storageport1 sslport = Cassandra.sslport if numaNode == 0 else Cassandra.sslport1 cassandraConf = '{0}/conf/cassandra.yaml' if numaNode == 0 else '{0}/conf1/cassandra.yaml' cassandraConf = cassandraConf.format(Cassandra.casdir) if len(servers) == 0: continue for host in servers: f = open('cassandra.yaml.template', 'r') templateconf = f.read() f.close() templateconf = templateconf.replace("casseeds", "\"" + Storage.servers[0] + "\"") templateconf = templateconf.replace("caslistenaddr", host) templateconf = templateconf.replace("casdatadir", datadir) templateconf = templateconf.replace("caslogdir", logdir) templateconf = templateconf.replace("casnativeport", nativeport) templateconf = templateconf.replace("casrpcport", rpcport) templateconf = templateconf.replace("casstorageport", storageport) templateconf = templateconf.replace("cassslport", sslport) with open(cassandraConf, 'w') as f: f.write(templateconf) f.close() copyToHost([host], cassandraConf) mkClients = ThreadedClients(servers, "mkdir -p {0}".format(datadir), root=True) mkClients.start() mkClients.join() mntClients = ThreadedClients(servers, "mount -t tmpfs -o size={0}G tmpfs {1}".format(Cassandra.datadirSz, datadir), root=True) mntClients.start() mntClients.join() mkClients = ThreadedClients(servers , "mkdir -p {0}".format(logdir), root=True) mkClients.start() mkClients.join()
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import General import time server_cmd = 'cd {0}/watch/tpch; {0}/watch/tpch/tpch_server -s "{1}" -c "{2}"'.format( Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) client_cmd = '{0}/watch/tpch/tpch_client -H "{1}" -s {2} -P'.format( Tpch.builddir, Tpch.server, Tpch.scaling) server = ThreadedClients([Tpch.server], server_cmd) server.start() time.sleep(5) client = ThreadedClients([Tpch.client], client_cmd) client.start() client.join() print "Population done, please hit Ctr+C to finish" server.join()
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import Kudu from ServerConfig import General import time cmd = "" if Tpch.storage == Kudu: cmd = '{0}/watch/tpch/tpch_server -s "{1}" --network-threads 8 -P {2} -k'.format(Tpch.builddir, Kudu.master, len(Kudu.tservers)*4) elif Tpch.storage == TellStore: cmd = '{0}/watch/tpch/tpch_server -s "{1}" --network-threads 4 -c "{2}"'.format(Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) server0 = ThreadedClients(Tpch.servers0, "numactl -m 0 -N 0 {0}".format(cmd)) server1 = ThreadedClients(Tpch.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd)) server0.start() server1.start() server0.join() server1.join()
def confHdfs(): # mount tmpfs for master and servers on numa 0 mkClients = ThreadedClients([Storage.master] + Storage.servers, "mkdir -p {0}; mount -t tmpfs -o size={1}G tmpfs {0}".format(Hadoop.datadir, Hadoop.datadirSz), root=True) mkClients.start() mkClients.join() # mount tmpfs for servers on numa 1 mkClients = ThreadedClients(Storage.servers1, "mkdir -p {0}; mount -t tmpfs -o size={1}G tmpfs {0}".format(Hadoop.datadir1, Hadoop.datadirSz), root=True) mkClients.start() mkClients.join() # modify core-site.xml coreSiteXml = '{0}/etc/hadoop/core-site.xml'.format(Hadoop.hadoopdir) with open(coreSiteXml, 'w+') as f: f.write("<configuration>\n") f.write(xmlProp("fs.default.name", "hdfs://{0}:{1}".format(Storage.master, Hadoop.hdfsport))) f.write(xmlProp("hadoop.tmp.dir", Hadoop.datadir)) f.write(xmlProp("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem")) f.write(xmlProp("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem")) f.write("</configuration>") # hadoop_env.sh hadoopEnv = '{0}/etc/hadoop/hadoop-env.sh'.format(Hadoop.hadoopdir) with open(hadoopEnv, 'w+') as f: f.write('export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT $HADOOP_NAMENODE_OPTS"\n') f.write('HADOOP_JOBTRACKER_OPTS="-Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA $HADOOP_JOBTRACKER_OPTS"\n') f.write('HADOOP_TASKTRACKER_OPTS="-Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console $HADOOP_TASKTRACKER_OPTS"\n') f.write('HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,DRFAS $HADOOP_DATANODE_OPTS"\n') f.write('export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT $HADOOP_SECONDARYNAMENODE_OPTS"\n') f.write('export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_CLIENT_OPTS"\n') f.write('export HADOOP_CLIENT_OPTS="-Xmx2048m $HADOOP_CLIENT_OPTS"\n') f.write('export HADOOP_SECURE_DN_USER=\n') f.write("export JAVA_HOME={0}\n".format(General.javahome)) f.write("export HADOOP_LOG_DIR={0}\n".format(Hadoop.datadir)) f.write("export HADOOP_SECURE_DN_LOG_DIR={0}\n".format(Hadoop.datadir)) f.write("export HADOOP_CONF_DIR={0}/etc/hadoop/\n".format(Hadoop.hadoopdir)) # hdfs-site.xml hdfsSiteXml = '{0}/etc/hadoop/hdfs-site.xml'.format(Hadoop.hadoopdir) with open(hdfsSiteXml, 'w+') as f: f.write("<configuration>\n") f.write(xmlProp("dfs.replication", Hadoop.dfsreplication)) f.write(xmlProp("dfs.permissions", "true")) f.write(xmlProp("dfs.namenode.rpc-address", "{0}:{1}".format(Storage.master, Hadoop.hdfsport))) f.write("</configuration>") copyToHost(Storage.servers + Storage.servers1 + [Storage.master], coreSiteXml) copyToHost(Storage.servers + Storage.servers1 + [Storage.master], hadoopEnv) copyToHost(Storage.servers + Storage.servers1 + [Storage.master], hdfsSiteXml) # master file - probably not used anymore as we do not use start-dfs.sh masterFile = '{0}/etc/hadoop/masters'.format(Hadoop.hadoopdir) with open(masterFile, "w+") as f: f.write(Storage.master) # slaves file - probably not used anymore as we do not use start-dfs.sh slavesFile = '{0}/etc/hadoop/slaves'.format(Hadoop.hadoopdir) with open(slavesFile, 'w+') as f: for host in (Storage.servers + Storage.servers1): f.write(host + "\n") copyToHost([Storage.master], masterFile) copyToHost([Storage.master], slavesFile) # format namenode nn_format_cmd = "numactl -m 0 -N 0 {0}/bin/hadoop namenode -format".format(Hadoop.hadoopdir) nnFormatClients = ThreadedClients([Storage.master], nn_format_cmd, root=True) nnFormatClients.start() nnFormatClients.join()
with open(sparkEnv, 'w+') as f: f.write('export JAVA_HOME={0}\n'.format(Spark.javahome)) f.write('export LD_LIBRARY_PATH={0}\n'.format(Spark.telljava)) tmpDirCommand = lambda host: os.system("ssh root@{0} 'rm -rf {1}; mkdir {1}'".format(host, Spark.tmpDir)) configCopyCommand = lambda host: os.system('scp {0} {1} root@{2}:{3}/conf/'.format(sparkEnv, sparkDefault, host, Spark.sparkdir)) jarCopyCommand = lambda host: os.system('scp {0}/*.jar root@{1}:{0}'.format(Spark.jarsDir, host)) tmpDirCommand(Spark.master) configCopyCommand(Spark.master) jarCopyCommand(Spark.master) for host in Spark.slaves: tmpDirCommand(host) configCopyCommand(host) jarCopyCommand(host) rmClients = ThreadedClients([Spark.master] + Spark.slaves, "rm -rf {0}/work".format(Spark.sparkdir), root=True) rmClients.start() rmClients.join() print master_cmd master = ThreadedClients([Spark.master], master_cmd, root=True) master.start() master.join() time.sleep(2) print slave_cmd slave = ThreadedClients(Spark.slaves, slave_cmd, root=True) slave.start() slave.join()
#!/usr/bin/env python from argparse import ArgumentParser from threaded_ssh import ThreadedClients from ServerConfig import Tpch from ServerConfig import TellStore from ServerConfig import General import time server_cmd = 'cd {0}/watch/tpch; {0}/watch/tpch/tpch_server -s "{1}" -c "{2}"'.format(Tpch.builddir, TellStore.getServerList(), TellStore.getCommitManagerAddress()) client_cmd = '{0}/watch/tpch/tpch_client -H "{1}" -s {2} -P'.format(Tpch.builddir, Tpch.server, Tpch.scaling) server = ThreadedClients([Tpch.server], server_cmd) server.start() time.sleep(5) client = ThreadedClients([Tpch.client], client_cmd) client.start() client.join() print "Population done, please hit Ctr+C to finish" server.join()
def startMemsql(): # start memsql-ops as sudo ops_cmd = "numactl -m 0 -N 0 {0}/memsql-ops/memsql-ops start --port {1} -u root --foreground".format(Memsql.msqlopsdir, Memsql.msqlopsport) observer = Observer("memsql_platform.jobs.engine: Ready.") # start master master_ops = ThreadedClients([Storage.master], ops_cmd, root=True, observers=[observer]) master_ops.start() observer.waitFor(1) # add client binary add_bin_cmd = "{0}/memsql-ops/memsql-ops file-add -t memsql {1}".format(Memsql.msqlopsdir, Memsql.msqlbin) add_client = ThreadedClients([Storage.master], add_bin_cmd, root=True) add_client.start() add_client.join() # add master aggregator maggr_cmd = "{0}/memsql-ops/memsql-ops memsql-deploy --role master --community-edition".format(Memsql.msqlopsdir) maggr_client = ThreadedClients([Storage.master], maggr_cmd, root=True) maggr_client.start() maggr_client.join() # start agents slave_ops = [] for s in Storage.servers: # start obs = Observer("memsql_platform.jobs.engine: Ready.") node_client = ThreadedClients([s], ops_cmd, root=True, observers=[obs]) node_client.start() obs.waitFor(1) slave_ops.append(node_client) # unfollow unfollow_cmd = "{0}/memsql-ops/memsql-ops unfollow".format(Memsql.msqlopsdir) ufw_client = ThreadedClients([s], unfollow_cmd, root=True) ufw_client.start() ufw_client.join() # follow master follow_cmd = "{0}/memsql-ops/memsql-ops follow -h {1} -P {2}".format(Memsql.msqlopsdir, Storage.master, Memsql.msqlopsport) fclient = ThreadedClients([s], follow_cmd, root=True) fclient.start() fclient.join() # start leaves start_leaves = "{0}/memsql-ops/memsql-ops memsql-deploy --role leaf --community-edition".format(Memsql.msqlopsdir) lclient = ThreadedClients([s], start_leaves, root=True) lclient.start() lclient.join() #sudo ./memsql-ops/memsql-ops/memsql-ops follow -h euler07 -P 9000 return [master_ops] + slave_ops
YCSB.networkThread) fst = None fstZero = True cCommand = "-C" if args.createTable else "" if len(YCSB.servers0) > 0: fst = YCSB.servers0.pop() else: fstZero = False fst = YCSB.servers1.pop() if fstZero: fst = ThreadedClients([fst], "numactl -m 0 -N 0 {0} {1}".format(cmd, cCommand)) else: fst = ThreadedClients([fst], "numactl -m 1 -N 1 {0} -p 8712 {1}".format( cmd, cCommand)) client0 = ThreadedClients(YCSB.servers0, "numactl -m 0 -N 0 {0}".format(cmd)) client1 = ThreadedClients(YCSB.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd)) fst.start() client0.start() client1.start() fst.join() client0.join() client1.join()
f.write("export LD_LIBRARY_PATH={0}\n".format(Spark.telljava)) tmpDirCommand = lambda host: os.system("ssh root@{0} 'rm -rf {1}; mkdir {1}'".format(host, Spark.tmpDir)) configCopyCommand = lambda host: os.system( "scp {0} {1} root@{2}:{3}/conf/".format(sparkEnv, sparkDefault, host, Spark.sparkdir) ) jarCopyCommand = lambda host: os.system("scp {0}/*.jar root@{1}:{0}".format(Spark.jarsDir, host)) tmpDirCommand(Spark.master) configCopyCommand(Spark.master) jarCopyCommand(Spark.master) for host in Spark.slaves: tmpDirCommand(host) configCopyCommand(host) jarCopyCommand(host) rmClients = ThreadedClients([Spark.master] + Spark.slaves, "rm -rf {0}/work".format(Spark.sparkdir), root=True) rmClients.start() rmClients.join() print master_cmd master = ThreadedClients([Spark.master], master_cmd, root=True) master.start() master.join() time.sleep(2) print slave_cmd slave = ThreadedClients(Spark.slaves, slave_cmd, root=True) slave.start() slave.join()
#!/usr/bin/env python from ServerConfig import Spark from threaded_ssh import ThreadedClients master_cmd ="{0}/sbin/stop-master.sh".format(Spark.sparkdir) slave_cmd ="{0}/sbin/stop-slave.sh".format(Spark.sparkdir, Spark.master) print master_cmd master = ThreadedClients([Spark.master], master_cmd, root=True) master.start() master.join() print slave_cmd slave = ThreadedClients(Spark.slaves, slave_cmd, root=True) slave.start() slave.join()
#!/usr/bin/env python from threaded_ssh import ThreadedClients from ServerConfig import Tpcc from ServerConfig import Kudu from ServerConfig import TellStore from ServerConfig import General def hostToIp(host): return General.infinibandIp[host] def semicolonReduce(x, y): return x + ';' + y cmd = "" if Tpcc.storage == Kudu: cmd = "{0}/watch/tpcc/tpcc_kudu -H `hostname` -W {1} --network-threads 8 -s {2} -P {3}".format(Tpcc.builddir, Tpcc.warehouses, Kudu.master, len(Kudu.tservers)*4) elif Tpcc.storage == TellStore: Tpcc.rsyncBuild() cmd = '{0}/watch/tpcc/tpcc_server -W {1} --network-threads 4 -c "{2}" -s "{3}"'.format(Tpcc.builddir, Tpcc.warehouses, General.infinibandIp[TellStore.commitmanager] + ":7242", reduce(semicolonReduce, map(lambda x: hostToIp(x) + ":7241", TellStore.servers))) client0 = ThreadedClients(Tpcc.servers0, "numactl -m 0 -N 0 {0}".format(cmd), rnd_start=True, root=False) client1 = ThreadedClients(Tpcc.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd), rnd_start=True, root=False) client0.start() client1.start() client0.join() client1.join()
cmd = "" if Tpcc.storage == Kudu: cmd = "{0}/watch/tpcc/tpcc_kudu -H `hostname` -W {1} --network-threads 8 -s {2} -P {3}".format( Tpcc.builddir, Tpcc.warehouses, Kudu.master, len(Kudu.tservers) * 4) elif Tpcc.storage == TellStore: Tpcc.rsyncBuild() cmd = '{0}/watch/tpcc/tpcc_server -W {1} --network-threads 4 -c "{2}" -s "{3}"'.format( Tpcc.builddir, Tpcc.warehouses, General.infinibandIp[TellStore.commitmanager] + ":7242", reduce(semicolonReduce, map(lambda x: hostToIp(x) + ":7241", TellStore.servers))) client0 = ThreadedClients(Tpcc.servers0, "numactl -m 0 -N 0 {0}".format(cmd), rnd_start=True, root=False) client1 = ThreadedClients(Tpcc.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd), rnd_start=True, root=False) client0.start() client1.start() client0.join() client1.join()
#!/usr/bin/env python from ServerConfig import Spark from threaded_ssh import ThreadedClients master_cmd = "{0}/sbin/stop-master.sh".format(Spark.sparkdir) slave_cmd = "{0}/sbin/stop-slave.sh".format(Spark.sparkdir, Spark.master) print master_cmd master = ThreadedClients([Spark.master], master_cmd, root=True) master.start() master.join() print slave_cmd slave = ThreadedClients(Spark.slaves, slave_cmd, root=True) slave.start() slave.join()
cmd = '{0}/watch/ycsb-server/ycsb-server -H `hostname` --network-threads {3} -c "{1}" -s "{2}"'.format(YCSB.builddir, General.infinibandIp[TellStore.commitmanager] + ":7242", reduce(semicolonReduce, map(lambda x: hostToIp(x) + ":7241", TellStore.servers)), YCSB.networkThread) fst = None fstZero = True cCommand = "-C" if args.createTable else "" if len(YCSB.servers0) > 0: fst = YCSB.servers0.pop() else: fstZero = False fst = YCSB.servers1.pop() if fstZero: fst = ThreadedClients([fst], "numactl -m 0 -N 0 {0} {1}".format(cmd, cCommand)) else: fst = ThreadedClients([fst], "numactl -m 1 -N 1 {0} -p 8712 {1}".format(cmd, cCommand)) client0 = ThreadedClients(YCSB.servers0, "numactl -m 0 -N 0 {0}".format(cmd)) client1 = ThreadedClients(YCSB.servers1, "numactl -m 1 -N 1 {0} -p 8712".format(cmd)) fst.start() client0.start() client1.start() fst.join() client0.join() client1.join()
def checkTopics(): chkCmd = '{0}/bin/kafka-topics.sh --list --zookeeper {1}:2181'.format( Config.exec_path, Config.server) chkClient = ThreadedClients([Config.server], chkCmd, root=True) chkClient.start() chkClient.join()