def setup_hosts(clusters):
pprint("Appending authorized key...")
run_cmd("all-hosts", "sudo chown ubuntu /etc/security/limits.conf; sudo chmod u+w /etc/security/limits.conf; sudo echo '* soft nofile 1000000\n* hard nofile 1000000' >> /etc/security/limits.conf; sudo chown ubuntu /etc/pam.d/common-session; sudo echo 'session required pam_limits.so' >> /etc/pam.d/common-session")
run_cmd("all-hosts", "cat /home/ubuntu/.ssh/kaiju_rsa.pub >> /home/ubuntu/.ssh/authorized_keys", user="******")
pprint("Done")
run_cmd("all-hosts", " wget --output-document sigar.tar.gz 'http://downloads.sourceforge.net/project/sigar/sigar/1.6/hyperic-sigar-1.6.4.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsigar%2Ffiles%2Fsigar%2F1.6%2F&ts=1375479576&use_mirror=iweb'; tar -xvf sigar*; sudo rm /usr/local/lib/libsigar*; sudo cp ./hyperic-sigar-1.6.4/sigar-bin/lib/libsigar-amd64-linux.so /usr/local/lib/; rm -rf *sigar*")
run_cmd("all-hosts", "sudo echo 'include /usr/local/lib' >> /etc/ld.so.conf; sudo ldconfig")
def start_ycsb_clients(clusters, **kwargs):
def fmt_ycsb_string(runType, cluster):
return (('cd /home/ubuntu/kaiju/contrib/YCSB;' +
netCmd+
'rm *.log;' \
'bin/ycsb %s kaiju -p hosts=%s -threads %d -p txnlen=%d -p readproportion=%s -p updateproportion=%s -p fieldlength=%d -p histogram.buckets=%d -p fieldcount=1 -p operationcount=100000000 -p recordcount=%d -p isolation_level=%s -p read_atomic_algorithm=%s -t -s ' \
' -p requestdistribution=%s -p maxexecutiontime=%d -P %s' \
' 1>%s_out.log 2>%s_err.log') % (runType,
KAIJU_HOSTS_EXTERNAL,
kwargs.get("threads", 10) if runType != 'load' else min(1000, kwargs.get("recordcount")/10),
kwargs.get("txnlen", 8),
kwargs.get("readprop", .5),
1-kwargs.get("readprop", .5),
kwargs.get("valuesize", 1),
kwargs.get("numbuckets", 10000),
kwargs.get("recordcount", 10000),
kwargs.get("isolation_level", "READ_COMMITTED"),
kwargs.get("ra_algorithm", "KEY_LIST"),
kwargs.get("keydistribution", "zipfian"),
kwargs.get("time", 60) if runType != 'load' else 10000,
kwargs.get("workload", "workloads/workloada"),
runType,
runType))
cluster = clusters[0]
pprint("Loading YCSB on single client: %s." % (cluster.clients[0].ip))
run_cmd_single(cluster.clients[0].ip,
fmt_ycsb_string("load", cluster),
time=kwargs.get("recordcount", 180))
pprint("Done")
sleep(10)
pprint("Running YCSB on all clients.")
if kwargs.get("bgrun", False):
for client in cluster.clients:
start_cmd_disown(client.ip, fmt_ycsb_string("run", cluster))
sleep(kwargs.get("time") + 15)
else:
run_cmd("all-clients",
fmt_ycsb_string("run", cluster),
time=kwargs.get("time", 60) + 30)
pprint("Done")
def start_ycsb_clients(clusters, **kwargs):
def fmt_ycsb_string(runType, cluster):
return (('cd /home/ubuntu/kaiju/contrib/YCSB;' +
netCmd+
'rm *.log;' \
'bin/ycsb %s kaiju -p hosts=%s -threads %d -p txnlen=%d -p readproportion=%s -p updateproportion=%s -p fieldlength=%d -p histogram.buckets=%d -p fieldcount=1 -p operationcount=100000000 -p recordcount=%d -p isolation_level=%s -p read_atomic_algorithm=%s -t -s ' \
' -p requestdistribution=%s -p maxexecutiontime=%d -P %s' \
' 1>%s_out.log 2>%s_err.log') % (runType,
KAIJU_HOSTS_EXTERNAL,
kwargs.get("threads", 10) if runType != 'load' else min(1000, kwargs.get("recordcount")/10),
kwargs.get("txnlen", 8),
kwargs.get("readprop", .5),
1-kwargs.get("readprop", .5),
kwargs.get("valuesize", 1),
kwargs.get("numbuckets", 10000),
kwargs.get("recordcount", 10000),
kwargs.get("isolation_level", "READ_COMMITTED"),
kwargs.get("ra_algorithm", "KEY_LIST"),
kwargs.get("keydistribution", "zipfian"),
kwargs.get("time", 60) if runType != 'load' else 10000,
kwargs.get("workload", "workloads/workloada"),
runType,
runType))
cluster = clusters[0]
pprint("Loading YCSB on single client: %s." % (cluster.clients[0].ip))
run_cmd_single(cluster.clients[0].ip, fmt_ycsb_string("load", cluster), time=kwargs.get("recordcount", 180))
pprint("Done")
sleep(10)
pprint("Running YCSB on all clients.")
if kwargs.get("bgrun", False):
for client in cluster.clients:
start_cmd_disown(client.ip, fmt_ycsb_string("run", cluster))
sleep(kwargs.get("time")+15)
else:
run_cmd("all-clients", fmt_ycsb_string("run", cluster), time=kwargs.get("time", 60)+30)
pprint("Done")
def setup_hosts(clusters):
pprint("Appending authorized key...")
run_cmd(
"all-hosts",
"sudo chown ubuntu /etc/security/limits.conf; sudo chmod u+w /etc/security/limits.conf; sudo echo '* soft nofile 1000000\n* hard nofile 1000000' >> /etc/security/limits.conf; sudo chown ubuntu /etc/pam.d/common-session; sudo echo 'session required pam_limits.so' >> /etc/pam.d/common-session"
)
run_cmd(
"all-hosts",
"cat /home/ubuntu/.ssh/kaiju_rsa.pub >> /home/ubuntu/.ssh/authorized_keys",
user="******")
pprint("Done")
run_cmd(
"all-hosts",
" wget --output-document sigar.tar.gz 'http://downloads.sourceforge.net/project/sigar/sigar/1.6/hyperic-sigar-1.6.4.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsigar%2Ffiles%2Fsigar%2F1.6%2F&ts=1375479576&use_mirror=iweb'; tar -xvf sigar*; sudo rm /usr/local/lib/libsigar*; sudo cp ./hyperic-sigar-1.6.4/sigar-bin/lib/libsigar-amd64-linux.so /usr/local/lib/; rm -rf *sigar*"
)
run_cmd(
"all-hosts",
"sudo echo 'include /usr/local/lib' >> /etc/ld.so.conf; sudo ldconfig")
def setup_cluster():
print "Enabling root SSH...",
run_script("all-hosts", "scripts/enable_root_ssh.sh", user="******")
print "Done"
print "Setting up XFS...",
run_script("all-hosts", "scripts/set_up_xfs.sh")
print "Done"
print "Fixing host file bugs...",
run_script("all-hosts", "scripts/fix-hosts-file.sh")
print "Done"
print "Installing NTP (Ignore failures)..."
run_cmd("all-hosts", "sudo apt-get -q -y install ntp")
run_cmd("all-hosts", "sudo ntpd -q")
print "Done"
print "Installing Jmxterm..."
run_cmd("all-hosts", "wget http://downloads.sourceforge.net/"\
"cyclops-group/jmxterm-1.0-alpha-4-uber.jar",
user="******")
print "Done"
def clone_cassandra_pbs():
run_cmd("all-hosts", "rm -rf cassandra", user="******")
run_cmd("all-hosts",
"git clone https://github.com/pbailis/cassandra-pbs cassandra",
user="******")
def run_cmd_in_ycsb(hosts, cmd, user='******'):
run_cmd(hosts, "cd /home/ubuntu/kaiju/contrib/YCSB/; %s" % cmd, user)
def run_cmd_in_kaiju(hosts, cmd, user='******'):
run_cmd(hosts, "cd /home/ubuntu/kaiju/; %s" % cmd, user)
def stop_kaiju_clients(clusters):
pprint("Terminating client java processes...")
run_cmd("all-clients", "killall -9 java;")
pprint('Termination command sent.')
def stop_kaiju_processes(clusters):
pprint("Terminating java processes...")
run_cmd("all-hosts", "killall -9 java; pkill -9 java")
sleep(10)
pprint('Termination command sent.')
def run_cmd_in_ycsb(hosts, cmd, user='******'):
run_cmd(hosts, "cd /home/ubuntu/kaiju/contrib/YCSB/; %s" % cmd, user)
def run_cmd_in_kaiju(hosts, cmd, user='******'):
run_cmd(hosts, "cd /home/ubuntu/kaiju/; %s" % cmd, user)
def stop_kaiju_clients(clusters):
pprint("Terminating client java processes...")
run_cmd("all-clients", "killall -9 java;")
pprint('Termination command sent.')
def stop_kaiju_processes(clusters):
pprint("Terminating java processes...")
run_cmd("all-hosts", "killall -9 java; pkill -9 java")
sleep(10)
pprint('Termination command sent.')