def __prepare(self, ip, kill_java):
log_machine(ip, f'Preparing: started')
ssh = self.__new_ssh(ip)
# we need to make sure that the no old load generator is still running.
if kill_java:
ssh.exec(f'killall -q -9 java || true')
log_machine(ip, f'Preparing: done')
def __start_exporter(self, ip):
log_machine(ip, 'Starting exporter')
ssh = self.__new_ssh(ip)
ssh.exec(f"""
java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar ~/cassandra_exporter-2.3.5.jar ~/config.yml >> ~/cassandra_exporter.out 2>&1 &
""")
log_machine(ip, 'Starting exporter: done')
def __collect(self, ip, dir):
dest_dir = os.path.join(dir, ip)
os.makedirs(dest_dir, exist_ok=True)
log_machine(ip, f'Collecting to [{dest_dir}]')
ssh = self.__new_ssh(ip)
ssh.scp_from_remote(f'*.{{html,hdr,log}}', dest_dir)
ssh.exec(f'rm -fr *.html *.hdr *.log')
log_machine(ip, f'Collecting to [{dest_dir}] done')
def start(self):
log_important(f"Starting Cassandra nodes {self.cluster_public_ips}")
for public_ip in self.cluster_public_ips:
self.__start(public_ip)
wait_for_cql_start(public_ip)
log_machine(public_ip, f"""Node finished bootstrapping""")
self.__start_exporter(public_ip)
log_important(
f"Starting Cassandra nodes {self.cluster_public_ips}: done")
def __download(self, ip, dir):
dest_dir = os.path.join(dir, ip)
os.makedirs(dest_dir, exist_ok=True)
log_machine(ip, f'[{ip}] Downloading to [{dest_dir}]')
self.__new_ssh(ip).scp_from_remote(f'diskplorer/*.{{svg,csv}}', dest_dir)
if self.capture_lsblk:
self.__new_ssh(ip).scp_from_remote(f'lsblk.out', dest_dir)
log_machine(ip, f'Downloading to [{dest_dir}] done')
def start(self):
log(f"Starting Scylla nodes {self.cluster_public_ips}")
for public_ip in self.cluster_public_ips:
ssh = self.__new_ssh(public_ip)
ssh.exec("sudo systemctl start scylla-server")
for public_ip in self.cluster_public_ips:
wait_for_cql_start(public_ip)
log_machine(public_ip, "Node finished bootstrapping")
log(f"Starting Scylla nodes {self.cluster_public_ips}: done")
def __download(self, ip, dir):
dest_dir = os.path.join(dir, ip)
os.makedirs(dest_dir, exist_ok=True)
log_machine(ip, f'Downloading to [{dest_dir}]')
ssh = self.__new_ssh(ip)
ssh.scp_from_remote(f'{self.dir_name}/*', dest_dir)
if self.capture_lsblk:
self.__new_ssh(ip).scp_from_remote(f'lsblk.out', dest_dir)
log_machine(ip, f'Downloading to [{dest_dir}] done')
def __install(self, ip):
log_machine(ip, 'Installing disk-explorer: started')
ssh = self.__new_ssh(ip)
ssh.update()
ssh.install('git', 'fio', 'python3', 'python3-pip')
ssh.exec(f"""
sudo pip3 install -qqq matplotlib
rm -fr diskplorer
git clone -q https://github.com/scylladb/diskplorer.git
""")
log_machine(ip, 'Installing disk-explorer: done')
def __run(self, ip, options):
log_machine(ip, 'fio: started')
ssh = self.__new_ssh(ip)
if self.capture_lsblk:
ssh.exec(f'lsblk > lsblk.txt')
ssh.exec(f"""
mkdir -p {self.dir_name}
cd {self.dir_name}
sudo fio {options}
""")
log_machine(ip, 'fio: done')
def __install(self, ip):
ssh = self.__new_ssh(ip)
log_machine(ip, f'Installing scylla_bench: started')
ssh.update()
if self.performance_governor:
ssh.set_governor("performance")
ssh.install("golang")
# ssh.exec("go get github.com/scylladb/scylla-bench")
ssh.exec(f"""
git clone https://github.com/scylladb/scylla-bench
cd scylla-bench/
go install .
""")
log_machine(ip, f'Installing scylla_bench: done')
def __stop(self, ip):
log_machine(ip, "Stopping Cassandra: started")
ssh = self.__new_ssh(ip)
path_prefix = 'cassandra-raid/' if self.setup_raid else './'
ssh.exec(f"""
set -e
cd {path_prefix}apache-cassandra-{self.cassandra_version}
if [ -f 'cassandra.pid' ]; then
pid=$(cat cassandra.pid)
kill $pid || true
while kill -0 $pid; do
sleep 1
done
rm -f 'cassandra.pid'
fi
""")
log_machine(ip, 'Stopping Cassandra: done')
def __run(self, ip, cmd):
log_machine(ip, 'Run: started')
ssh = self.__new_ssh(ip)
ssh.exec('rm -fr diskplorer/*.svg')
ssh.exec(f'rm -fr diskplorer/fiotest.tmp')
if self.capture_lsblk:
ssh.exec(f'lsblk > lsblk.out')
ssh.exec(f"""
ulimit -n 64000
ulimit -n
ulimit -Sn 64000
ulimit -Sn
cd diskplorer
python3 diskplorer.py {cmd}
""")
# the file is 100 GB; so we want to remove it.
ssh.exec(f'rm -fr diskplorer/fiotest.tmp')
log_machine(ip, 'Run: done')
def __install(self, ip):
ssh = self.__new_ssh(ip)
ssh.update()
log_machine(ip, "Installing Cassandra: started")
ssh.install_one('openjdk-16-jdk', 'java-16-openjdk')
ssh.install('wget')
private_ip = self.__find_private_ip(ip)
path_prefix = 'cassandra-raid/' if self.setup_raid else './'
ssh.exec(f"""
set -e
if [ -d '{path_prefix}apache-cassandra-{self.cassandra_version}' ]; then
echo Cassandra {self.cassandra_version} already installed.
exit 0
fi
wget -q -N https://archive.apache.org/dist/cassandra/{self.cassandra_version}/apache-cassandra-{self.cassandra_version}-bin.tar.gz
tar -xzf apache-cassandra-{self.cassandra_version}-bin.tar.gz -C {path_prefix}
wget -q https://github.com/criteo/cassandra_exporter/releases/download/2.3.5/cassandra_exporter-2.3.5.jar
""")
ssh.scp_to_remote(
"jvm11-server.options",
f"{path_prefix}apache-cassandra-{self.cassandra_version}/conf/jvm11-server.options"
)
ssh.scp_to_remote(
"cassandra.yaml",
f"{path_prefix}apache-cassandra-{self.cassandra_version}/conf/cassandra.yaml"
)
ssh.scp_to_remote("cassandra-exporter.yml", f"config.yml")
# FIXME - heap in MB * 2
ssh.exec("""
sudo sh -c "echo 262144 > /proc/sys/vm/max_map_count"
""")
ssh.exec(f"""
cd {path_prefix}apache-cassandra-{self.cassandra_version}
sudo sed -i \"s/seeds:.*/seeds: {self.seed_private_ip} /g\" conf/cassandra.yaml
sudo sed -i \"s/listen_address:.*/listen_address: {private_ip} /g\" conf/cassandra.yaml
sudo sed -i \"s/rpc_address:.*/rpc_address: {private_ip} /g\" conf/cassandra.yaml
""")
log_machine(ip, "Installing Cassandra: done")
def __install(self, ip):
ssh = self.__new_ssh(ip)
ssh.update()
if self.performance_governor:
ssh.set_governor("performance")
if self.scylla_tools:
log_machine(ip, f'Installing cassandra-stress (Scylla): started')
ssh.exec(f"""
set -e
if hash apt-get 2>/dev/null; then
sudo apt-get install -y apt-transport-https
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 5e08fbd8b5d6ec9c
sudo curl -L --output /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/ubuntu/scylla-4.5-$(lsb_release -s -c).list
sudo apt-get update -y
sudo apt-get install -y scylla-tools
elif hash yum 2>/dev/null; then
sudo yum install -y -q https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
sudo curl -o /etc/yum.repos.d/scylla.repo -L http://downloads.scylladb.com/rpm/centos/scylla-4.5.repo
sudo yum install -y -q scylla-tools
else
echo "Cannot install scylla-tools: yum/apt not found"
exit 1
fi
""")
else:
log_machine(ip,
f'Installing cassandra-stress (Cassandra): started')
cassandra_version = self.properties['cassandra_version']
ssh.install_one('openjdk-8-jdk', 'java-1.8.0-openjdk')
ssh.install('wget')
ssh.exec(f"""
set -e
wget -q -N https://mirrors.netix.net/apache/cassandra/{cassandra_version}/apache-cassandra-{cassandra_version}-bin.tar.gz
tar -xzf apache-cassandra-{cassandra_version}-bin.tar.gz
""")
log_machine(ip, f'Installing cassandra-stress: done')
def wait_for_cql_start(node_ip,
timeout=7200,
connect_timeout=10,
max_tries_per_second=2):
log_machine(
node_ip,
'Waiting for CQL port to start (meaning node bootstrap finished). This could take a while.'
)
backoff_interval = 1.0 / max_tries_per_second
timeout_point = datetime.now() + timedelta(seconds=timeout)
feedback_interval = 20
print_feedback_point = datetime.now() + timedelta(
seconds=feedback_interval)
while datetime.now() < timeout_point:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(connect_timeout)
try:
sock.connect((node_ip, 9042))
except:
# There was a problem connecting to CQL port.
sleep(backoff_interval)
if datetime.now() > print_feedback_point:
print_feedback_point = datetime.now() + timedelta(
seconds=feedback_interval)
log_machine(node_ip,
'Still waiting for CQL port to start...')
else:
log_machine(node_ip, 'Successfully connected to CQL port.')
return
raise Exception(
f'Waiting for CQL to start timed out after {timeout} seconds for node: {node_ip}.'
)
def __prepare(self, ip):
log_machine(ip, f'Preparing: started')
ssh = self.__new_ssh(ip)
# we need to make sure that the no old load generator is still running.
ssh.exec(f'killall -q -9 go/bin/scylla-bench')
log_machine(ip, f'Preparing: done')
def __install(self, ip):
log_machine(ip, 'Installing fio: started')
ssh = self.__new_ssh(ip)
ssh.update()
ssh.install('fio')
log_machine(ip, 'Installing fio: done')
