def append_env_configuration(self, configuration):
log(f"Appending cassandra-env.sh configuration to nodes {self.cluster_public_ips}: {configuration}"
)
pssh = PSSH(self.cluster_public_ips, self.ssh_user,
self.properties['ssh_options'])
path_prefix = 'cassandra-raid/' if self.setup_raid else './'
log("configuration[" + configuration + "]")
pssh.exec(
f'''echo '{configuration}' >> {path_prefix}apache-cassandra-{self.cassandra_version}/conf/cassandra-env.sh'''
)
log(f"echo '{configuration}' >> {path_prefix}apache-cassandra-{self.cassandra_version}/conf/cassandra-env.sh"
)
def exec(self, command):
"""
Returns the perf command on the remote machine.
The command needs to be the full command like 'sudo perf record ...'
"""
log_important(f"Perf: started")
log(command)
pssh = PSSH(self.ip_list, self.user, self.ssh_options)
pssh.exec(f"""
cd /tmp
{command}
""")
log_important(f"Perf: done")
def clear_cluster(cluster_public_ips, cluster_user, ssh_options, duration_seconds=90):
log_important("Shutting down cluster and removing all data")
pssh = PSSH(cluster_public_ips, cluster_user, ssh_options)
# pssh.exec("nodetool flush")
log("Stopping scylla")
pssh.exec("sudo systemctl stop scylla-server")
log("Removing data dir")
pssh.exec("sudo rm -fr /var/lib/scylla/data/*")
log("Removing commit log")
pssh.exec("sudo rm -fr /var/lib/scylla/commitlog/*")
log("Starting scylla")
pssh.exec("sudo systemctl start scylla-server")
log(f"Waiting {duration_seconds} seconds")
sleep(duration_seconds)
log_important("Cluster cleared and restarted")
def install(self):
ips = ','.join(self.public_ips)
log(f'[{ips}] raid: starting creating RAID')
pssh = PSSH(self.public_ips, self.user, self.properties['ssh_options'])
pssh.exec(f"""
if [[ ! -b /dev/md/{self.raid_device_name} ]]; then
sudo mdadm --create --verbose /dev/md/{self.raid_device_name} --chunk=256 --metadata=1.2 --level={self.level} --force --raid-devices=$(ls {self.device_name_wildcard} | wc -l) {self.device_name_wildcard}
# /dev/md/raid_device_name maps to /dev/md[0-9]+
MD_NAME=$(basename $(readlink /dev/md/{self.raid_device_name}))
# Tuning
sudo sh -c "echo 1 > /sys/block/$MD_NAME/queue/nomerges"
sudo sh -c "echo 8 > /sys/block/$MD_NAME/queue/read_ahead_kb"
sudo sh -c "echo none > /sys/block/$MD_NAME/queue/scheduler"
sudo mkfs.xfs -f /dev/$MD_NAME
mkdir {self.raid_device_name}
sudo mount /dev/$MD_NAME {self.raid_device_name}
sudo chown $(id -u) {self.raid_device_name}
fi
""")
log(f'[{ips}] raid: finished creating RAID')
def collect_flamegraph(self,
dir,
data_file="perf.data",
flamegraph_file="flamegraph.svg"):
"""
Collect the remotely created flame-graphs
"""
log_important(f"Perf collecting flamegraph: started")
pssh = PSSH(self.ip_list, self.user, self.ssh_options)
# --no-online
pssh.exec(f"""
cd /tmp
sudo perf script -i {data_file} | FlameGraph/stackcollapse-perf.pl | FlameGraph/flamegraph.pl --hash > {flamegraph_file}
""")
pssh.scp_from_remote(f"/tmp/{flamegraph_file}", dir)
pssh.exec(f"rm /tmp/{flamegraph_file}")
log_important(f"Perf collecting flamegraph: done")
def restart_cluster(cluster_public_ips, cluster_user, ssh_options, duration_seconds=90):
log_important("Restart cluster ")
pssh = PSSH(cluster_public_ips, cluster_user, ssh_options)
log("nodetool drain")
pssh.exec("nodetool drain")
log("sudo systemctl restart scylla-server")
pssh.exec("sudo systemctl restart scylla-server")
log(f"Waiting {duration_seconds} seconds")
sleep(duration_seconds)
log_important("Cluster restarted")
def __pssh(self):
return PSSH(self.ip_list, self.user, self.ssh_options)
def nodes_start(cluster_user, ssh_options, *public_ips):
log_important(f"Starting nodes {public_ips}")
pssh = PSSH(public_ips, cluster_user, ssh_options)
pssh.exec("sudo systemctl start scylla-server")
log_important(f"Starting nodes {public_ips}: done")
def nodes_remove_data(cluster_user, ssh_options, *public_ips):
log_important(f"Removing data from nodes {public_ips}")
pssh = PSSH(public_ips, cluster_user, ssh_options)
pssh.exec("sudo rm -fr /var/lib/scylla/data/*")
pssh.exec("sudo rm -fr /var/lib/scylla/commitlog/*")
log_important(f"Removing data from nodes {public_ips}: done")
def append_configuration(self, configuration):
log(f"Appending configuration to nodes {self.cluster_public_ips}: {configuration}")
pssh = PSSH(self.cluster_public_ips, self.ssh_user, self.properties['ssh_options'])
pssh.exec(f"sudo sh -c \"echo '{configuration}' >> /etc/scylla/scylla.yaml\"")
def cli():
props = common.load_yaml('properties.yml')
env = common.load_yaml('environment.yml')
pssh = PSSH(env['loadgenerator_public_ips'], props['load_generator_user'], props['ssh_options'])
pssh.exec(f'killall -q -9 java')
pssh.exec(f'killall -q -9 go/bin/scylla-bench')