def run_fio(self):
run_cmd = """\
cd cassandra-fio
./fio_runner.sh lcs
"""
self._remoter.sudo(shell_script_cmd(run_cmd),
new_session=True,
ignore_status=True)
self._get_fio_results()
def _build_and_install_sysbench(self):
build_and_install_script = """\
cd ./sysbench
./autogen.sh
./configure --without-mysql
make -j
make install
"""
self._remoter.sudo(shell_script_cmd(build_and_install_script),
ignore_status=True)
def install_encryption_at_rest_files(remoter):
if remoter.sudo('ls /etc/encrypt_conf/system_key_dir',
ignore_status=True).ok:
return
remoter.send_files(src="./data_dir/encrypt_conf", dst="/tmp/")
remoter.sudo(
shell_script_cmd(
dedent("""
rm -rf /etc/encrypt_conf
mv -f /tmp/encrypt_conf /etc
mkdir -p /etc/scylla/encrypt_conf /etc/encrypt_conf/system_key_dir
chown -R scylla:scylla /etc/scylla /etc/encrypt_conf
""")))
remoter.sudo("md5sum /etc/encrypt_conf/*.pem", ignore_status=True)
def setup_jepsen(self):
remoter = self.jepsen_node.remoter
remoter.sudo("apt-get install -y libjna-java gnuplot graphviz git")
remoter.run(
shell_script_cmd(f"""\
curl -O https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein
chmod +x lein
./lein
git clone {self.params.get('jepsen_scylla_repo')} jepsen-scylla
"""))
for db_node in self.db_cluster.nodes:
remoter.run(
f"ssh-keyscan -t rsa {db_node.ip_address} >> ~/.ssh/known_hosts"
)
remoter.send_files(
os.path.expanduser(
self.db_cluster.nodes[0].ssh_login_info["key_file"]),
DB_SSH_KEY)
def save_jepsen_report(self):
url = f"http://{self.jepsen_node.external_address}:8080/"
self.log.info(
"Start web server to serve Jepsen results (will listen on %s)...",
url)
self.jepsen_node.remoter.run(shell_script_cmd(f"""\
cd ~/jepsen-scylla
setsid ~/lein run serve > save_jepsen_report.log 2>&1 < /dev/null &
sleep {JEPSEN_WEB_SERVER_START_DELAY}
"""),
verbose=True)
with open(os.path.join(self.logdir, "jepsen_report.html"),
"wt") as jepsen_report:
jepsen_report.write(requests.get(url).text)
self.log.info("Report has been saved to %s", jepsen_report.name)
return jepsen_report.name
class IptablesClusterOpsMixin:
def hydra_iptables_redirect_rules(self,
command: IptablesChainCommand = "A",
nodes: Optional[list] = None) -> List[str]:
if nodes is None:
nodes = self.nodes
return list(chain.from_iterable(node.iptables_node_redirect_rules(dest_ip=node.hydra_dest_ip,
iptables_bin=IPTABLES_LEGACY_BIN,
command=command) for node in nodes))
def nodes_iptables_redirect_rules(self,
command: IptablesChainCommand = "A",
nodes: Optional[list] = None) -> List[str]:
if nodes is None:
nodes = self.nodes
return list(chain.from_iterable(node.iptables_node_redirect_rules(dest_ip=node.nodes_dest_ip,
command=command) for node in nodes))
def add_hydra_iptables_rules(self, nodes: Optional[list] = None) -> None:
add_rules_commands = self.hydra_iptables_redirect_rules(nodes=nodes)
del_rules_commands = self.hydra_iptables_redirect_rules(command="D", nodes=nodes)
LOCALRUNNER.sudo(shell_script_cmd("\n".join(add_rules_commands)))
atexit.register(LOCALRUNNER.sudo, shell_script_cmd("\n".join(del_rules_commands)))
def update_nodes_iptables_redirect_rules(self,
command: IptablesChainCommand = "A",
nodes: Optional[list] = None,
loaders: bool = True,
monitors: bool = True) -> None:
nodes_to_update = []
if tester := TestConfig().tester_obj():
if loaders and tester.loaders:
nodes_to_update.extend(tester.loaders.nodes)
if monitors and tester.monitors:
nodes_to_update.extend(tester.monitors.nodes)
if nodes_to_update:
LOGGER.debug("Found following nodes to apply new iptables rules: %s", nodes_to_update)
iptables_rules = "\n".join(self.nodes_iptables_redirect_rules(command=command, nodes=nodes))
for node in nodes_to_update:
node.remoter.sudo(shell_script_cmd(iptables_rules))
def setup_jepsen(self):
remoter = self.jepsen_node.remoter
remoter.sudo(
"apt-get install -y openjdk-11-jre openjdk-11-jre-headless libjna-java gnuplot graphviz git"
)
remoter.run(
shell_script_cmd("""\
curl -O https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein
chmod +x lein
./lein
"""))
clone_repo(remoter=remoter,
repo_url=self.params.get('jepsen_scylla_repo'),
destination_dir_name="jepsen-scylla",
clone_as_root=False)
for db_node in self.db_cluster.nodes:
remoter.run(
f"ssh-keyscan -t rsa {db_node.ip_address} >> ~/.ssh/known_hosts"
)
remoter.send_files(
os.path.expanduser(
self.db_cluster.nodes[0].ssh_login_info["key_file"]),
DB_SSH_KEY)
def add_hydra_iptables_rules(self, nodes: Optional[list] = None) -> None:
add_rules_commands = self.hydra_iptables_redirect_rules(nodes=nodes)
del_rules_commands = self.hydra_iptables_redirect_rules(command="D", nodes=nodes)
LOCALRUNNER.sudo(shell_script_cmd("\n".join(add_rules_commands)))
atexit.register(LOCALRUNNER.sudo, shell_script_cmd("\n".join(del_rules_commands)))
def restart(self):
# We differentiate between "Restart" and "Reboot".
# Restart in AWS will be a Stop and Start of an instance.
# When using storage optimized instances like i2 or i3, the data on disk is deleted upon STOP. Therefore, we
# need to setup the instance and treat it as a new instance.
if self._instance.spot_instance_request_id:
LOGGER.debug("target node is spot instance, impossible to stop this instance, skipping the restart")
return
if self.is_seed:
# Due to https://github.com/scylladb/scylla/issues/7588, when we restart a node that is defined as "seed",
# we must state a different, living node as the seed provider in the scylla yaml of the restarted node
other_nodes = list(set(self.parent_cluster.nodes) - {self})
free_nodes = [node for node in other_nodes if not node.running_nemesis]
random_node = random.choice(free_nodes)
seed_provider = SeedProvider(
class_name="org.apache.cassandra.locator.SimpleSeedProvider",
parameters=[{"seeds": f"{random_node.ip_address}"}]
)
with self.remote_scylla_yaml() as scylla_yml:
scylla_yml.seed_provider = [seed_provider]
with ExitStack() as stack:
if self.is_data_device_lost_after_reboot:
# There is no disk yet, lots of the errors here are acceptable, and we'll ignore them.
for db_filter in (DbEventsFilter(db_event=DatabaseLogEvent.DATABASE_ERROR, node=self),
DbEventsFilter(db_event=DatabaseLogEvent.SCHEMA_FAILURE, node=self),
DbEventsFilter(db_event=DatabaseLogEvent.NO_SPACE_ERROR, node=self),
DbEventsFilter(db_event=DatabaseLogEvent.FILESYSTEM_ERROR, node=self),
DbEventsFilter(db_event=DatabaseLogEvent.RUNTIME_ERROR, node=self), ):
stack.enter_context(db_filter)
self.remoter.sudo(shell_script_cmd(f"""\
sed -e '/.*scylla/s/^/#/g' -i /etc/fstab
sed -e '/auto_bootstrap:.*/s/false/true/g' -i /etc/scylla/scylla.yaml
if ! grep ^replace_address_first_boot: /etc/scylla/scylla.yaml; then
echo 'replace_address_first_boot: {self.ip_address}' | tee --append /etc/scylla/scylla.yaml
fi
"""))
self._instance.stop()
self._instance_wait_safe(self._instance.wait_until_stopped)
self._instance.start()
self._instance_wait_safe(self._instance.wait_until_running)
self._wait_public_ip()
self.log.debug("Got a new public IP: %s", self._instance.public_ip_address)
self.refresh_ip_address()
self.wait_ssh_up()
if self.is_data_device_lost_after_reboot:
self.stop_scylla_server(verify_down=False)
# Moving var-lib-scylla.mount away, since scylla_create_devices fails if it already exists
mount_path = "/etc/systemd/system/var-lib-scylla.mount"
if self.remoter.sudo(f"test -e {mount_path}", ignore_status=True).ok:
self.remoter.sudo(f"mv {mount_path} /tmp/")
# The scylla_create_devices has been moved to the '/opt/scylladb' folder in the master branch.
for create_devices_file in ("/usr/lib/scylla/scylla-ami/scylla_create_devices",
"/opt/scylladb/scylla-ami/scylla_create_devices",
"/opt/scylladb/scylla-machine-image/scylla_create_devices", ):
if self.remoter.sudo(f"test -x {create_devices_file}", ignore_status=True).ok:
self.remoter.sudo(create_devices_file)
break
else:
raise IOError("scylla_create_devices file isn't found")
self.start_scylla_server(verify_up=False)
self.remoter.sudo(shell_script_cmd("""\
sed -e '/auto_bootstrap:.*/s/true/false/g' -i /etc/scylla/scylla.yaml
sed -e 's/^replace_address_first_boot:/# replace_address_first_boot:/g' -i /etc/scylla/scylla.yaml
"""))
def install_prereqs(self,
public_ip: str,
connect_timeout: Optional[int] = None) -> None:
from sdcm.cluster_docker import AIO_MAX_NR_RECOMMENDED_VALUE # pylint: disable=import-outside-toplevel
LOGGER.info("Connecting instance...")
remoter = self.get_remoter(host=public_ip,
connect_timeout=connect_timeout)
LOGGER.info("Installing required packages...")
login_user = self.LOGIN_USER
public_key = self.key_pair.public_key.decode()
result = remoter.sudo(shell_script_cmd(quote="'",
cmd=f"""\
# Make sure that cloud-init finished running.
until [ -f /var/lib/cloud/instance/boot-finished ]; do sleep 1; done
echo "fs.aio-max-nr = {AIO_MAX_NR_RECOMMENDED_VALUE}" >> /etc/sysctl.conf
echo "{login_user} soft nofile 4096" >> /etc/security/limits.conf
echo "jenkins soft nofile 4096" >> /etc/security/limits.conf
echo "root soft nofile 4096" >> /etc/security/limits.conf
# Configure default user account.
sudo -u {login_user} mkdir -p /home/{login_user}/.ssh || true
echo "{public_key}" >> /home/{login_user}/.ssh/authorized_keys
chmod 600 /home/{login_user}/.ssh/authorized_keys
mkdir -p -m 777 /home/{login_user}/sct-results
echo "cd ~/sct-results" >> /home/{login_user}/.bashrc
chown -R {login_user}:{login_user} /home/{login_user}/
# Disable apt-key warnings and set non-interactive frontend.
export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
export DEBIAN_FRONTEND=noninteractive
apt-get -qq clean
apt-get -qq update
apt-get -qq install --no-install-recommends python3-pip htop screen tree
pip3 install awscli
# Install Docker.
apt-get -qq install --no-install-recommends \
apt-transport-https ca-certificates curl gnupg-agent software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
apt-get -qq install --no-install-recommends docker-ce docker-ce-cli containerd.io
usermod -aG docker {login_user}
# Install kubectl.
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
# Configure Jenkins user.
apt-get -qq install --no-install-recommends openjdk-11-jre-headless # https://www.jenkins.io/doc/administration/requirements/java/
adduser --disabled-password --gecos "" jenkins || true
usermod -aG docker jenkins
mkdir -p /home/jenkins/.ssh
echo "{public_key}" >> /home/jenkins/.ssh/authorized_keys
chmod 600 /home/jenkins/.ssh/authorized_keys
chown -R jenkins:jenkins /home/jenkins
echo "jenkins ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/jenkins
# Jenkins pipelines run /bin/sh for some reason.
ln -sf /bin/bash /bin/sh
"""),
ignore_status=True)
remoter.stop()
if result.ok:
LOGGER.info("All packages successfully installed.")
else:
raise Exception(
f"Unable to install required packages:\n{result.stdout}{result.stderr}"
)
def test_shell_script_cmd(self):
self.assertEqual(shell_script_cmd("true"), 'bash -cxe "true"')
