def run_fio(self): run_cmd = """\ cd cassandra-fio ./fio_runner.sh lcs """ self._remoter.sudo(shell_script_cmd(run_cmd), new_session=True, ignore_status=True) self._get_fio_results()
def _build_and_install_sysbench(self): build_and_install_script = """\ cd ./sysbench ./autogen.sh ./configure --without-mysql make -j make install """ self._remoter.sudo(shell_script_cmd(build_and_install_script), ignore_status=True)
def install_encryption_at_rest_files(remoter): if remoter.sudo('ls /etc/encrypt_conf/system_key_dir', ignore_status=True).ok: return remoter.send_files(src="./data_dir/encrypt_conf", dst="/tmp/") remoter.sudo( shell_script_cmd( dedent(""" rm -rf /etc/encrypt_conf mv -f /tmp/encrypt_conf /etc mkdir -p /etc/scylla/encrypt_conf /etc/encrypt_conf/system_key_dir chown -R scylla:scylla /etc/scylla /etc/encrypt_conf """))) remoter.sudo("md5sum /etc/encrypt_conf/*.pem", ignore_status=True)
def setup_jepsen(self): remoter = self.jepsen_node.remoter remoter.sudo("apt-get install -y libjna-java gnuplot graphviz git") remoter.run( shell_script_cmd(f"""\ curl -O https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein chmod +x lein ./lein git clone {self.params.get('jepsen_scylla_repo')} jepsen-scylla """)) for db_node in self.db_cluster.nodes: remoter.run( f"ssh-keyscan -t rsa {db_node.ip_address} >> ~/.ssh/known_hosts" ) remoter.send_files( os.path.expanduser( self.db_cluster.nodes[0].ssh_login_info["key_file"]), DB_SSH_KEY)
def save_jepsen_report(self): url = f"http://{self.jepsen_node.external_address}:8080/" self.log.info( "Start web server to serve Jepsen results (will listen on %s)...", url) self.jepsen_node.remoter.run(shell_script_cmd(f"""\ cd ~/jepsen-scylla setsid ~/lein run serve > save_jepsen_report.log 2>&1 < /dev/null & sleep {JEPSEN_WEB_SERVER_START_DELAY} """), verbose=True) with open(os.path.join(self.logdir, "jepsen_report.html"), "wt") as jepsen_report: jepsen_report.write(requests.get(url).text) self.log.info("Report has been saved to %s", jepsen_report.name) return jepsen_report.name
class IptablesClusterOpsMixin: def hydra_iptables_redirect_rules(self, command: IptablesChainCommand = "A", nodes: Optional[list] = None) -> List[str]: if nodes is None: nodes = self.nodes return list(chain.from_iterable(node.iptables_node_redirect_rules(dest_ip=node.hydra_dest_ip, iptables_bin=IPTABLES_LEGACY_BIN, command=command) for node in nodes)) def nodes_iptables_redirect_rules(self, command: IptablesChainCommand = "A", nodes: Optional[list] = None) -> List[str]: if nodes is None: nodes = self.nodes return list(chain.from_iterable(node.iptables_node_redirect_rules(dest_ip=node.nodes_dest_ip, command=command) for node in nodes)) def add_hydra_iptables_rules(self, nodes: Optional[list] = None) -> None: add_rules_commands = self.hydra_iptables_redirect_rules(nodes=nodes) del_rules_commands = self.hydra_iptables_redirect_rules(command="D", nodes=nodes) LOCALRUNNER.sudo(shell_script_cmd("\n".join(add_rules_commands))) atexit.register(LOCALRUNNER.sudo, shell_script_cmd("\n".join(del_rules_commands))) def update_nodes_iptables_redirect_rules(self, command: IptablesChainCommand = "A", nodes: Optional[list] = None, loaders: bool = True, monitors: bool = True) -> None: nodes_to_update = [] if tester := TestConfig().tester_obj(): if loaders and tester.loaders: nodes_to_update.extend(tester.loaders.nodes) if monitors and tester.monitors: nodes_to_update.extend(tester.monitors.nodes) if nodes_to_update: LOGGER.debug("Found following nodes to apply new iptables rules: %s", nodes_to_update) iptables_rules = "\n".join(self.nodes_iptables_redirect_rules(command=command, nodes=nodes)) for node in nodes_to_update: node.remoter.sudo(shell_script_cmd(iptables_rules))
def setup_jepsen(self): remoter = self.jepsen_node.remoter remoter.sudo( "apt-get install -y openjdk-11-jre openjdk-11-jre-headless libjna-java gnuplot graphviz git" ) remoter.run( shell_script_cmd("""\ curl -O https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein chmod +x lein ./lein """)) clone_repo(remoter=remoter, repo_url=self.params.get('jepsen_scylla_repo'), destination_dir_name="jepsen-scylla", clone_as_root=False) for db_node in self.db_cluster.nodes: remoter.run( f"ssh-keyscan -t rsa {db_node.ip_address} >> ~/.ssh/known_hosts" ) remoter.send_files( os.path.expanduser( self.db_cluster.nodes[0].ssh_login_info["key_file"]), DB_SSH_KEY)
def add_hydra_iptables_rules(self, nodes: Optional[list] = None) -> None: add_rules_commands = self.hydra_iptables_redirect_rules(nodes=nodes) del_rules_commands = self.hydra_iptables_redirect_rules(command="D", nodes=nodes) LOCALRUNNER.sudo(shell_script_cmd("\n".join(add_rules_commands))) atexit.register(LOCALRUNNER.sudo, shell_script_cmd("\n".join(del_rules_commands)))
def restart(self): # We differentiate between "Restart" and "Reboot". # Restart in AWS will be a Stop and Start of an instance. # When using storage optimized instances like i2 or i3, the data on disk is deleted upon STOP. Therefore, we # need to setup the instance and treat it as a new instance. if self._instance.spot_instance_request_id: LOGGER.debug("target node is spot instance, impossible to stop this instance, skipping the restart") return if self.is_seed: # Due to https://github.com/scylladb/scylla/issues/7588, when we restart a node that is defined as "seed", # we must state a different, living node as the seed provider in the scylla yaml of the restarted node other_nodes = list(set(self.parent_cluster.nodes) - {self}) free_nodes = [node for node in other_nodes if not node.running_nemesis] random_node = random.choice(free_nodes) seed_provider = SeedProvider( class_name="org.apache.cassandra.locator.SimpleSeedProvider", parameters=[{"seeds": f"{random_node.ip_address}"}] ) with self.remote_scylla_yaml() as scylla_yml: scylla_yml.seed_provider = [seed_provider] with ExitStack() as stack: if self.is_data_device_lost_after_reboot: # There is no disk yet, lots of the errors here are acceptable, and we'll ignore them. for db_filter in (DbEventsFilter(db_event=DatabaseLogEvent.DATABASE_ERROR, node=self), DbEventsFilter(db_event=DatabaseLogEvent.SCHEMA_FAILURE, node=self), DbEventsFilter(db_event=DatabaseLogEvent.NO_SPACE_ERROR, node=self), DbEventsFilter(db_event=DatabaseLogEvent.FILESYSTEM_ERROR, node=self), DbEventsFilter(db_event=DatabaseLogEvent.RUNTIME_ERROR, node=self), ): stack.enter_context(db_filter) self.remoter.sudo(shell_script_cmd(f"""\ sed -e '/.*scylla/s/^/#/g' -i /etc/fstab sed -e '/auto_bootstrap:.*/s/false/true/g' -i /etc/scylla/scylla.yaml if ! grep ^replace_address_first_boot: /etc/scylla/scylla.yaml; then echo 'replace_address_first_boot: {self.ip_address}' | tee --append /etc/scylla/scylla.yaml fi """)) self._instance.stop() self._instance_wait_safe(self._instance.wait_until_stopped) self._instance.start() self._instance_wait_safe(self._instance.wait_until_running) self._wait_public_ip() self.log.debug("Got a new public IP: %s", self._instance.public_ip_address) self.refresh_ip_address() self.wait_ssh_up() if self.is_data_device_lost_after_reboot: self.stop_scylla_server(verify_down=False) # Moving var-lib-scylla.mount away, since scylla_create_devices fails if it already exists mount_path = "/etc/systemd/system/var-lib-scylla.mount" if self.remoter.sudo(f"test -e {mount_path}", ignore_status=True).ok: self.remoter.sudo(f"mv {mount_path} /tmp/") # The scylla_create_devices has been moved to the '/opt/scylladb' folder in the master branch. for create_devices_file in ("/usr/lib/scylla/scylla-ami/scylla_create_devices", "/opt/scylladb/scylla-ami/scylla_create_devices", "/opt/scylladb/scylla-machine-image/scylla_create_devices", ): if self.remoter.sudo(f"test -x {create_devices_file}", ignore_status=True).ok: self.remoter.sudo(create_devices_file) break else: raise IOError("scylla_create_devices file isn't found") self.start_scylla_server(verify_up=False) self.remoter.sudo(shell_script_cmd("""\ sed -e '/auto_bootstrap:.*/s/true/false/g' -i /etc/scylla/scylla.yaml sed -e 's/^replace_address_first_boot:/# replace_address_first_boot:/g' -i /etc/scylla/scylla.yaml """))
def install_prereqs(self, public_ip: str, connect_timeout: Optional[int] = None) -> None: from sdcm.cluster_docker import AIO_MAX_NR_RECOMMENDED_VALUE # pylint: disable=import-outside-toplevel LOGGER.info("Connecting instance...") remoter = self.get_remoter(host=public_ip, connect_timeout=connect_timeout) LOGGER.info("Installing required packages...") login_user = self.LOGIN_USER public_key = self.key_pair.public_key.decode() result = remoter.sudo(shell_script_cmd(quote="'", cmd=f"""\ # Make sure that cloud-init finished running. until [ -f /var/lib/cloud/instance/boot-finished ]; do sleep 1; done echo "fs.aio-max-nr = {AIO_MAX_NR_RECOMMENDED_VALUE}" >> /etc/sysctl.conf echo "{login_user} soft nofile 4096" >> /etc/security/limits.conf echo "jenkins soft nofile 4096" >> /etc/security/limits.conf echo "root soft nofile 4096" >> /etc/security/limits.conf # Configure default user account. sudo -u {login_user} mkdir -p /home/{login_user}/.ssh || true echo "{public_key}" >> /home/{login_user}/.ssh/authorized_keys chmod 600 /home/{login_user}/.ssh/authorized_keys mkdir -p -m 777 /home/{login_user}/sct-results echo "cd ~/sct-results" >> /home/{login_user}/.bashrc chown -R {login_user}:{login_user} /home/{login_user}/ # Disable apt-key warnings and set non-interactive frontend. export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 export DEBIAN_FRONTEND=noninteractive apt-get -qq clean apt-get -qq update apt-get -qq install --no-install-recommends python3-pip htop screen tree pip3 install awscli # Install Docker. apt-get -qq install --no-install-recommends \ apt-transport-https ca-certificates curl gnupg-agent software-properties-common curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" apt-get -qq install --no-install-recommends docker-ce docker-ce-cli containerd.io usermod -aG docker {login_user} # Install kubectl. curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl # Configure Jenkins user. apt-get -qq install --no-install-recommends openjdk-11-jre-headless # https://www.jenkins.io/doc/administration/requirements/java/ adduser --disabled-password --gecos "" jenkins || true usermod -aG docker jenkins mkdir -p /home/jenkins/.ssh echo "{public_key}" >> /home/jenkins/.ssh/authorized_keys chmod 600 /home/jenkins/.ssh/authorized_keys chown -R jenkins:jenkins /home/jenkins echo "jenkins ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/jenkins # Jenkins pipelines run /bin/sh for some reason. ln -sf /bin/bash /bin/sh """), ignore_status=True) remoter.stop() if result.ok: LOGGER.info("All packages successfully installed.") else: raise Exception( f"Unable to install required packages:\n{result.stdout}{result.stderr}" )
def test_shell_script_cmd(self): self.assertEqual(shell_script_cmd("true"), 'bash -cxe "true"')