def __prepare(self, ip, kill_java): log_machine(ip, f'Preparing: started') ssh = self.__new_ssh(ip) # we need to make sure that the no old load generator is still running. if kill_java: ssh.exec(f'killall -q -9 java || true') log_machine(ip, f'Preparing: done')
def __start_exporter(self, ip): log_machine(ip, 'Starting exporter') ssh = self.__new_ssh(ip) ssh.exec(f""" java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar ~/cassandra_exporter-2.3.5.jar ~/config.yml >> ~/cassandra_exporter.out 2>&1 & """) log_machine(ip, 'Starting exporter: done')
def __collect(self, ip, dir): dest_dir = os.path.join(dir, ip) os.makedirs(dest_dir, exist_ok=True) log_machine(ip, f'Collecting to [{dest_dir}]') ssh = self.__new_ssh(ip) ssh.scp_from_remote(f'*.{{html,hdr,log}}', dest_dir) ssh.exec(f'rm -fr *.html *.hdr *.log') log_machine(ip, f'Collecting to [{dest_dir}] done')
def start(self): log_important(f"Starting Cassandra nodes {self.cluster_public_ips}") for public_ip in self.cluster_public_ips: self.__start(public_ip) wait_for_cql_start(public_ip) log_machine(public_ip, f"""Node finished bootstrapping""") self.__start_exporter(public_ip) log_important( f"Starting Cassandra nodes {self.cluster_public_ips}: done")
def __download(self, ip, dir): dest_dir = os.path.join(dir, ip) os.makedirs(dest_dir, exist_ok=True) log_machine(ip, f'[{ip}] Downloading to [{dest_dir}]') self.__new_ssh(ip).scp_from_remote(f'diskplorer/*.{{svg,csv}}', dest_dir) if self.capture_lsblk: self.__new_ssh(ip).scp_from_remote(f'lsblk.out', dest_dir) log_machine(ip, f'Downloading to [{dest_dir}] done')
def start(self): log(f"Starting Scylla nodes {self.cluster_public_ips}") for public_ip in self.cluster_public_ips: ssh = self.__new_ssh(public_ip) ssh.exec("sudo systemctl start scylla-server") for public_ip in self.cluster_public_ips: wait_for_cql_start(public_ip) log_machine(public_ip, "Node finished bootstrapping") log(f"Starting Scylla nodes {self.cluster_public_ips}: done")
def __download(self, ip, dir): dest_dir = os.path.join(dir, ip) os.makedirs(dest_dir, exist_ok=True) log_machine(ip, f'Downloading to [{dest_dir}]') ssh = self.__new_ssh(ip) ssh.scp_from_remote(f'{self.dir_name}/*', dest_dir) if self.capture_lsblk: self.__new_ssh(ip).scp_from_remote(f'lsblk.out', dest_dir) log_machine(ip, f'Downloading to [{dest_dir}] done')
def __install(self, ip): log_machine(ip, 'Installing disk-explorer: started') ssh = self.__new_ssh(ip) ssh.update() ssh.install('git', 'fio', 'python3', 'python3-pip') ssh.exec(f""" sudo pip3 install -qqq matplotlib rm -fr diskplorer git clone -q https://github.com/scylladb/diskplorer.git """) log_machine(ip, 'Installing disk-explorer: done')
def __run(self, ip, options): log_machine(ip, 'fio: started') ssh = self.__new_ssh(ip) if self.capture_lsblk: ssh.exec(f'lsblk > lsblk.txt') ssh.exec(f""" mkdir -p {self.dir_name} cd {self.dir_name} sudo fio {options} """) log_machine(ip, 'fio: done')
def __install(self, ip): ssh = self.__new_ssh(ip) log_machine(ip, f'Installing scylla_bench: started') ssh.update() if self.performance_governor: ssh.set_governor("performance") ssh.install("golang") # ssh.exec("go get github.com/scylladb/scylla-bench") ssh.exec(f""" git clone https://github.com/scylladb/scylla-bench cd scylla-bench/ go install . """) log_machine(ip, f'Installing scylla_bench: done')
def __stop(self, ip): log_machine(ip, "Stopping Cassandra: started") ssh = self.__new_ssh(ip) path_prefix = 'cassandra-raid/' if self.setup_raid else './' ssh.exec(f""" set -e cd {path_prefix}apache-cassandra-{self.cassandra_version} if [ -f 'cassandra.pid' ]; then pid=$(cat cassandra.pid) kill $pid || true while kill -0 $pid; do sleep 1 done rm -f 'cassandra.pid' fi """) log_machine(ip, 'Stopping Cassandra: done')
def __run(self, ip, cmd): log_machine(ip, 'Run: started') ssh = self.__new_ssh(ip) ssh.exec('rm -fr diskplorer/*.svg') ssh.exec(f'rm -fr diskplorer/fiotest.tmp') if self.capture_lsblk: ssh.exec(f'lsblk > lsblk.out') ssh.exec(f""" ulimit -n 64000 ulimit -n ulimit -Sn 64000 ulimit -Sn cd diskplorer python3 diskplorer.py {cmd} """) # the file is 100 GB; so we want to remove it. ssh.exec(f'rm -fr diskplorer/fiotest.tmp') log_machine(ip, 'Run: done')
def __install(self, ip): ssh = self.__new_ssh(ip) ssh.update() log_machine(ip, "Installing Cassandra: started") ssh.install_one('openjdk-16-jdk', 'java-16-openjdk') ssh.install('wget') private_ip = self.__find_private_ip(ip) path_prefix = 'cassandra-raid/' if self.setup_raid else './' ssh.exec(f""" set -e if [ -d '{path_prefix}apache-cassandra-{self.cassandra_version}' ]; then echo Cassandra {self.cassandra_version} already installed. exit 0 fi wget -q -N https://archive.apache.org/dist/cassandra/{self.cassandra_version}/apache-cassandra-{self.cassandra_version}-bin.tar.gz tar -xzf apache-cassandra-{self.cassandra_version}-bin.tar.gz -C {path_prefix} wget -q https://github.com/criteo/cassandra_exporter/releases/download/2.3.5/cassandra_exporter-2.3.5.jar """) ssh.scp_to_remote( "jvm11-server.options", f"{path_prefix}apache-cassandra-{self.cassandra_version}/conf/jvm11-server.options" ) ssh.scp_to_remote( "cassandra.yaml", f"{path_prefix}apache-cassandra-{self.cassandra_version}/conf/cassandra.yaml" ) ssh.scp_to_remote("cassandra-exporter.yml", f"config.yml") # FIXME - heap in MB * 2 ssh.exec(""" sudo sh -c "echo 262144 > /proc/sys/vm/max_map_count" """) ssh.exec(f""" cd {path_prefix}apache-cassandra-{self.cassandra_version} sudo sed -i \"s/seeds:.*/seeds: {self.seed_private_ip} /g\" conf/cassandra.yaml sudo sed -i \"s/listen_address:.*/listen_address: {private_ip} /g\" conf/cassandra.yaml sudo sed -i \"s/rpc_address:.*/rpc_address: {private_ip} /g\" conf/cassandra.yaml """) log_machine(ip, "Installing Cassandra: done")
def __install(self, ip): ssh = self.__new_ssh(ip) ssh.update() if self.performance_governor: ssh.set_governor("performance") if self.scylla_tools: log_machine(ip, f'Installing cassandra-stress (Scylla): started') ssh.exec(f""" set -e if hash apt-get 2>/dev/null; then sudo apt-get install -y apt-transport-https sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 5e08fbd8b5d6ec9c sudo curl -L --output /etc/apt/sources.list.d/scylla.list http://downloads.scylladb.com/deb/ubuntu/scylla-4.5-$(lsb_release -s -c).list sudo apt-get update -y sudo apt-get install -y scylla-tools elif hash yum 2>/dev/null; then sudo yum install -y -q https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm sudo curl -o /etc/yum.repos.d/scylla.repo -L http://downloads.scylladb.com/rpm/centos/scylla-4.5.repo sudo yum install -y -q scylla-tools else echo "Cannot install scylla-tools: yum/apt not found" exit 1 fi """) else: log_machine(ip, f'Installing cassandra-stress (Cassandra): started') cassandra_version = self.properties['cassandra_version'] ssh.install_one('openjdk-8-jdk', 'java-1.8.0-openjdk') ssh.install('wget') ssh.exec(f""" set -e wget -q -N https://mirrors.netix.net/apache/cassandra/{cassandra_version}/apache-cassandra-{cassandra_version}-bin.tar.gz tar -xzf apache-cassandra-{cassandra_version}-bin.tar.gz """) log_machine(ip, f'Installing cassandra-stress: done')
def wait_for_cql_start(node_ip, timeout=7200, connect_timeout=10, max_tries_per_second=2): log_machine( node_ip, 'Waiting for CQL port to start (meaning node bootstrap finished). This could take a while.' ) backoff_interval = 1.0 / max_tries_per_second timeout_point = datetime.now() + timedelta(seconds=timeout) feedback_interval = 20 print_feedback_point = datetime.now() + timedelta( seconds=feedback_interval) while datetime.now() < timeout_point: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.settimeout(connect_timeout) try: sock.connect((node_ip, 9042)) except: # There was a problem connecting to CQL port. sleep(backoff_interval) if datetime.now() > print_feedback_point: print_feedback_point = datetime.now() + timedelta( seconds=feedback_interval) log_machine(node_ip, 'Still waiting for CQL port to start...') else: log_machine(node_ip, 'Successfully connected to CQL port.') return raise Exception( f'Waiting for CQL to start timed out after {timeout} seconds for node: {node_ip}.' )
def __prepare(self, ip): log_machine(ip, f'Preparing: started') ssh = self.__new_ssh(ip) # we need to make sure that the no old load generator is still running. ssh.exec(f'killall -q -9 go/bin/scylla-bench') log_machine(ip, f'Preparing: done')
def __install(self, ip): log_machine(ip, 'Installing fio: started') ssh = self.__new_ssh(ip) ssh.update() ssh.install('fio') log_machine(ip, 'Installing fio: done')