) cluster.nodetool("flush") confirm = input("Has compaction finished? Input 'yes':") while confirm != 'yes': confirm = input("Has compaction finished? Input 'yes':") print("Run started at:", datetime.now().strftime("%H:%M:%S")) # Background load background_load = cs.loop_stress( f'mixed ratio\\(write=1,read=1\\) duration=5m cl=QUORUM -pop dist=UNIFORM\\(1..{ROW_COUNT}\\) -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=700 throttle={BACKGROUND_LOAD_OPS // loadgenerator_count}/s" -node {cluster_string}' ) iteration = Iteration(f'{profile_name}/replace-node', ignore_git=True) # Turn off node to be replaced. cluster.stop(load_index=(start_count - 1), erase_data=True) replace_node_start = datetime.now() # Start a replacement node if props['cluster_type'] == 'scylla': cluster = Scylla(new_node_public_ips, new_node_private_ips, all_private_ips[0], props) cluster.append_configuration( f"replace_address_first_boot: {cluster_private_ips[-1]}") cluster.start() else: cluster = Cassandra(new_node_public_ips, new_node_private_ips,
ops = props["ops"] profile = props["profile"] # Load information about the created machines. env = common.load_yaml('environment.yml') cluster_private_ips = env['cluster_private_ips'] cluster_string = ",".join(cluster_private_ips) # Start with a clear prometheus. prometheus.clear(env, props) # Restart to cluster to make sure the Scylla starts fresh # e.g. the memtable is flushed. scylla.restart_cluster(env['cluster_public_ips'], props['cluster_user'], props['ssh_options']) iteration = Iteration(props["benchmark_name"]) # Setup cassandra stress cs = CassandraStress(env['loadgenerator_public_ips'], props) cs.install() cs.prepare() cs.upload(profile) cs.stress( f'user profile=./{profile} "{ops}" duration={duration} -pop seq=1..{items} -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 maxPending=1024 -rate {rate} -node {cluster_string}' ) cs.collect_results(iteration.dir, warmup_seconds=warmup_seconds) prometheus.download_and_clear(env, props, iteration) print("Call 'unprovision_terraform.py' to destroy the created infrastructure!")
#!/bin/python3 import sys from scyllaso import common from scyllaso.common import Iteration from datetime import datetime from scyllaso import prometheus print("Test started at:", datetime.now().strftime("%H:%M:%S")) if len(sys.argv) < 2: raise Exception("Usage: ./download_prometheus.py [PROFILE_NAME]") profile_name = sys.argv[1] # Load properties props = common.load_yaml(f'{profile_name}.yml') env = common.load_yaml(f'environment_{profile_name}.yml') iteration = Iteration(f'{profile_name}/prometheus-dump', ignore_git=True) prometheus.download(env, props, iteration)
cs.install() cs.prepare() print("Loading started at:", datetime.now().strftime("%H:%M:%S")) cs.stress_seq_range( ROW_COUNT, 'write cl=QUORUM', f'-schema "replication(strategy=SimpleStrategy,replication_factor={REPLICATION_FACTOR})" "compaction(strategy={COMPACTION_STRATEGY})" -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=700 throttle=33000/s" -node {cluster_string}' ) print("Sleeping 2h") time.sleep(60 * 60 * 2) print("Run started at:", datetime.now().strftime("%H:%M:%S")) iteration = Iteration(f'{profile_name}/repair_no_errors', ignore_git=True) repair_start = datetime.now() print("Reparing node 0 started at:", datetime.now().strftime("%H:%M:%S")) cluster.nodetool("repair -full", 0) print("Reparing node 0 ended at:", datetime.now().strftime("%H:%M:%S")) repair_end = datetime.now() with open(f'{iteration.dir}/result.txt', 'a') as writer: writer.write( f'Reparing node 0 took (s): {(repair_end - repair_start).total_seconds()}\n' ) print("Run ended at:", datetime.now().strftime("%H:%M:%S"))
cluster.nodetool("flush") confirm = input("Has compaction finished? Input 'yes':") while confirm != 'yes': confirm = input("Has compaction finished? Input 'yes':") print("Run started at:", datetime.now().strftime("%H:%M:%S")) # Background load BACKGROUND_LOAD_OPS = props["background_total_load_ops"] // loadgenerator_count background_load = cs.loop_stress( f'mixed ratio\\(write=1,read=1\\) duration=5m cl=QUORUM -pop dist=UNIFORM\\(1..{ROW_COUNT}\\) -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=700 fixed={BACKGROUND_LOAD_OPS}/s" -node {cluster_string}' ) iteration = Iteration(f'{profile_name}/compact', ignore_git=True) compact_start = datetime.now() for i in range(len(cluster_public_ips)): print("Compacting node", i, "started at:", datetime.now().strftime("%H:%M:%S")) cluster.nodetool("compact", i) print("Compacting node", i, "ended at:", datetime.now().strftime("%H:%M:%S")) compact_end = datetime.now() with open(f'{iteration.dir}/result.txt', 'a') as writer: writer.write( f'Major compaction on all nodes took (s): {(compact_end - compact_start).total_seconds()}\n'
confirm = input("Has compaction finished? Input 'yes':") while confirm != 'yes': confirm = input("Has compaction finished? Input 'yes':") print("Run started at:", datetime.now().strftime("%H:%M:%S")) # Background load BACKGROUND_LOAD_OPS = props["background_total_load_ops"] // loadgenerator_count background_load = cs.loop_stress( f'mixed ratio\\(write=1,read=1\\) duration=5m cl=QUORUM -pop dist=UNIFORM\\(1..{ROW_COUNT}\\) -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=700 fixed={BACKGROUND_LOAD_OPS}/s" -node {cluster_string}' ) add_nodes_start = datetime.now() iteration = Iteration(f'{profile_name}/add-node', ignore_git=True) # Start Scylla/Cassandra nodes if props['cluster_type'] == 'scylla': s = Scylla(new_node_public_ips, new_node_private_ips, all_private_ips[0], props) s.start() else: cassandra = Cassandra(new_node_public_ips, new_node_private_ips, all_private_ips[0], props) cassandra.start() add_nodes_end = datetime.now() print("Run ended at:", datetime.now().strftime("%H:%M:%S")) print("Adding nodes took:", (add_nodes_end - add_nodes_start).total_seconds(),
cs.stress_seq_range(ROW_COUNT, 'write cl=QUORUM', f'-schema "replication(strategy=SimpleStrategy,replication_factor={REPLICATION_FACTOR})" "compaction(strategy={COMPACTION_STRATEGY})" -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=700 throttle={THROTTLE}/s" -node {cluster_string}') cluster.nodetool("flush") # Warm-up cache: 35k read/s for 180 minutes cs.stress(f'mixed ratio\\(write=0,read=1\\) duration=180m cl=QUORUM -pop dist=GAUSSIAN\\(1..{ROW_COUNT},{GAUSS_CENTER},{GAUSS_SIGMA}\\) -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=500 throttle=35000/s" -node {cluster_string}') print("Run started at:", datetime.now().strftime("%H:%M:%S")) rate = START_RATE while True: print("Run iteration started at:", datetime.now().strftime("%H:%M:%S")) iteration = Iteration(f'{profile_name}/cassandra-stress-{rate}', ignore_git=True) cs.stress(f'mixed ratio\\(write={WRITE_COUNT},read={READ_COUNT}\\) duration={DURATION_MINUTES}m cl=QUORUM -pop dist=GAUSSIAN\\(1..{ROW_COUNT},{GAUSS_CENTER},{GAUSS_SIGMA}\\) -log hdrfile=profile.hdr -graph file=report.html title=benchmark revision=benchmark-0 -mode native cql3 -rate "threads=500 fixed={rate // loadgenerator_count}/s" -node {cluster_string}') cs.collect_results(iteration.dir) if WRITE_COUNT > 0: write_profile_summary = parse_profile_summary_file(f'{iteration.dir}/profile-summary.txt', 'WRITE') print('WRITE_PROFILE', write_profile_summary) if READ_COUNT > 0: read_profile_summary = parse_profile_summary_file(f'{iteration.dir}/profile-summary.txt', 'READ') print('READ_PROFILE', read_profile_summary) with open(f'{iteration.dir}/parsed_profile_summary_file.txt', 'a') as writer: if WRITE_COUNT > 0: