] if opts["use-proxy"]: proxy_process = driver.ProxyProcess(cluster, 'proxy-logfile', log_path='proxy-output', executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) processes.append(proxy_process) for process in processes: process.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", p.http_port) for p in processes]) dc = http.add_datacenter() for machine_id in http.machines: http.move_server_to_datacenter(machine_id, dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary=dc) for i in xrange(opts["num-shards"] - 1): http.add_namespace_shard(ns, chr(ord('a') + 26 * i // opts["num-shards"])) http.wait_until_blueprint_satisfied(ns) workload_ports = scenario_common.get_workload_ports( opts, ns, processes if not opts["use-proxy"] else [proxy_process]) workload_runner.run(opts["protocol"], opts["workload"], workload_ports, opts["timeout"]) cluster.check_and_stop()
print "Starting cluster..." files1 = driver.Files(metacluster, db_path = "db-first", log_path = "create-output-first", executable_path = executable_path, command_prefix = command_prefix) process1 = driver.Process(cluster, files1, log_path = "serve-output-first", executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) process1.wait_until_started_up() print "Creating namespace..." http1 = http_admin.ClusterAccess([("localhost", process1.http_port)]) dc = http1.add_datacenter() http1.move_server_to_datacenter(files1.machine_name, dc) ns = scenario_common.prepare_table_for_workload(opts, http1, primary = dc) http1.wait_until_blueprint_satisfied(ns) workload_ports_1 = scenario_common.get_workload_ports(opts, ns, [process1]) workload_runner.run(opts["protocol"], opts["workload1"], workload_ports_1, opts["timeout"]) print "Bringing up new server..." files2 = driver.Files(metacluster, db_path = "db-second", log_path = "create-output-second", executable_path = executable_path, command_prefix = command_prefix) process2 = driver.Process(cluster, files2, log_path = "serve-output-second", executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) process2.wait_until_started_up() http1.update_cluster_data(3) http1.move_server_to_datacenter(files2.machine_name, dc) http1.set_namespace_affinities(ns, {dc: 1}) http1.check_no_issues() print "Waiting for backfill..." backfill_start_time = time.time()
secondary = driver.Process(cluster, secondary_files, log_path = "serve-output-secondary", executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) secondary.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", primary.http_port)]) primary_dc = http.add_datacenter() http.move_server_to_datacenter(primary.files.machine_name, primary_dc) secondary_dc = http.add_datacenter() http.move_server_to_datacenter(secondary.files.machine_name, secondary_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary = primary_dc, affinities = {primary_dc: 0, secondary_dc: 1}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(opts, ns, [primary]) with workload_runner.SplitOrContinuousWorkload(opts, opts["protocol"], workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() print "Killing the secondary..." secondary.close() time.sleep(30) # Wait for the other node to notice it's dead http.declare_machine_dead(secondary.files.machine_name) http.set_namespace_affinities(ns, {secondary_dc: 0}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload.run_after() http.check_no_issues()
executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) process1.wait_until_started_up() print "Creating table..." http1 = http_admin.ClusterAccess([("localhost", process1.http_port)]) dc = http1.add_datacenter() http1.move_server_to_datacenter(files1.machine_name, dc) ns = scenario_common.prepare_table_for_workload(http1, primary=dc) http1.wait_until_blueprint_satisfied(ns) rdb_workload_common.wait_for_table(host='localhost', port=process1.driver_port, table=ns.name) workload_ports_1 = scenario_common.get_workload_ports(ns, [process1]) workload_runner.run(opts["workload1"], workload_ports_1, opts["timeout"]) print "Bringing up new server..." files2 = driver.Files(metacluster, db_path="db-second", log_path="create-output-second", executable_path=executable_path, command_prefix=command_prefix) process2 = driver.Process(cluster, files2, log_path="serve-output-second", executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) process2.wait_until_started_up()
executable_path=executable_path, command_prefix=command_prefix) serve_process = driver.Process(cluster, serve_files, log_path="serve-output", executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) proxy_process = driver.ProxyProcess(cluster, 'proxy-logfile', log_path='proxy-output', executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) processes = [serve_process, proxy_process] for process in processes: process.wait_until_started_up() print "Creating table..." http = http_admin.ClusterAccess([("localhost", proxy_process.http_port)]) dc = http.add_datacenter() for machine_id in http.machines: http.move_server_to_datacenter(machine_id, dc) ns = scenario_common.prepare_table_for_workload(http, primary=dc) http.wait_until_blueprint_satisfied(ns) workload_ports = scenario_common.get_workload_ports(ns, [proxy_process]) workload_runner.run(opts["workload"], workload_ports, opts["timeout"]) cluster.check_and_stop()
log_path="create-output", executable_path=executable_path, command_prefix=command_prefix) process = driver.Process(cluster, files, executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) process.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", process.http_port)]) dc = http.add_datacenter() http.move_server_to_datacenter(http.machines.keys()[0], dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary=dc) http.wait_until_blueprint_satisfied(ns) workload_ports = scenario_common.get_workload_ports(opts, ns, [process]) workload_runner.run(opts["protocol"], opts["workload1"], workload_ports, opts["timeout"]) print "Restarting server..." process.check_and_stop() process2 = driver.Process(cluster, files, executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) process2.wait_until_started_up() http.wait_until_blueprint_satisfied(ns) workload_runner.run(opts["protocol"], opts["workload2"], workload_ports, opts["timeout"]) cluster.check_and_stop()
http.move_server_to_datacenter(primary.files.machine_name, primary_dc) secondary_dc = http.add_datacenter() http.move_server_to_datacenter(secondary.files.machine_name, secondary_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary=primary_dc, affinities={ primary_dc: 0, secondary_dc: 1 }) http.set_namespace_ack_expectations(ns, {secondary_dc: 1}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(opts, ns, [secondary]) with workload_runner.SplitOrContinuousWorkload(opts, opts["protocol"], workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() print "Killing the primary..." primary.close() http.declare_machine_dead(primary.files.machine_name) http.move_namespace_to_datacenter(ns, secondary_dc) http.set_namespace_affinities(ns, {secondary_dc: 0}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload.run_after()
op["timeout"] = IntFlag("--timeout", 600) opts = op.parse(sys.argv) with driver.Metacluster() as metacluster: cluster = driver.Cluster(metacluster) executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts) print "Starting cluster..." serve_files = driver.Files(metacluster, db_path = "db", log_path = "create-output", executable_path = executable_path, command_prefix = command_prefix) serve_process = driver.Process(cluster, serve_files, log_path = "serve-output", executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) proxy_process = driver.ProxyProcess(cluster, 'proxy-logfile', log_path = 'proxy-output', executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) processes = [serve_process, proxy_process] for process in processes: process.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", proxy_process.http_port)]) dc = http.add_datacenter() for machine_id in http.machines: http.move_server_to_datacenter(machine_id, dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary = dc) http.wait_until_blueprint_satisfied(ns) workload_ports = scenario_common.get_workload_ports(opts, ns, [proxy_process]) workload_runner.run(opts["protocol"], opts["workload"], workload_ports, opts["timeout"]) cluster.check_and_stop()
print "Starting cluster..." processes = [driver.Process(cluster, driver.Files(metacluster, db_path = "db-%d" % i, log_path = "create-output-%d" % i, executable_path = executable_path, command_prefix = command_prefix), log_path = "serve-output-%d" % i, executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) for i in xrange(opts["num-nodes"])] if opts["use-proxy"]: proxy_process = driver.ProxyProcess(cluster, 'proxy-logfile', log_path = 'proxy-output', executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) processes.append(proxy_process) for process in processes: process.wait_until_started_up() print "Creating table..." http = http_admin.ClusterAccess([("localhost", p.http_port) for p in processes]) dc = http.add_datacenter() for machine_id in http.machines: http.move_server_to_datacenter(machine_id, dc) ns = scenario_common.prepare_table_for_workload(http, primary = dc) for i in xrange(opts["num-shards"] - 1): http.add_table_shard(ns, chr(ord('a') + 26 * i // opts["num-shards"])) http.wait_until_blueprint_satisfied(ns) workload_ports = scenario_common.get_workload_ports(ns, processes if not opts["use-proxy"] else [proxy_process]) workload_runner.run(opts["workload"], workload_ports, opts["timeout"]) cluster.check_and_stop()
print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", database_machine.http_port) ]) database_dc = http.add_datacenter() http.move_server_to_datacenter(database_machine.files.machine_name, database_dc) other_dc = http.add_datacenter() http.move_server_to_datacenter(access_machine.files.machine_name, other_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary=database_dc) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports( opts, ns, [access_machine]) with workload_runner.ContinuousWorkload(opts["workload"], opts["protocol"], workload_ports) as workload: workload.start() print "Running workload for 10 seconds..." time.sleep(10) cluster.check() http.check_no_issues() print "Killing the access machine..." access_machine.close() # Don't bother stopping the workload, just exit and it will get killed http.declare_machine_dead(access_machine.files.machine_name) http.check_no_issues() database_machine.check() print "The other machine dosn't seem to have crashed."
replica_dc = http.add_datacenter() for replica_process in replica_processes: http.move_server_to_datacenter(replica_process.files.machine_name, replica_dc) ns = scenario_common.prepare_table_for_workload( http, primary=primary_dc, affinities={ primary_dc: 0, replica_dc: opts["sequence"].initial }) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports( ns, [primary_process] + replica_processes) with workload_runner.SplitOrContinuousWorkload(opts, workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() workload.check() current = opts["sequence"].initial for i, s in enumerate(opts["sequence"].steps): if i != 0: workload.run_between() print "Changing the number of secondaries from %d to %d..." % ( current, current + s) current += s http.set_table_affinities(ns, {primary_dc: 0, replica_dc: current}) http.wait_until_blueprint_satisfied(ns, timeout=3600)
files = driver.Files(metacluster, log_path="create-output", executable_path=executable_path, command_prefix=command_prefix) process = driver.Process(cluster, files, executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) process.wait_until_started_up() print "Creating table..." with r.connect('localhost', process.driver_port) as conn: r.db_create('test').run(conn) r.db('test').table_create('restart').run(conn) ns = TableShim(name='restart') workload_ports = scenario_common.get_workload_ports(ns, [process]) workload_runner.run(opts["workload1"], workload_ports, opts["timeout"]) print "Restarting server..." process.check_and_stop() process2 = driver.Process(cluster, files, executable_path=executable_path, command_prefix=command_prefix, extra_options=serve_options) process2.wait_until_started_up() cluster.check() rdb_workload_common.wait_for_table(host="localhost", port=process2.driver_port, table=ns.name) workload_ports2 = scenario_common.get_workload_ports(ns, [process2]) workload_runner.run(opts["workload2"], workload_ports2, opts["timeout"])
print("Establishing ReQL connection (%.2fs)" % (time.time() - startTime)) conn = r.connect(host=server.host, port=server.driver_port) print("Creating table (%.2fs)" % (time.time() - startTime)) if not dbName in r.db_list().run(conn): r.db_create(dbName).run(conn) if tableName in r.db(dbName).table_list().run(conn): r.db(dbName).table_drop(tableName).run(conn) r.db(dbName).table_create(tableName).run(conn) print("Running first workload (%.2fs)" % (time.time() - startTime)) workload_ports = scenario_common.get_workload_ports([server], tableName, dbName) workload_runner.run(opts["workload1"], workload_ports, opts["timeout"]) print("Restarting server (%.2fs)" % (time.time() - startTime)) server.check_and_stop() serverRestarted = driver.Process(cluster, files, command_prefix=command_prefix, extra_options=serve_options, wait_until_ready=True) conn = r.connect(host=serverRestarted.host, port=serverRestarted.driver_port) r.db(dbName).table(tableName).wait().run(conn) cluster.check()
break # we can only handle one elif option.startswith('--cache-size='): serve_options.remove(option) proxy_process = driver.ProxyProcess(server.cluster, 'proxy-logfile', console_output='proxy-output', command_prefix=command_prefix, extra_options=serve_options) server.cluster.wait_until_ready() print("Creating db/table %s/%s (%.2fs)" % (dbName, tableName, time.time() - startTime)) if dbName not in r.db_list().run(conn): r.db_create(dbName).run(conn) if tableName in r.db(dbName).table_list().run(conn): r.db(dbName).table_drop(tableName).run(conn) r.db(dbName).table_create(tableName).run(conn) print("Starting workload: %s (%.2fs)" % (opts["workload"], time.time() - startTime)) workload_ports = scenario_common.get_workload_ports([proxy_process], tableName, dbName) workload_runner.run(opts["workload"], workload_ports, opts["timeout"]) print("Ended workload: %s (%.2fs)" % (opts["workload"], time.time() - startTime)) print("Done (%.2fs)" % (time.time() - startTime))
primary.wait_until_started_up() secondary.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", secondary.http_port)]) primary_dc = http.add_datacenter() http.move_server_to_datacenter(primary.files.machine_name, primary_dc) secondary_dc = http.add_datacenter() http.move_server_to_datacenter(secondary.files.machine_name, secondary_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary = primary_dc, affinities = {primary_dc: 0, secondary_dc: 1}) http.set_namespace_ack_expectations(ns, {secondary_dc: 1}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(opts, ns, [secondary]) with workload_runner.SplitOrContinuousWorkload(opts, opts["protocol"], workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() print "Killing the primary..." primary.close() http.declare_machine_dead(primary.files.machine_name) http.move_namespace_to_datacenter(ns, secondary_dc) http.set_namespace_affinities(ns, {secondary_dc: 0}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload.run_after() http.check_no_issues()
primary_dc = http.add_datacenter() http.move_server_to_datacenter(primary.files.machine_name, primary_dc) secondary_dc = http.add_datacenter() http.move_server_to_datacenter(secondary.files.machine_name, secondary_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary=primary_dc, affinities={ primary_dc: 0, secondary_dc: 1 }) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(opts, ns, [primary]) with workload_runner.SplitOrContinuousWorkload(opts, opts["protocol"], workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() print "Killing the secondary..." secondary.close() time.sleep(30) # Wait for the other node to notice it's dead http.declare_machine_dead(secondary.files.machine_name) http.set_namespace_affinities(ns, {secondary_dc: 0}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload.run_after()
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) database_machine.wait_until_started_up access_machine.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", database_machine.http_port)]) database_dc = http.add_datacenter() http.move_server_to_datacenter(database_machine.files.machine_name, database_dc) other_dc = http.add_datacenter() http.move_server_to_datacenter(access_machine.files.machine_name, other_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary = database_dc) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(opts, ns, [access_machine]) with workload_runner.ContinuousWorkload(opts["workload"], opts["protocol"], workload_ports) as workload: workload.start() print "Running workload for 10 seconds..." time.sleep(10) cluster.check() http.check_no_issues() print "Killing the access machine..." access_machine.close() # Don't bother stopping the workload, just exit and it will get killed http.declare_machine_dead(access_machine.files.machine_name) http.check_no_issues() database_machine.check() print "The other machine dosn't seem to have crashed."
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) process1.wait_until_started_up() process2.wait_until_started_up() print "Creating table..." http = http_admin.ClusterAccess([("localhost", p.http_port) for p in [process1, process2]]) dc1 = http.add_datacenter() http.move_server_to_datacenter(process1.files.machine_name, dc1) dc2 = http.add_datacenter() http.move_server_to_datacenter(process2.files.machine_name, dc2) ns = scenario_common.prepare_table_for_workload(http, primary = dc1, affinities = {dc1: 0, dc2: 1}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(ns, [process1, process2]) with workload_runner.SplitOrContinuousWorkload(opts, workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() workload.check() print "Changing the primary..." http.set_table_affinities(ns, {dc1: 1, dc2: 1}) http.move_table_to_datacenter(ns, dc2) http.set_table_affinities(ns, {dc1: 1, dc2: 0}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload.run_after() http.check_no_issues()
for process in processes: process.wait_until_started_up() print "Creating table..." http = http_admin.ClusterAccess([("localhost", p.http_port) for p in processes]) primary_dc = http.add_datacenter() secondary_dc = http.add_datacenter() machines = http.machines.keys() http.move_server_to_datacenter(machines[0], primary_dc) http.move_server_to_datacenter(machines[1], secondary_dc) ns = scenario_common.prepare_table_for_workload(http, primary = primary_dc) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(ns, processes) with workload_runner.SplitOrContinuousWorkload(opts, workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() http.move_table_to_datacenter(ns, secondary_dc) http.wait_until_blueprint_satisfied(ns) rdb_workload_common.wait_for_table(host=workload_ports.host, port=workload_ports.rdb_port, table=workload_ports.table_name) cluster.check() http.check_no_issues() workload.run_after() cluster.check_and_stop()
for replica_process in replica_processes: replica_process.wait_until_started_up() print "Creating namespace..." http = http_admin.ClusterAccess([("localhost", p.http_port) for p in [primary_process] + replica_processes]) primary_dc = http.add_datacenter() http.move_server_to_datacenter(primary_process.files.machine_name, primary_dc) replica_dc = http.add_datacenter() for replica_process in replica_processes: http.move_server_to_datacenter(replica_process.files.machine_name, replica_dc) ns = scenario_common.prepare_table_for_workload(opts, http, primary = primary_dc, affinities = {primary_dc: 0, replica_dc: opts["sequence"].initial}) http.wait_until_blueprint_satisfied(ns) cluster.check() http.check_no_issues() workload_ports = scenario_common.get_workload_ports(opts, ns, [primary_process] + replica_processes) with workload_runner.SplitOrContinuousWorkload(opts, opts["protocol"], workload_ports) as workload: workload.run_before() cluster.check() http.check_no_issues() workload.check() current = opts["sequence"].initial for i, s in enumerate(opts["sequence"].steps): if i != 0: workload.run_between() print "Changing the number of secondaries from %d to %d..." % (current, current + s) current += s http.set_namespace_affinities(ns, {primary_dc: 0, replica_dc: current}) http.wait_until_blueprint_satisfied(ns, timeout = 3600) cluster.check() http.check_no_issues()
print "Starting cluster..." files1 = driver.Files(metacluster, db_path = "db-first", log_path = "create-output-first", executable_path = executable_path, command_prefix = command_prefix) process1 = driver.Process(cluster, files1, log_path = "serve-output-first", executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) process1.wait_until_started_up() print "Creating table..." http1 = http_admin.ClusterAccess([("localhost", process1.http_port)]) dc = http1.add_datacenter() http1.move_server_to_datacenter(files1.machine_name, dc) ns = scenario_common.prepare_table_for_workload(http1, primary = dc) http1.wait_until_blueprint_satisfied(ns) rdb_workload_common.wait_for_table(host='localhost', port=process1.driver_port, table=ns.name) workload_ports_1 = scenario_common.get_workload_ports(ns, [process1]) workload_runner.run(opts["workload1"], workload_ports_1, opts["timeout"]) print "Bringing up new server..." files2 = driver.Files(metacluster, db_path = "db-second", log_path = "create-output-second", executable_path = executable_path, command_prefix = command_prefix) process2 = driver.Process(cluster, files2, log_path = "serve-output-second", executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options) process2.wait_until_started_up() http1.update_cluster_data(3) http1.move_server_to_datacenter(files2.machine_name, dc) http1.set_table_affinities(ns, {dc: 1}) http1.check_no_issues() print "Waiting for backfill..." backfill_start_time = time.time()