secondary_files = driver.Files(metacluster,
db_path="db-2",
log_path="create-output-2",
executable_path=executable_path,
command_prefix=command_prefix)
secondary = driver.Process(cluster,
secondary_files,
log_path="serve-output-2",
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
primary.wait_until_started_up()
secondary.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", secondary.http_port)])
primary_dc = http.add_datacenter()
http.move_server_to_datacenter(primary.files.machine_name, primary_dc)
secondary_dc = http.add_datacenter()
http.move_server_to_datacenter(secondary.files.machine_name, secondary_dc)
ns = scenario_common.prepare_table_for_workload(opts,
http,
primary=primary_dc,
affinities={
primary_dc: 0,
secondary_dc: 1
})
http.set_namespace_ack_expectations(ns, {secondary_dc: 1})
http.wait_until_blueprint_satisfied(ns)
cluster.check()
http.check_no_issues()
files = [
driver.Files(metacluster,
db_path="db-%d" % i,
log_path="create-output-%d" % i)
for i in xrange(num_nodes)
]
processes = [
driver.Process(cluster,
files[i],
log_path="serve-output-%d" % i,
executable_path=driver.find_rethinkdb_executable())
for i in xrange(num_nodes)
]
time.sleep(3)
print "Creating table..."
http = http_admin.ClusterAccess([("localhost", p.http_port)
for p in processes])
db = http.add_database("test")
dc = http.add_datacenter()
for machine_id in http.machines:
http.move_server_to_datacenter(machine_id, dc)
ns = http.add_namespace(protocol="rdb",
primary=dc,
name="stress",
database=db)
time.sleep(3)
host, port = driver.get_namespace_host(ns.port, processes)
cluster.check()
print "Increasing replication factor..."
http.set_namespace_affinities(ns, {dc: 1})
time.sleep(3)
with driver.Metacluster() as metacluster:
cluster1 = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
print "Spinning up two processes..."
files1 = driver.Files(metacluster, log_path = "create-output-1",
executable_path = executable_path, command_prefix = command_prefix)
proc1 = driver.Process(cluster1, files1,
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
files2 = driver.Files(metacluster, log_path = "create-output-2",
executable_path = executable_path, command_prefix = command_prefix)
proc2 = driver.Process(cluster1, files2,
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
proc1.wait_until_started_up()
proc2.wait_until_started_up()
cluster1.check()
access1 = http_admin.ClusterAccess([("localhost", proc1.http_port)])
access2 = http_admin.ClusterAccess([("localhost", proc2.http_port)])
assert len(access1.get_directory()) == len(access2.get_directory()) == 2
print "Splitting cluster, then waiting 20s..."
cluster2 = driver.Cluster(metacluster)
metacluster.move_processes(cluster1, cluster2, [proc2])
time.sleep(20)
print "Checking that they detected the netsplit..."
assert len(access1.get_directory()) == len(access2.get_directory()) == 1
cluster1.check()
cluster2.check()
print "Joining cluster, then waiting 10s..."
metacluster.move_processes(cluster2, cluster1, [proc2])
time.sleep(10)
print "Checking that they detected the resolution..."
assert len(access1.get_directory()) == len(access2.get_directory()) == 2
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(
opts)
print "Starting cluster..."
files = driver.Files(metacluster,
log_path="create-output",
executable_path=executable_path,
command_prefix=command_prefix)
process = driver.Process(cluster,
files,
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
process.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", process.http_port)])
dc = http.add_datacenter()
http.move_server_to_datacenter(http.machines.keys()[0], dc)
ns = scenario_common.prepare_table_for_workload(opts, http, primary=dc)
http.wait_until_blueprint_satisfied(ns)
workload_ports = scenario_common.get_workload_ports(opts, ns, [process])
workload_runner.run(opts["protocol"], opts["workload1"], workload_ports,
opts["timeout"])
print "Restarting server..."
process.check_and_stop()
process2 = driver.Process(cluster,
files,
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
process2.wait_until_started_up()
op["timeout"] = IntFlag("--timeout", 600)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
print "Starting cluster..."
files1 = driver.Files(metacluster, db_path = "db-first", log_path = "create-output-first",
executable_path = executable_path, command_prefix = command_prefix)
process1 = driver.Process(cluster, files1, log_path = "serve-output-first",
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
process1.wait_until_started_up()
print "Creating namespace..."
http1 = http_admin.ClusterAccess([("localhost", process1.http_port)])
dc = http1.add_datacenter()
http1.move_server_to_datacenter(files1.machine_name, dc)
ns = scenario_common.prepare_table_for_workload(opts, http1, primary = dc)
http1.wait_until_blueprint_satisfied(ns)
workload_ports_1 = scenario_common.get_workload_ports(opts, ns, [process1])
workload_runner.run(opts["protocol"], opts["workload1"], workload_ports_1, opts["timeout"])
print "Bringing up new server..."
files2 = driver.Files(metacluster, db_path = "db-second", log_path = "create-output-second",
executable_path = executable_path, command_prefix = command_prefix)
process2 = driver.Process(cluster, files2, log_path = "serve-output-second",
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
process2.wait_until_started_up()
http1.update_cluster_data(3)
king_hamlet_files = driver.Files(metacluster,
machine_name="KingHamlet",
db_path="king-hamlet-db",
log_path="king-hamlet-create-output",
executable_path=executable_path,
command_prefix=command_prefix)
king_hamlet = driver.Process(cluster,
king_hamlet_files,
log_path="king-hamlet-log",
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
prince_hamlet.wait_until_started_up()
king_hamlet.wait_until_started_up()
cluster.check()
access = http_admin.ClusterAccess([("localhost", prince_hamlet.http_port)])
assert access.get_issues() == []
print "Killing one of them..."
king_hamlet.close()
time.sleep(1)
cluster.check()
print "Checking that the other has an issue..."
issues = access.get_issues()
assert len(issues) == 1
assert issues[0]["type"] == "MACHINE_DOWN"
print "Declaring it dead..."
access.declare_machine_dead(issues[0]["victim"])
time.sleep(1)
cluster.check()
print "Checking that the issue is gone..."
assert access.get_issues() == []
log_path="create-output-%d" % i,
executable_path=executable_path,
command_prefix=command_prefix),
log_path="serve_output-%d" % i,
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
for i in xrange(opts["sequence"].peak())
]
primary_process.wait_until_started_up()
for replica_process in replica_processes:
replica_process.wait_until_started_up()
print "Creating table..."
http = http_admin.ClusterAccess([
("localhost", p.http_port)
for p in [primary_process] + replica_processes
])
primary_dc = http.add_datacenter()
http.move_server_to_datacenter(primary_process.files.machine_name,
primary_dc)
replica_dc = http.add_datacenter()
for replica_process in replica_processes:
http.move_server_to_datacenter(replica_process.files.machine_name,
replica_dc)
ns = scenario_common.prepare_table_for_workload(
http,
primary=primary_dc,
affinities={
primary_dc: 0,
replica_dc: opts["sequence"].initial
})
access_machine_files = driver.Files(metacluster,
db_path="db-access",
log_path="create-db-access-output",
executable_path=executable_path,
command_prefix=command_prefix)
access_machine = driver.Process(cluster,
access_machine_files,
log_path="serve-output-access",
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
database_machine.wait_until_started_up
access_machine.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", database_machine.http_port)
])
database_dc = http.add_datacenter()
http.move_server_to_datacenter(database_machine.files.machine_name,
database_dc)
other_dc = http.add_datacenter()
http.move_server_to_datacenter(access_machine.files.machine_name, other_dc)
ns = scenario_common.prepare_table_for_workload(opts,
http,
primary=database_dc)
http.wait_until_blueprint_satisfied(ns)
cluster.check()
http.check_no_issues()
workload_ports = scenario_common.get_workload_ports(
opts, ns, [access_machine])
with workload_runner.ContinuousWorkload(opts["workload"], opts["protocol"],
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
files1 = driver.Files(metacluster, db_path = "db-1", log_path = "create-output-1",
executable_path = executable_path, command_prefix = command_prefix)
process1 = driver.Process(cluster, files1, log_path = "serve-output-1",
executable_path = executable_path, command_prefix = command_prefix,
extra_options = serve_options)
files2 = driver.Files(metacluster, db_path = "db-2", log_path = "create-output-2",
executable_path = executable_path, command_prefix = command_prefix)
process2 = driver.Process(cluster, files2, log_path = "serve-output-2",
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
process1.wait_until_started_up()
process2.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", p.http_port) for p in [process1, process2]])
dc1 = http.add_datacenter()
http.move_server_to_datacenter(process1.files.machine_name, dc1)
dc2 = http.add_datacenter()
http.move_server_to_datacenter(process2.files.machine_name, dc2)
ns = scenario_common.prepare_table_for_workload(opts, http, primary = dc1, affinities = {dc1: 0, dc2: 1})
http.wait_until_blueprint_satisfied(ns)
cluster.check()
http.check_no_issues()
workload_ports = scenario_common.get_workload_ports(opts, ns, [process1, process2])
with workload_runner.SplitOrContinuousWorkload(opts, opts["protocol"], workload_ports) as workload:
workload.run_before()
cluster.check()
http.check_no_issues()
workload.check()