def test_basic_stderr(self):
cmd = "echo 'testing stderr' 1>&2"
expected_output = "testing stderr\n"
out = system_utils.call(cmd)
self.assertEqual(expected_output, out)
with open("stdout.txt", "w") as test_out:
system_utils.call(cmd, stdout=test_out)
with open("stdout.txt", "r") as test_in:
self.assertEqual(expected_output, test_in.read())
def test_separate_stdout_stderr(self):
cmd = "echo 'testing stdout'; echo 'testing stderr' 1>&2"
with open("stderr.txt", "w") as test_err:
out = system_utils.call(cmd, stderr=test_err)
self.assertEqual("testing stdout\n", out)
with open("stderr.txt", "r") as test_in:
self.assertEqual("testing stderr\n", test_in.read())
def start_cluster(nodes):
# first = nodes[0]
# start_cockroach_node(first).wait()
processes = []
for i in range(len(nodes)):
# for node in nodes:
# start_cockroach_node(node, join=first["ip"]).wait()
# set_cluster_settings_on_single_node(first)
node = nodes[i]
processes.append(start_cockroach_node(node, other_urls=nodes))
for process in processes:
process.wait()
if len(nodes) > 1:
system_utils.call(
"/root/go/src/github.com/cockroachdb/cockroach/cockroach init "
"--insecure "
"--host={0}".format(nodes[0]["ip"]))
def main():
cfg_lt_pairs = process_and_setup_configs(DB_DIR, CONFIG_OBJ_LIST)
# db_connector = setup_sqlite_db(DB_DIR)
for cfg_fpath, lt_fpath in cfg_lt_pairs:
# generate config object
cfg = generate_configs.generate_configs_from_files_and_add_fields(
cfg_fpath)
# generate lt_config objects that match those config objects
lt_cfg = config_io.read_config_from_file(lt_fpath)
# make directory in which trial will be run
logs_dir = generate_dir_name(cfg[constants.CONFIG_FPATH_KEY], DB_DIR)
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
# copy over config into directory
system_utils.call("cp {0} {1}".format(cfg[constants.CONFIG_FPATH_KEY],
logs_dir))
# generate latency throughput trials
cfg = adjust_cfg(cfg)
lt_fpath_csv = latency_throughput.run(
cfg, lt_cfg, logs_dir, run_func=run_single_trial_wrapper)
# run trial
cfg["concurrency"] = latency_throughput.find_optimal_concurrency(
lt_fpath_csv)
cfg = adjust_cfg(cfg)
results_fpath_csv = run_single_trial_wrapper(cfg, logs_dir)
# insert results in sqlite db # THIS INSERTION METHOD ISN'T CORRECT
# insert_into_sqlite_db(db_connector,
# results_fpath_csv,
# logs_dir,
# cfg)
return 0
def run_kv_workload(client_nodes,
server_nodes,
concurrency,
keyspace,
warm_up_duration,
duration,
read_percent,
n_keys_per_statement,
skew,
log_dir,
mode=RunMode.WARMUP_AND_TRIAL_RUN):
server_urls = [
"postgresql://root@{0}:26257?sslmode=disable".format(n["ip"])
for n in server_nodes
]
# warmup and trial run commands are the same
args = [
"--concurrency {}".format(int(concurrency)),
"--read-percent={}".format(read_percent),
"--batch={}".format(n_keys_per_statement),
"--zipfian --s={}".format(skew), "--keyspace={}".format(keyspace)
]
# cmd = "{0} workload run kv {1} {2} --useOriginal=False".format(EXE, " ".join(server_urls), " ".join(args))
# initialize the workload from driver node
# for url in server_urls:
init_cmd = "{0} workload init kv {1}".format(EXE, server_urls[0])
# init_cmd = "{0} workload init kv {1}".format(EXE, url)
driver_node = client_nodes[0]
system_utils.call_remote(driver_node["ip"], init_cmd)
# set database settings
a_server_node = server_nodes[0]
settings_cmd = 'echo "alter range default configure zone using num_replicas = 1;" | ' \
'{0} sql --insecure --database=kv --url="postgresql://root@{1}?sslmode=disable"' \
.format(EXE, a_server_node["ip"])
system_utils.call_remote(driver_node["ip"], settings_cmd)
# prepopulate data
data_csv_leaf = "init_data.csv"
data_csv = os.path.join(constants.SCRATCH_DIR, data_csv_leaf)
populate_crdb_data.populate(data_csv, keyspace)
nfs_location = "data/{0}".format(data_csv_leaf)
upload_cmd = "{0} nodelocal upload {1} {2} --host={3} --insecure".format(
EXE, data_csv, nfs_location, a_server_node["ip"])
system_utils.call(upload_cmd)
import_cmd = 'echo "IMPORT INTO kv (k, v) CSV DATA(\\\"nodelocal://1/{1}\\\");" | ' \
"{0} sql --insecure --database=kv".format(EXE, nfs_location)
system_utils.call_remote(a_server_node["ip"], import_cmd)
if mode == RunMode.WARMUP_ONLY or mode == RunMode.WARMUP_AND_TRIAL_RUN:
# run warmup
# warmup_cmd = cmd + " --duration={}s".format(warm_up_duration)
warmup_processes = []
for i in range(len(client_nodes)):
node = client_nodes[i]
cmd = "{0} workload run kv {1} {2} --useOriginal=False".format(
EXE, server_urls[i % len(server_nodes)], " ".join(args))
warmup_cmd = cmd + " --duration={}s".format(warm_up_duration)
# for node in client_nodes:
individual_node_cmd = "sudo ssh {0} '{1}'".format(
node["ip"], warmup_cmd)
print(individual_node_cmd)
warmup_processes.append(
subprocess.Popen(shlex.split(individual_node_cmd)))
for wp in warmup_processes:
wp.wait()
if mode == RunMode.TRIAL_RUN_ONLY or mode == RunMode.WARMUP_AND_TRIAL_RUN:
# making the logs directory, if it doesn't already exist
log_fpath = os.path.join(log_dir, "logs")
if not os.path.exists(log_fpath):
os.makedirs(log_fpath)
# run trial
# trial_cmd = cmd + " --duration={}s".format(duration)
trial_processes = []
bench_log_files = []
for i in range(len(client_nodes)):
node = client_nodes[i]
cmd = "{0} workload run kv {1} {2} --useOriginal=False".format(
EXE, server_urls[i % len(server_nodes)], " ".join(args))
trial_cmd = cmd + " --duration={}s".format(duration)
# for node in client_nodes:
# logging output for each node
individual_log_fpath = os.path.join(
log_fpath, "bench_{}.txt".format(node["ip"]))
bench_log_files.append(individual_log_fpath)
# run command
individual_node_cmd = "sudo ssh {0} '{1}'".format(
node["ip"], trial_cmd)
print(individual_node_cmd)
with open(individual_log_fpath, "w") as f:
trial_processes.append(
subprocess.Popen(shlex.split(individual_node_cmd),
stdout=f))
for tp in trial_processes:
tp.wait()
return bench_log_files
def gnuplot(exe, *args):
args_str = " ".join(args)
cmd = "gnuplot -c {0} {1}".format(exe, args_str)
system_utils.call(cmd)
def main():
# takes arguments in the case of recovery
parser = argparse.ArgumentParser()
parser.add_argument("--recovery_mode", action="store_true")
parser.add_argument("--recovery_file")
parser.add_argument("--db_dir")
args = parser.parse_args()
assert_args_are_correct(args)
db_dir = args.db_dir if args.recovery_mode else DB_DIR
files_to_process = args.recovery_file if args.recovery_mode else os.path.join(
db_dir, "configs_to_process.csv")
if not args.recovery_mode:
# create the database and table
if not os.path.exists(db_dir):
os.makedirs(db_dir)
# populate configs to process
for cfg_obj, lt_fpath in CONFIG_OBJ_LIST:
cfg_fpath_list = cfg_obj.generate_all_config_files()
data = [{
constants.CONFIG_FPATH_KEY: cfg_fpath,
"lt_fpath": lt_fpath
} for cfg_fpath in cfg_fpath_list]
csv_utils.append_data_to_file(data, files_to_process)
# file of failed configs
failed_configs_csv = os.path.join(db_dir, "failed_configs.csv")
f = open(failed_configs_csv,
"w") # make sure it's only the failures from this round
f.close()
# connect to db
db = sqlite_helper_object.SQLiteHelperObject(
os.path.join(db_dir, "trials.db"))
db.connect()
_, cfg_lt_tuples = csv_utils.read_in_data_as_tuples(files_to_process,
has_header=False)
for cfg_fpath, lt_fpath in cfg_lt_tuples:
# generate config object
cfg = generate_configs.generate_configs_from_files_and_add_fields(
cfg_fpath)
# generate lt_config objects that match those config objects
lt_cfg = config_io.read_config_from_file(lt_fpath)
#try:
# make directory in which trial will be run
logs_dir = generate_dir_name(cfg[constants.CONFIG_FPATH_KEY], db_dir)
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
# copy over config into directory
system_utils.call("cp {0} {1}".format(cfg[constants.CONFIG_FPATH_KEY],
logs_dir))
# generate latency throughput trials
lt_fpath_csv = latency_throughput.run(cfg, lt_cfg, logs_dir)
# run trial
cfg["concurrency"] = latency_throughput.find_optimal_concurrency(
lt_fpath_csv)
results_fpath_csv = run_single_data_point.run(cfg, logs_dir)
# insert into sqlite db
# TODO get the actual commit hash, not the branch
db.insert_csv_data_into_sqlite_table(
"trials_table",
results_fpath_csv,
None,
logs_dir=logs_dir,
cockroach_commit=cfg["cockroach_commit"],
server_nodes=cfg["num_warm_nodes"],
disabled_cores=cfg["disable_cores"],
keyspace=cfg["keyspace"],
read_percent=cfg["read_percent"],
n_keys_per_statement=cfg["n_keys_per_statement"],
skews=cfg["skews"])
#except BaseException as e:
# print("Config {0} failed to run, continue with other configs. e:[{1}]"
# .format(cfg[constants.CONFIG_FPATH_KEY], e))
# csv_utils.append_data_to_file([{constants.CONFIG_FPATH_KEY: cfg[constants.CONFIG_FPATH_KEY],
# "lt_fpath": lt_fpath}],
# failed_configs_csv)
db.close()