def test_basic_stderr(self): cmd = "echo 'testing stderr' 1>&2" expected_output = "testing stderr\n" out = system_utils.call(cmd) self.assertEqual(expected_output, out) with open("stdout.txt", "w") as test_out: system_utils.call(cmd, stdout=test_out) with open("stdout.txt", "r") as test_in: self.assertEqual(expected_output, test_in.read())
def test_separate_stdout_stderr(self): cmd = "echo 'testing stdout'; echo 'testing stderr' 1>&2" with open("stderr.txt", "w") as test_err: out = system_utils.call(cmd, stderr=test_err) self.assertEqual("testing stdout\n", out) with open("stderr.txt", "r") as test_in: self.assertEqual("testing stderr\n", test_in.read())
def start_cluster(nodes): # first = nodes[0] # start_cockroach_node(first).wait() processes = [] for i in range(len(nodes)): # for node in nodes: # start_cockroach_node(node, join=first["ip"]).wait() # set_cluster_settings_on_single_node(first) node = nodes[i] processes.append(start_cockroach_node(node, other_urls=nodes)) for process in processes: process.wait() if len(nodes) > 1: system_utils.call( "/root/go/src/github.com/cockroachdb/cockroach/cockroach init " "--insecure " "--host={0}".format(nodes[0]["ip"]))
def main(): cfg_lt_pairs = process_and_setup_configs(DB_DIR, CONFIG_OBJ_LIST) # db_connector = setup_sqlite_db(DB_DIR) for cfg_fpath, lt_fpath in cfg_lt_pairs: # generate config object cfg = generate_configs.generate_configs_from_files_and_add_fields( cfg_fpath) # generate lt_config objects that match those config objects lt_cfg = config_io.read_config_from_file(lt_fpath) # make directory in which trial will be run logs_dir = generate_dir_name(cfg[constants.CONFIG_FPATH_KEY], DB_DIR) if not os.path.exists(logs_dir): os.makedirs(logs_dir) # copy over config into directory system_utils.call("cp {0} {1}".format(cfg[constants.CONFIG_FPATH_KEY], logs_dir)) # generate latency throughput trials cfg = adjust_cfg(cfg) lt_fpath_csv = latency_throughput.run( cfg, lt_cfg, logs_dir, run_func=run_single_trial_wrapper) # run trial cfg["concurrency"] = latency_throughput.find_optimal_concurrency( lt_fpath_csv) cfg = adjust_cfg(cfg) results_fpath_csv = run_single_trial_wrapper(cfg, logs_dir) # insert results in sqlite db # THIS INSERTION METHOD ISN'T CORRECT # insert_into_sqlite_db(db_connector, # results_fpath_csv, # logs_dir, # cfg) return 0
def run_kv_workload(client_nodes, server_nodes, concurrency, keyspace, warm_up_duration, duration, read_percent, n_keys_per_statement, skew, log_dir, mode=RunMode.WARMUP_AND_TRIAL_RUN): server_urls = [ "postgresql://root@{0}:26257?sslmode=disable".format(n["ip"]) for n in server_nodes ] # warmup and trial run commands are the same args = [ "--concurrency {}".format(int(concurrency)), "--read-percent={}".format(read_percent), "--batch={}".format(n_keys_per_statement), "--zipfian --s={}".format(skew), "--keyspace={}".format(keyspace) ] # cmd = "{0} workload run kv {1} {2} --useOriginal=False".format(EXE, " ".join(server_urls), " ".join(args)) # initialize the workload from driver node # for url in server_urls: init_cmd = "{0} workload init kv {1}".format(EXE, server_urls[0]) # init_cmd = "{0} workload init kv {1}".format(EXE, url) driver_node = client_nodes[0] system_utils.call_remote(driver_node["ip"], init_cmd) # set database settings a_server_node = server_nodes[0] settings_cmd = 'echo "alter range default configure zone using num_replicas = 1;" | ' \ '{0} sql --insecure --database=kv --url="postgresql://root@{1}?sslmode=disable"' \ .format(EXE, a_server_node["ip"]) system_utils.call_remote(driver_node["ip"], settings_cmd) # prepopulate data data_csv_leaf = "init_data.csv" data_csv = os.path.join(constants.SCRATCH_DIR, data_csv_leaf) populate_crdb_data.populate(data_csv, keyspace) nfs_location = "data/{0}".format(data_csv_leaf) upload_cmd = "{0} nodelocal upload {1} {2} --host={3} --insecure".format( EXE, data_csv, nfs_location, a_server_node["ip"]) system_utils.call(upload_cmd) import_cmd = 'echo "IMPORT INTO kv (k, v) CSV DATA(\\\"nodelocal://1/{1}\\\");" | ' \ "{0} sql --insecure --database=kv".format(EXE, nfs_location) system_utils.call_remote(a_server_node["ip"], import_cmd) if mode == RunMode.WARMUP_ONLY or mode == RunMode.WARMUP_AND_TRIAL_RUN: # run warmup # warmup_cmd = cmd + " --duration={}s".format(warm_up_duration) warmup_processes = [] for i in range(len(client_nodes)): node = client_nodes[i] cmd = "{0} workload run kv {1} {2} --useOriginal=False".format( EXE, server_urls[i % len(server_nodes)], " ".join(args)) warmup_cmd = cmd + " --duration={}s".format(warm_up_duration) # for node in client_nodes: individual_node_cmd = "sudo ssh {0} '{1}'".format( node["ip"], warmup_cmd) print(individual_node_cmd) warmup_processes.append( subprocess.Popen(shlex.split(individual_node_cmd))) for wp in warmup_processes: wp.wait() if mode == RunMode.TRIAL_RUN_ONLY or mode == RunMode.WARMUP_AND_TRIAL_RUN: # making the logs directory, if it doesn't already exist log_fpath = os.path.join(log_dir, "logs") if not os.path.exists(log_fpath): os.makedirs(log_fpath) # run trial # trial_cmd = cmd + " --duration={}s".format(duration) trial_processes = [] bench_log_files = [] for i in range(len(client_nodes)): node = client_nodes[i] cmd = "{0} workload run kv {1} {2} --useOriginal=False".format( EXE, server_urls[i % len(server_nodes)], " ".join(args)) trial_cmd = cmd + " --duration={}s".format(duration) # for node in client_nodes: # logging output for each node individual_log_fpath = os.path.join( log_fpath, "bench_{}.txt".format(node["ip"])) bench_log_files.append(individual_log_fpath) # run command individual_node_cmd = "sudo ssh {0} '{1}'".format( node["ip"], trial_cmd) print(individual_node_cmd) with open(individual_log_fpath, "w") as f: trial_processes.append( subprocess.Popen(shlex.split(individual_node_cmd), stdout=f)) for tp in trial_processes: tp.wait() return bench_log_files
def gnuplot(exe, *args): args_str = " ".join(args) cmd = "gnuplot -c {0} {1}".format(exe, args_str) system_utils.call(cmd)
def main(): # takes arguments in the case of recovery parser = argparse.ArgumentParser() parser.add_argument("--recovery_mode", action="store_true") parser.add_argument("--recovery_file") parser.add_argument("--db_dir") args = parser.parse_args() assert_args_are_correct(args) db_dir = args.db_dir if args.recovery_mode else DB_DIR files_to_process = args.recovery_file if args.recovery_mode else os.path.join( db_dir, "configs_to_process.csv") if not args.recovery_mode: # create the database and table if not os.path.exists(db_dir): os.makedirs(db_dir) # populate configs to process for cfg_obj, lt_fpath in CONFIG_OBJ_LIST: cfg_fpath_list = cfg_obj.generate_all_config_files() data = [{ constants.CONFIG_FPATH_KEY: cfg_fpath, "lt_fpath": lt_fpath } for cfg_fpath in cfg_fpath_list] csv_utils.append_data_to_file(data, files_to_process) # file of failed configs failed_configs_csv = os.path.join(db_dir, "failed_configs.csv") f = open(failed_configs_csv, "w") # make sure it's only the failures from this round f.close() # connect to db db = sqlite_helper_object.SQLiteHelperObject( os.path.join(db_dir, "trials.db")) db.connect() _, cfg_lt_tuples = csv_utils.read_in_data_as_tuples(files_to_process, has_header=False) for cfg_fpath, lt_fpath in cfg_lt_tuples: # generate config object cfg = generate_configs.generate_configs_from_files_and_add_fields( cfg_fpath) # generate lt_config objects that match those config objects lt_cfg = config_io.read_config_from_file(lt_fpath) #try: # make directory in which trial will be run logs_dir = generate_dir_name(cfg[constants.CONFIG_FPATH_KEY], db_dir) if not os.path.exists(logs_dir): os.makedirs(logs_dir) # copy over config into directory system_utils.call("cp {0} {1}".format(cfg[constants.CONFIG_FPATH_KEY], logs_dir)) # generate latency throughput trials lt_fpath_csv = latency_throughput.run(cfg, lt_cfg, logs_dir) # run trial cfg["concurrency"] = latency_throughput.find_optimal_concurrency( lt_fpath_csv) results_fpath_csv = run_single_data_point.run(cfg, logs_dir) # insert into sqlite db # TODO get the actual commit hash, not the branch db.insert_csv_data_into_sqlite_table( "trials_table", results_fpath_csv, None, logs_dir=logs_dir, cockroach_commit=cfg["cockroach_commit"], server_nodes=cfg["num_warm_nodes"], disabled_cores=cfg["disable_cores"], keyspace=cfg["keyspace"], read_percent=cfg["read_percent"], n_keys_per_statement=cfg["n_keys_per_statement"], skews=cfg["skews"]) #except BaseException as e: # print("Config {0} failed to run, continue with other configs. e:[{1}]" # .format(cfg[constants.CONFIG_FPATH_KEY], e)) # csv_utils.append_data_to_file([{constants.CONFIG_FPATH_KEY: cfg[constants.CONFIG_FPATH_KEY], # "lt_fpath": lt_fpath}], # failed_configs_csv) db.close()