def gather_over_time(config):
def write_out_stats(stats, out):
with open(out, "w") as f:
writer = csv.DictWriter(f,
delimiter='\t',
fieldnames=stats[0].keys())
writer.writeheader()
for stat in stats:
writer.writerow(stat)
return out
out_dir = os.path.join(config["out_dir"])
print("jenndebug", out_dir)
dir_path = os.path.join(out_dir, "skew-0")
for i in range(len(config["workload_nodes"])):
path = os.path.join(dir_path, "bench_out_{0}.txt".format(i))
stats = over_time.parse_file(path)
sorted(stats, key=lambda stat: stat["elapsed-r"])
filename = write_out_stats(
stats, os.path.join(out_dir, "stats_over_time_{0}.csv".format(i)))
print(filename)
csv_file = os.path.basename(
os.path.dirname(out_dir)) + "_stats_{0}.csv".format(i)
print(csv_file)
cmd = "mv {0} /usr/local/temp/go/src/github.com/cockroachdb/cockroach/gnuplot/{1}".format(
filename, csv_file)
lib.call_remote(DRIVER_NODE, cmd, "gather_time_err")
def move_logs(src_logs, dest_logs):
""" Moves logs from src to dest.
Args:
src_logs (str)
dest_logs (str)
Returns:
None.
"""
lib.call_remote("localhost", "mv {0} {1}".format(src_logs, dest_logs),
"unable to move logs")
def create_dir(location, dir_name):
""" Creates a directory at location.
Args:
location (str): absolute path name, directory location of new dir
dir_name (str): name of new directory.
Returns:
Absolute path of new directory.
"""
new_dir = os.path.join(location, dir_name)
lib.call_remote("localhost", "mkdir {0}".format(new_dir),
"unable to make new dir")
return new_dir
def disable_core(core_num, hostname):
""" Disables core with core_num on specified node.
:param core_num: core number to disable
:param hostname: specified node
:returns none
"""
if core_num >= 16:
raise AssertionError("Cannot specify core larger than 15")
elif core_num <= 0:
raise AssertionError("Cannot specify core 0 or under")
lib.call_remote(
hostname,
"echo 0 | tee /sys/devices/system/cpu/cpu{0}/online".format(core_num),
"could not disable core {0} on host {1}".format(core_num, hostname))
def generate_csv_file(
config,
skews,
accumulate_fn,
suffix,
driver_node=DRIVER_NODE,
csv_path="/usr/local/temp/go/src/github.com/cockroachdb/cockroach/gnuplot",
csv_file=None):
""" Generates csv file from skews.
Args:
config: experimental config
skews: take a guess buddy
Returns:
None.
"""
out_dir = os.path.join(config["out_dir"])
data = []
for i in range(len(skews)):
dir_path = os.path.join(out_dir, "skew-{0}".format(i))
datum, succeeded = accumulate_fn(config, dir_path)
if succeeded:
datum_with_skew = {"skew": skews[i]}
datum_with_skew.update(datum)
data.append(datum_with_skew)
else:
print("failed on skew[{0}]".format(skews[i]))
continue
filename = write_out_data_wrapper(data, out_dir, suffix + ".csv")
print(filename)
if csv_file is None:
csv_file = os.path.basename(
os.path.dirname(out_dir)) + "_" + suffix + ".csv"
print(csv_file)
cmd = "cp {0} {1}/{2}".format(filename, csv_path, csv_file)
lib.call_remote(driver_node, cmd, "i like to move it move it")