def basic_statistics_per_batch():
out_path = "{}/prints/basic_statistics_per_batch.csv".format(script_dir)
with open(out_path, "w") as f:
f.write("str_dt," "cnt_servers," "cnt_valid_clients," "cnt_clients\n")
for str_dt in os.listdir("{}/prints".format(script_dir)):
if os.path.isdir("{}/prints/{}".format(script_dir, str_dt)):
servers = set()
cnt_clients = 0
cnt_valid_clients = 0
in_path = (
"{}/prints/{}/filtered/traceroute_per_mac.csv".format(
script_dir, str_dt))
df = pd.read_csv(in_path)
for _, row in df.iterrows():
cnt_clients += 1
for traceroute_type in \
unsupervised_utils.iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
if (row["valid_cnt_samples"]
and row[valid_traceroute_field]):
servers.add(row["server"])
cnt_valid_clients += 1
break
cnt_servers = len(servers)
l = "{}" + ",{}" * 3 + "\n"
l = l.format(str_dt, cnt_servers, cnt_valid_clients,
cnt_clients)
f.write(l)
def process_graphs(dt_start, dt_end):
str_dt = utils.get_str_dt(dt_start, dt_end)
out_dir = "{}/prints/{}/filtered/graph/".format(script_dir, str_dt)
utils.create_dirs([out_dir])
in_path = "{}/prints/{}/filtered/traceroute_per_mac.csv".format(
script_dir, str_dt)
servers = np.unique(pd.read_csv(in_path)["server"].values)
for traceroute_type in unsupervised_utils.iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
for server in servers:
utils.create_dirs([
"{}/prints/{}/filtered/graph/".format(script_dir, str_dt),
"{}/prints/{}/filtered/graph/{}".format(
script_dir, str_dt, server)
])
out_dir = "{}/prints/{}/filtered/graph/{}".format(
script_dir, str_dt, server)
out_path = "{}/{}_graph.gv".format(out_dir, traceroute_field)
name_neigh = get_graph(dt_start, dt_end, valid_traceroute_field,
traceroute_field, server)
write_graph(out_path, name_neigh)
check_graph(out_dir, name_neigh, traceroute_field)
def plot_latencies_traceroute(dt_start, dt_end, preprocess_args):
str_dt = utils.get_str_dt(dt_start, dt_end)
in_path = "{}/prints/{}/filtered/traceroute_per_mac.csv".format(script_dir,
str_dt)
df = pd.read_csv(in_path)
for _, row, in df.iterrows():
if row["valid_cnt_samples"]:
in_path = utils.get_in_path(row["server"], row["mac"], dt_start,
dt_end)
ts_traceroute = TimeSeries(in_path=in_path, metric="traceroute",
dt_start=dt_start, dt_end=dt_end)
for traceroute_type in unsupervised_utils.iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
if row[valid_traceroute_field]:
traceroute = ast.literal_eval(row[traceroute_field])
name_ts = get_ts_per_name(traceroute_type, ts_traceroute,
dt_start, dt_end)
dir_path = ("{}/plots/paths/{}/{}/{}/{}".
format(script_dir, str_dt, "latency",
traceroute_type, row["server"]))
traceroute_path = "/".join(map(str,
list(reversed(traceroute))))
dir_path = "{}/{}".format(dir_path, traceroute_path)
utils.create_dirs(["{}/traceroute_latencies/".
format(dir_path),
"{}/traceroute_latencies/{}".
format(dir_path, row["mac"])])
for i in range(len(traceroute) - 1):
name = traceroute[i][0][0]
traceroute_path = "hop{}_{}".format(str(i).zfill(2),
name)
out_path = ("{}/traceroute_latencies/{}/{}.png".
format(dir_path, row["mac"],
traceroute_path))
ts_preprocessed = name_ts[name].copy()
cp_utils.preprocess(ts_preprocessed, preprocess_args)
# plot_procedures.plot_ts_share_x(
# name_ts[name],
# ts_preprocessed,
# out_path,
# plot_type2="scatter",
# title1="raw",
# title2="median filtered",
# default_ylabel=True)
ts_preprocessed.metric = "latency"
plot_procedures.plot_ts(ts_preprocessed, out_path,
title="median filtered")
def get_first_hops(dt_start, dt_end, server, traceroute_type):
str_dt = utils.get_str_dt(dt_start, dt_end)
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
first_hops = set()
in_path = "{}/prints/{}/filtered/traceroute_per_mac.csv".format(script_dir,
str_dt)
df = pd.read_csv(in_path)
df = df[df["server"] == server]
for idx, row in df.iterrows():
if row["valid_cnt_samples"] and row[valid_traceroute_field]:
traceroute = ast.literal_eval(row[traceroute_field])
first_hops.add(traceroute[0])
return first_hops
def print_per_path(dt_start, dt_end, metric, file_name):
str_dt = utils.get_str_dt(dt_start, dt_end)
out_dir = "{}/change_point/unsupervised/".format(base_dir)
utils.create_dirs([
"{}/plots".format(out_dir), "{}/plots/paths".format(out_dir),
"{}/plots/paths/{}".format(out_dir, str_dt),
"{}/plots/paths/{}/{}".format(out_dir, str_dt, metric)
])
in_path = "{}/prints/{}/filtered/{}/{}".format(script_dir, str_dt, metric,
file_name)
for traceroute_type in iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
client_traceroute = cp_utils.get_client_traceroute(
dt_start, dt_end, traceroute_type)
path_dirs = set()
df = pd.read_csv(in_path)
for idx, row in df.iterrows():
client = utils.get_client(row["server"], row["mac"])
if client in client_traceroute:
traceroute = client_traceroute[client]
dir_path = "{}/plots/paths/{}/{}/{}/{}".format(
out_dir, str_dt, metric, traceroute_type, row["server"])
utils.create_dirs([dir_path])
for name in reversed(traceroute):
if name[0][0].split(".")[0] == "192":
continue
dir_path = "{}/{}".format(dir_path, name)
utils.create_dirs([dir_path])
out_path = "{}/{}".format(dir_path, file_name)
if dir_path not in path_dirs:
create_csv_with_same_header(out_path, df)
pd.DataFrame(row).T.to_csv(out_path,
mode="a",
header=False,
index=False)
path_dirs.add(dir_path)
def plot_per_name(dt_start, dt_end, metric, preprocess_args, plot_cps=True):
dt_dir = utils.get_dt_dir(dt_start, dt_end)
str_dt = utils.get_str_dt(dt_start, dt_end)
utils.create_dirs([
"{}/plots/".format(script_dir), "{}/plots/names".format(script_dir),
"{}/plots/names/{}".format(script_dir, str_dt),
"{}/plots/names/{}/{}".format(script_dir, str_dt, metric)
])
client_cps = unsupervised_utils.get_client_cps(plot_cps, str_dt, metric)
# avoid reploting
client_plotPath = {}
for traceroute_type in unsupervised_utils.iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
utils.create_dirs([
"{}/plots/names/{}/{}/{}".format(script_dir, str_dt, metric,
traceroute_type)
])
df = pd.read_csv("{}/prints/{}/filtered/traceroute_per_mac.csv".format(
script_dir, str_dt))
cnt = 0
for idx, row in df.iterrows():
if row["valid_cnt_samples"] and row[valid_traceroute_field]:
print("cnt={}, traceroute_type={}, str_dt={}".format(
cnt, traceroute_type, str_dt))
cnt += 1
client = utils.get_client(row["server"], row["mac"])
for name in cp_utils.iter_names_traceroute_filtered(
ast.literal_eval(row[traceroute_field])):
utils.create_dirs([
"{}/plots/names/{}/{}/{}/{}".format(
script_dir, str_dt, metric, traceroute_type,
row["server"]),
"{}/plots/names/{}/{}/{}/{}/{}".format(
script_dir, str_dt, metric, traceroute_type,
row["server"], name)
])
out_file_name = utils.get_out_file_name(
row["server"], row["mac"], dt_start, dt_end)
out_path = ("{}/plots/names/{}/{}/{}/{}/{}/{}.png".format(
script_dir, str_dt, metric, traceroute_type,
row["server"], name, out_file_name))
# avoid reploting
if client in client_plotPath:
shutil.copyfile(client_plotPath[client], out_path)
else:
client_plotPath[client] = out_path
cp_dts = client_cps[client]
in_path = "{}/input/{}/{}/{}.csv".format(
base_dir, dt_dir, row["server"], row["mac"])
ts = TimeSeries(in_path, metric, dt_start, dt_end)
cp_utils.preprocess(ts, preprocess_args)
plot_procedures.plot_ts(ts,
out_path,
dt_axvline=cp_dts,
title="median filtered")