def plot_over(set_params, by=None, ax=None, title="Title", path=""):
path_candidates = sorted(Path(path).glob("*"))
dfs = []
for path in path_candidates:
suffix = str(path).split("/")[-1]
params = stem_to_params(suffix)
passing = False
for p, v in set_params.items():
if params[p] != v:
passing = True
if not passing:
reader = TidyReader(str(path) + "/data")
df = reader.read(
"pred_from_latent",
["Epoch", "Rank", "Step", "Value", "Metric", "Type", "Agent"],
)
df[by] = params[by]
dfs.append(df)
df = pd.concat(dfs)
df = df[(df["Epoch"] == 39999)
& (df["Metric"] == "Accuracy")
& (df["Type"] == "Latent")
& (df["Agent"] != "baseline_1")
& (df["Agent"] != "baseline_2")]
df["Title"] = title
return df
def read_single_latent(path):
reader = TidyReader(os.path.join(path, "data"))
df = reader.read(
tag="pred_from_latent",
columns=["Epoch", "Rank", "Step", "Value", "Metric", "Type", "Agent"],
)
data = clean_latent_data(df)
data = data[data["Step"] > 4800]
return data
def load_df_and_params(posixpath: PosixPath,
tag: str,
columns: List[str],
datafolder="data"):
"""
Args:
posixpath: Posixpath. Path to one specific epxeriment
tag: String. The name of the metric which we want to retrieve
columns: List[String]. The column headers of the resulting dataframe
Returns:
df: DataFrame
"""
reader = TidyReader(os.path.join(posixpath, datafolder))
df = reader.read(tag=tag, columns=columns)
params = stem_to_params(posixpath.name)
return df, params
def start_proc(experiment: BaseExperiment, cfg: Namespace, path: str,
rank: int, barrier: Barrier) -> None:
data_path = os.path.join(path, "data")
os.makedirs(data_path)
writer = MultiProcessingWriter(data_path, rank=rank)
reader = TidyReader(data_path)
exp = experiment(cfg, rank, writer, reader, path, barrier)
exp.start()
def start_procs_without_join(
cfg: Dict = {},
experiment: BaseExperiment = None,
path: str = "",
barrier: Barrier = None,
) -> List[Process]:
processes = []
data_path = os.path.join(path, "data")
os.makedirs(data_path)
for rank in range(cfg.nprocs):
writer = MultiProcessingWriter(data_path, rank=rank)
reader = TidyReader(data_path)
proc = mp.Process(
target=start_exp,
args=(experiment, copy.deepcopy(cfg), rank, writer, reader, path,
barrier),
)
# proc.start()
processes.append(proc)
return processes
def load_data(reader: TidyReader) -> Any:
df = reader.read(
tag="pred_from_latent",
columns=[
"Epoch", "Rank", "Step", "Value", "Metric", "Type", "Agent"
],
)
df = df[df.Epoch == 49999.0]
df = df[df.Metric == "Accuracy"]
df.Agent = df.Agent.map({
"A": "MA",
"B": "MA",
"C": "MA",
"baseline": "Baseline",
})
df = df.groupby(["Rank", "Agent", "Type"],
as_index=False).apply(lambda x: x[::4])
df_latent = df[df.Type == "Latent"]
df_rec = df[df.Type == "Reconstruction"]
return df_latent, df_rec
def start_procs(
fn: Callable,
cfg: Dict = {},
experiment: BaseExperiment = None,
path: str = "",
barrier: Barrier = None,
):
processes = []
data_path = os.path.join(path, "data")
os.makedirs(data_path)
for rank in range(cfg.nprocs):
writer = MultiProcessingWriter(data_path, rank=rank)
reader = TidyReader(data_path)
proc = mp.Process(
target=fn,
args=(experiment, copy.deepcopy(cfg), rank, writer, reader, path,
barrier),
)
proc.start()
processes.append(proc)
for proc in processes:
proc.join()
from functions import import_experiment_class
import os
import argparse
from reader.chunked_writer import TidyReader
parser = argparse.ArgumentParser()
parser.add_argument("exp")
parser.add_argument("path")
args = parser.parse_args()
exp_filename = args.exp.split("/")[-1].split(".")[0]
Experiment = import_experiment_class(exp_filename)
reader_path = os.path.join(args.path, "data")
plot_path = os.path.join(args.path, "plotting/plots")
if not os.path.exists(plot_path):
os.makedirs(plot_path)
reader = TidyReader(os.path.join(args.path, "data"))
dataframes = Experiment.load_data(reader)
Experiment.plot(dataframes, plot_path)
def read_single(path, tag, columns):
reader = TidyReader(os.path.join(path, "data"))
df = reader.read(tag=tag, columns=columns)
return df
def load_data(reader: TidyReader) -> Any:
return reader.read(
tag="loss",
columns=["Step", "Rank", "Loss", "Type", "Agent_i", "Agent_j"])
import os
from functions import merge_cfg_with_cli
from sweep_robustness import Experiment, Config
from reader.chunked_writer import MultiProcessingWriter, TidyReader
from argparse import ArgumentParser
cfg: Config = Config()
parser = ArgumentParser()
parser.add_argument("--path", type=str, required=True)
parser.add_argument("--rank", type=int, required=True)
merge_cfg_with_cli(cfg, parser)
args = parser.parse_args()
data_path = os.path.join(args.path, "data")
if not os.path.exists(data_path):
os.makedirs(data_path, exist_ok=True)
writer = MultiProcessingWriter(data_path, rank=args.rank)
reader = TidyReader(data_path)
exp = Experiment(args, args.rank, writer, reader, args.path)
exp.run(args)
def load_data(reader: TidyReader) -> Any:
return reader.read(columns=["Rank", "Step", "Agent", "Accuracy"])