def __init__(self, agent_info, env_info, experiment_info):
super().__init__()
self.agent_info = agent_info
self.env_info = env_info
self.experiment_info = experiment_info
self.agent = agents.get_agent(agent_info.get("algorithm"))
self.env = envs.get_environment(env_info.get("env"))
self.num_episodes = experiment_info.get("n_episodes")
self.episode_eval_freq = experiment_info.get("episode_eval_freq")
self.id = experiment_info.get("id")
self.max_episode_steps = experiment_info.get("max_episode_steps")
self.output_dir = Path(experiment_info.get("output_dir")).expanduser()
self.output_dir = path_exists(self.output_dir)
self.true_values = np.load(self.output_dir / "true_values.npy")
self.obs = np.load(self.output_dir / "states.npy")
self.num_obs = len(self.obs)
self.on_policy_dist = np.ones(self.num_obs) * 1 / self.num_obs
self.msve_error = np.zeros(
(self.num_episodes // self.episode_eval_freq + 1, ))
self.log_episodes = self.env_info.get("log_episodes")
if self.log_episodes:
self.episodes = [[]
for i in range(self.experiment_info.get("runs"))]
self.objective = get_objective(
"RMSVE",
self.true_values,
self.on_policy_dist,
np.ones(self.num_obs),
)
def _unlog_uncheckpointed_episodes_helper(self, folder_name,
filename_pattern,
num_logged_episodes):
if utils.path_exists(folder_name):
for task_id in range(self.num_tasks):
reward_episodes_file = filename_pattern % task_id
reward_episodes_file_path = os.path.join(
folder_name, reward_episodes_file)
if utils.path_exists(reward_episodes_file_path):
try:
df = pd.read_csv(reward_episodes_file_path,
nrows=num_logged_episodes,
sep=';',
header=None)
df.to_csv(reward_episodes_file_path,
sep=';',
index=False,
header=None)
except pd.errors.EmptyDataError:
pass
def init(self):
FR = get_representation(name=self.agent_info.get("representations"),
**self.agent_info)
self.representations = np.array([
FR[self.states[i]] for i in range(len(self.states))
]).reshape(len(self.states), FR.num_features)
if self.experiment_info.get("save_representations"):
path = path_exists(self.output_dir / "representations")
self.save(path / f"repr_{self.id}", self.representations)
self.rl_glue = RLGlue(self.env, self.agent)
self.rl_glue.rl_init(self.agent_info, self.env_info)
def __init__(self, agent_info, env_info, experiment_info):
super().__init__()
self.agent_info = agent_info
self.env_info = env_info
self.experiment_info = experiment_info
self.agent = agents.get_agent(agent_info.get("algorithm"))
self.N = env_info["num_states"]
self.env = envs.get_environment(env_info.get("env"))
self.n_episodes = experiment_info.get("n_episodes")
self.episode_eval_freq = experiment_info.get("episode_eval_freq")
self.id = experiment_info.get("id")
self.max_episode_steps = experiment_info.get("max_episode_steps")
self.output_dir = Path(experiment_info.get("output_dir")).expanduser()
self.initial_seed = experiment_info.get("seed")
path_exists(self.output_dir)
path_exists(self.output_dir / "logs")
self.logger = get_simple_logger(
__name__, self.output_dir / "logs" / f"{self.id}.txt")
self.logger.info(
json.dumps([self.agent_info, self.env_info, self.experiment_info],
indent=4))
path = self.output_dir.parents[
0] / f"true_v_{self.N}_{self.agent_info.get('discount_rate')}".replace(
".", "-")
self.true_values = np.load(f"{path}.npy")
self.states = np.arange(self.N).reshape((-1, 1))
self.state_distribution = np.ones_like(self.true_values) * 1 / len(
self.states)
self.msve_error = np.zeros(
(self.n_episodes // self.episode_eval_freq + 1, ))
self.error = get_objective(
"RMSVE",
self.true_values,
self.state_distribution,
np.ones(len(self.true_values)),
)
self.timesteps = []
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--steps", type=int, required=True)
parser.add_argument("--num_obs", type=int, required=True)
parser.add_argument("--env", type=str, required=True)
parser.add_argument("--discount_rate", type=float, required=True)
parser.add_argument("--num_episodes", type=int, required=True)
parser.add_argument("--policy_name", type=str, required=True)
parser.add_argument("--problem", type=str, required=True)
args = parser.parse_args()
save_rootpath = Path(f"{os.environ.get('SCRATCH')}") / f"{args.problem}"
save_rootpath = path_exists(save_rootpath)
args.__dict__["save_rootpath"] = save_rootpath
simulate_on_policy(**args.__dict__)
def plot(config_fn):
config_root_path = Path(__file__).parents[1] / "configs"
sweeper = Sweeper(config_root_path / f"{config_fn}.json")
config = sweeper.parse(0)
env = config.get("env")
data_path = Path(f"~/scratch/{env}").expanduser()
save_path = path_exists(Path(__file__).parents[0] / config_fn)
fig, ax = plt.subplots(1, 1, sharey="all", sharex="col")
num_runs = config.get("num_runs")
results = Result(config_fn, data_path, num_runs)
algorithm = "TD"
_config = {}
_config["algorithm"] = algorithm
step_sizes = results.get_value_param("step_size", _config)
for i, step_size in enumerate(step_sizes):
_config["step_size"] = step_size
ids = results.find_experiment_by(_config)
data = results.data[ids, :]
mean = data.mean(axis=0)
se = data.std(axis=0) / np.sqrt(num_runs)
ax.plot(mean, label=step_size)
ax.fill_between(
np.arange(len(mean)),
mean + 2.5 * se,
mean - 2.5 * se,
alpha=0.15,
)
plt.yticks([0.0, 0.05, 0.1, 0.15, 0.2, 0.25])
plt.xticks([0, 25, 50, 75, 100])
plt.legend()
fig.tight_layout()
filename = f"{config_fn}"
filename = filename.replace(".", "_")
plt.savefig(save_path / filename)
def _has_learned_automata(results_path):
ilasp_tasks_folder = os.path.join(results_path, ISAAlgorithmBase.AUTOMATON_TASK_FOLDER)
ilasp_solutions_folder = os.path.join(results_path, ISAAlgorithmBase.AUTOMATON_SOLUTION_FOLDER)
return path_exists(ilasp_solutions_folder) and path_exists(ilasp_tasks_folder)
def _has_run_finished(results_path):
abs_time_file = os.path.join(results_path, ISAAlgorithmBase.ABSOLUTE_RUNNING_TIME_FILENAME)
return path_exists(abs_time_file)
from utils.utils import path_exists
epochs = 100
batch_size = 64
batch_size_test = 1
learning_rate = 0.0001
lidar_input_size = 9 # number lidars obs var
joint_input_size = 7 # joint state cond var
n_samples_y = 10 # length timeseries
n_samples_z = 10 # sample from selector
clusters = 2 # clustering component (background/self | static/dynamic)
split = "train"
ckpt_test = "ckpt_cvae_gmm.pth"
result_dir = osp.join("..", "experiments", "CVAE_gmm")
path_exists(result_dir)
ckpt_dir = osp.join(result_dir, "ckpt/")
path_exists(ckpt_dir)
data_dir = "./data_mockup/"
parser = argparse.ArgumentParser()
parser.add_argument("--epochs", type=int, default=epochs)
parser.add_argument("--batch_size", type=int, default=batch_size)
parser.add_argument("--batch_size_test", type=int, default=batch_size_test)
parser.add_argument("--learning_rate", type=float, default=learning_rate)
parser.add_argument("--lidar_input_size", type=int, default=lidar_input_size)
parser.add_argument("--joint_input_size", type=int, default=joint_input_size)
parser.add_argument("--n_samples_y", type=int, default=n_samples_y)
parser.add_argument("--n_samples_z", type=int, default=n_samples_z)
def train(args):
ckpt = ckpt_utc()
loss_fn = nELBO(args.batch_size, args.n_samples_z, args.n_samples_y)
model = VAE(
encoder_layer_sizes=args.encoder_layer_sizes,
decoder_layer_sizes=args.decoder_layer_sizes,
latent_size=args.latent_size,
batch_size=args.batch_size,
conditional=args.conditional,
num_labels=args.num_labels,
)
model = move_to_cuda(model)
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
optimizer.zero_grad()
dataset = Loader(split=args.split, samples=args.n_samples_y)
# randomize an auxiliary index because we want to use sample of time-series (10 time steps)
data_loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=False)
loss_list = []
for epoch in range(args.epochs):
dataset.generate_index()
print("Epoch: ", epoch)
L = []
for itr, batch in enumerate(data_loader):
# observable
y, x = batch
y = tensor_to_variable(y)
x = tensor_to_variable(x)
if y.size(0) != args.batch_size:
continue
else:
mu_phi, log_var_phi, mu_theta, log_var_theta = model(y, x)
loss, kld, ll = loss_fn(y, mu_phi, log_var_phi, mu_theta,
log_var_theta)
if args.split == "train":
loss.backward()
optimizer.step()
optimizer.zero_grad()
# compute the loss averaging over epochs and dividing by batches
L.append(loss.cpu().data.numpy())
print("negative likelihood: ", -ll.cpu().data.numpy())
print("kl: ", kld.cpu().data.numpy())
print("loss:", loss.cpu().data.numpy())
loss_list.append(np.mean(L) / (len(data_loader)))
plt.plot(np.array(loss_list))
plt.grid()
plt.show()
path_exists(args.ckpt_dir)
th.save(model.state_dict(), args.ckpt_dir + ckpt)
print("done!")
def train(args):
ckpt = ckpt_utc()
loss_fn = Loss(
args.batch_size,
args.n_samples_y,
args.lidar_input_size,
args.n_clusters,
model_type=args.model_type,
is_entropy=args.is_entropy,
lmbda=args.lmbda,
)
model = Model(
encoder_layer_sizes=args.encoder_layer_sizes,
latent_size=args.latent_size,
n_clusters=args.n_clusters,
batch_size=args.batch_size,
model_type=args.model_type,
is_multimodal=args.is_multimodal,
)
model = move_to_cuda(model)
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
optimizer.zero_grad()
dataset = Dataset(path=args.data_dir, split=args.split, n_samples=args.n_samples_y)
# randomize an auxiliary index because we want to use random sample of time-series (10 time steps)
# but the time series have to be intact
data_loader = DataLoader(dataset=dataset, batch_size=args.batch_size, shuffle=False)
dataset_val = Dataset(path=args.data_dir, split="val", n_samples=args.n_samples_y)
data_loader_val = DataLoader(dataset=dataset_val, batch_size=args.batch_size, shuffle=False)
loss_train, loss_val = [], []
for epoch in range(args.epochs):
model.train()
dataset.generate_index()
print("Epoch: ", epoch)
loss_epoch = []
for itr, batch in enumerate(data_loader):
# observable
y, x, depth = batch
y = tensor_to_variable(y)
x = tensor_to_variable(x)
depth = tensor_to_variable(depth)
mu_c, std_c, clusters = model(x)
loss = loss_fn(y, mu_c, std_c, clusters)
loss.backward()
optimizer.step()
optimizer.zero_grad()
# compute the loss averaging over epochs and dividing by batches
loss_epoch.append(loss.cpu().data.numpy())
print("train loss:", np.mean(loss_epoch))
loss_train.append(np.mean(loss_epoch))
if epoch % args.test_every_n_epochs == 0:
model.eval()
loss_epoch = []
with th.no_grad():
for itr, batch in enumerate(data_loader):
y, x = batch
mu_c, std_c, clusters = model(x)
loss = loss_fn(y, mu_c, std_c, clusters)
loss_epoch.append(loss.cpu().data.numpy())
print("val loss:", np.mean(loss_epoch))
loss_val.append(np.mean(loss_epoch))
plt.plot(np.array(loss_train))
plt.plot(np.array(loss_val))
plt.grid()
plt.show()
path_exists(args.ckpt_dir)
th.save(model.state_dict(), args.ckpt_dir + ckpt)
print("done!")
def plot_random_walk_features():
experiments = ["RWTA", "RWD", "RWP", "RWF"]
num_runs = 60
env = "RandomWalk"
algorithms = ["TD", "ETD"]
metrics = {
"interim": {
"percent_metric": 0.0025,
"lca_ylim_bottom": 0.0,
"lca_ylim_top": 0.4,
"lca_xlim_bottom": 0.0,
"lca_xlim_top": 100,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": 0.4,
"ssa_xlim_bottom": 0.015625,
"ssa_xlim_top": 2,
},
"auc": {
"percent_metric": 1.0,
"lca_ylim_bottom": 0.0,
"lca_ylim_top": 0.4,
"lca_xlim_bottom": 0.0,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": 0.4,
"ssa_xlim_bottom": 3.051757e-05,
"ssa_xlim_top": 2,
},
"end": {
"percent_metric": 0.0025,
"lca_ylim_bottom": 0.0,
"lca_ylim_top": 0.5,
"lca_xlim_bottom": 0.0,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": 0.5,
"ssa_xlim_top": 2,
},
}
states = [5, 19]
data_path = Path(f"/Volumes/REINFORCE/Plop-Msc-2020/{env}").expanduser()
save_path = path_exists(Path(__file__).parents[0] / "plots")
n_cols = 2
n_rows = len(experiments)
for metric, config in metrics.items():
for state in states:
fig, axs = plt.subplots(
n_rows,
n_cols,
figsize=(n_cols * 5, n_rows * 4),
squeeze=False,
sharex="col",
sharey="all",
dpi=120,
)
for row, experiment in enumerate(experiments):
result = Result(experiment, data_path, num_runs)
cutoff = result.data[:, 0].mean()
for algorithm in algorithms:
_config = {}
_config["num_states"] = state
_config["algorithm"] = algorithm
step_sizes = result.get_value_param("step_size", _config)
mean, standard_error, opt_step_size = lca(
result,
_config,
step_sizes,
metric,
percent=config.get("percent_metric"),
)
lineplot(
axs[row, 0],
np.arange(len(mean)),
mean,
standard_error,
opt_step_size,
n_std=2.5,
color=color_algorithms.get(algorithm),
show_legend=True,
xscale={"value": "linear", "base": 10},
ylim={
"bottom": config.get("lca_ylim_bottom"),
"top": config.get("lca_ylim_top"),
},
xlim={
"bottom": config.get("lca_xlim_bottom"),
"top": config.get("lca_xlim_top"),
},
)
means, standard_errors = ssa(
result,
_config,
step_sizes,
metric,
percent=config.get("percent_metric"),
cutoff=cutoff,
)
lineplot(
axs[row, 1],
step_sizes,
means,
standard_errors,
algorithm,
n_std=2.5,
color=color_algorithms.get(algorithm),
marker="o",
show_legend=True,
xscale={"value": "log", "base": 2},
ylim={
"bottom": config.get("ssa_ylim_bottom"),
"top": config.get("ssa_ylim_top"),
},
xlim={
"bottom": config.get("ssa_xlim_bottom"),
"top": config.get("ssa_xlim_top"),
},
)
for row in range(n_rows):
for col in range(n_cols):
axs[row, col].spines["right"].set_visible(False)
axs[row, col].spines["top"].set_visible(False)
fig.tight_layout()
plt.savefig(save_path / f"{state}-RandomWalk-{metric}")
def train(args):
ckpt = ckpt_utc()
loss_fn = Loss()
model = Model(layer_sizes=args.encoder_layer_sizes,
latent_size=args.latent_size,
is_uq=False)
model = move_to_cuda(model)
optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
optimizer.zero_grad()
# randomize an auxiliary index because we want to use random sample of time-series (10 time steps)
# but the time series have to be intact
dataset = Dataset(
path=args.data_dir,
path_images=args.data_dir + "TRAIN_DATA/DEPTH/",
split=args.split,
n_samples=args.n_samples_y,
is_label_y=args.is_label_y,
is_multimodal=args.is_multimodal,
)
data_loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=False)
dataset_val = Dataset(
path=args.data_dir,
path_images=args.data_dir + "TRAIN_DATA/DEPTH/",
split="val",
n_samples=args.n_samples_y,
is_label_y=args.is_label_y,
is_multimodal=args.is_multimodal,
)
data_loader_val = DataLoader(dataset=dataset_val,
batch_size=args.batch_size,
shuffle=False)
loss_train, loss_val = [], []
for epoch in range(args.epochs):
model.train()
dataset.generate_index()
print("Epoch: ", epoch)
loss_epoch = []
for itr, batch in enumerate(data_loader):
# observable
y, x, lbl = batch
y = tensor_to_variable(y)
x = tensor_to_variable(x)
lbl = tensor_to_variable(lbl)
state = th.cat([y, x], dim=1)
pred = model(state)
pred = pred.reshape(-1, args.n_clusters,
args.lidar_input_size) # .permute(0,2,1)
loss = loss_fn(pred, lbl)
loss.backward()
optimizer.step()
optimizer.zero_grad()
# compute the loss averaging over epochs and dividing by batches
loss_epoch.append(loss.cpu().data.numpy())
print("train loss:", np.mean(loss_epoch))
loss_train.append(np.mean(loss_epoch))
if epoch % args.test_every_n_epochs == 0:
model.eval()
loss_epoch = []
with th.no_grad():
for itr, batch in enumerate(data_loader):
y, x, lbl = batch
state = th.cat([y, x], dim=1)
pred = model(state)
pred = pred.reshape(
-1, args.n_clusters,
args.lidar_input_size) # .permute(0,2,1)
loss = loss_fn(pred, lbl)
loss_epoch.append(loss.cpu().data.numpy())
print("val loss:", np.mean(loss_epoch))
loss_val.append(np.mean(loss_epoch))
plt.plot(np.array(loss_train))
plt.plot(np.array(loss_val))
plt.grid()
plt.show()
path_exists(args.ckpt_dir)
th.save(model.state_dict(), args.ckpt_dir + ckpt)
print("done!")
def plot(sweep_id, config_fn):
config_root_path = Path(__file__).parents[1] / "configs"
sweeper = Sweeper(config_root_path / f"{config_fn}.json")
config = sweeper.parse(sweep_id)
experiment = config.get("experiment")
algorithms = config.get("algorithms").split(",")
num_tilings = [int(x) for x in config.get("tilings").split(",")]
num_tilings = sorted(num_tilings)
env = config.get("env")
data_path = Path(f"~/scratch/{env}").expanduser()
save_path = path_exists(Path(__file__).parents[0] / config_fn)
n_rows = len(num_tilings)
n_cols = len(algorithms)
fig, axes = plt.subplots(n_rows,
n_cols,
figsize=(n_cols * 5, n_rows * 4),
sharey="all",
sharex="col")
fig.suptitle(f"{config.get('metric')} performance on {env}")
for row, tilings in enumerate(num_tilings):
_config = {}
results = Result(config_filename=f"{experiment}.json",
datapath=data_path)
representations = list(
results.get_value_param("representations", {}, 1))[0]
num_runs = list(results.get_value_param("num_runs", {}, 1))[0]
_config["representations"] = representations
_config["tilings"] = tilings
for algorithm in algorithms:
color = algorithms.get(algorithm)
_config["algorithm"] = algorithm
_config.pop("step_size", None)
step_sizes = results.get_value_param("step_size", _config,
num_runs)
step_sizes = sorted(step_sizes)
num_step_sizes = len(step_sizes)
means = np.zeros(num_step_sizes)
standard_errors = np.zeros(num_step_sizes)
optimum_step_size = None
optimum_error = np.inf
for i, step_size in enumerate(step_sizes):
_config["step_size"] = step_size
ids = results.find_experiment_by(_config, num_runs)
data = results._load(ids)
mean, se = get_data_by(
data,
name=config.get("metric"),
percent=config.get("percent_metric"),
)
cutoff = data[:, 0].mean()
# Find best instance of algorithm
if mean <= optimum_error:
optimum_error = mean
optimum_step_size = step_size
# Record step-size sensitivity
means[i] = mean
means = np.nan_to_num(means, nan=np.inf)
means = means.clip(0, cutoff)
standard_errors[i] = se
standard_errors = np.nan_to_num(standard_errors)
standard_errors[np.where(means >= cutoff)[0]] = 1e-8
# ################ PLOT LCA ####################
_config["step_size"] = optimum_step_size
ids = results.find_experiment_by(_config, num_runs)
data = results._load(ids)
mean = data.mean(axis=0)
se = data.std(axis=0) / np.sqrt(num_runs)
steps = int(np.ceil(len(mean) * config.get("percent_plot")))
mean = mean[:steps]
se = se[:steps]
axes[row, 0].plot(mean, c=color, label=algorithm)
axes[row, 0].fill_between(
np.arange(len(mean)),
mean + 2.5 * se,
mean - 2.5 * se,
color=color,
alpha=0.15,
)
axes[-1, 0].set_xlabel("Episodes")
axes[row, 0].set_ylabel(f"RMSVE over {num_runs} runs")
################ PLOT SSA ####################
axes[row, 1].errorbar(step_sizes,
means,
yerr=2.5 * standard_errors,
color=color,
fmt="o-")
axes[row, 1].set_xscale("log", basex=2)
axes[-1, 1].set_xlabel("Step size")
axes[row, 1].set_title(f"{to_name.get(representations)}",
loc="left")
fig.tight_layout()
fig.subplots_adjust(top=0.88)
filename = f"plot_{env}-{config.get('metric')}"
filename = filename.replace(".", "_")
plt.savefig(save_path / filename)
def fig_mountain_car_lca_ssa_lambdas():
experiment = "MountainCar"
num_runs = 50
env = "MountainCar"
algorithms = ["TDTileCoding", "ETDTileCoding"]
metrics = {
"interim": {
"percent_metric": 0.0025,
"lca_ylim_bottom": 0.0,
"lca_ylim_top": None,
"lca_xlim_bottom": 0.0,
"lca_xlim_top": 100,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": None,
"ssa_xlim_bottom": None,
"ssa_xlim_top": 2,
},
"auc": {
"percent_metric": 1.0,
"lca_ylim_bottom": 0.0,
"lca_ylim_top": None,
"lca_xlim_bottom": 0.0,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": None,
"ssa_xlim_bottom": None,
"ssa_xlim_top": 2,
},
"end": {
"percent_metric": 0.0025,
"lca_ylim_bottom": 0.0,
"lca_ylim_top": None,
"lca_xlim_bottom": 0.0,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": None,
"ssa_xlim_top": 2,
},
}
lambdas = [0.0]
data_path = Path(f"/Volumes/REINFORCE/Plop-Msc-2020/{env}").expanduser()
save_path = path_exists(Path(__file__).parents[0] / "plots")
n_cols = 2
n_rows = len(lambdas)
for metric, config in metrics.items():
fig, axs = plt.subplots(
n_rows,
n_cols,
figsize=(n_cols * 5, n_rows * 4),
squeeze=False,
sharex="col",
sharey="all",
dpi=120,
)
for row, lmbda in enumerate(lambdas):
result = Result(experiment, data_path, num_runs)
cutoff = result.data[:, 0].mean()
for algorithm in algorithms:
_config = {}
_config["trace_decay"] = lmbda
_config["algorithm"] = algorithm
step_sizes = result.get_value_param("step_size", _config)
mean, standard_error, opt_step_size = lca(
result,
_config,
step_sizes,
metric,
percent=config.get("percent_metric"),
)
lineplot(
axs[row, 0],
np.arange(len(mean)),
mean,
standard_error,
opt_step_size,
n_std=2.5,
color=color_algorithms.get(algorithm),
show_legend=False,
xscale={
"value": "linear",
"base": 10
},
ylim={
"bottom": config.get("lca_ylim_bottom"),
"top": config.get("lca_ylim_top"),
},
xlim={
"bottom": config.get("lca_xlim_bottom"),
"top": config.get("lca_xlim_top"),
},
)
means, standard_errors = ssa(
result,
_config,
step_sizes,
metric,
percent=config.get("percent_metric"),
cutoff=cutoff,
)
lineplot(
axs[row, 1],
step_sizes,
means,
standard_errors,
algorithm,
n_std=2.5,
color=color_algorithms.get(algorithm),
marker="o",
show_legend=False,
xscale={
"value": "log",
"base": 2
},
ylim={
"bottom": config.get("ssa_ylim_bottom"),
"top": config.get("ssa_ylim_top"),
},
xlim={
"bottom": config.get("ssa_xlim_bottom"),
"top": config.get("ssa_xlim_top"),
},
)
for row in range(n_rows):
for col in range(n_cols):
axs[row, col].spines["right"].set_visible(False)
axs[row, col].spines["top"].set_visible(False)
fig.tight_layout()
plt.savefig(save_path / f"MountainCar-{metric}")
def _remove_uncheckpointed_files_helper(self, folder, prefix, extension, automaton_counter):
if utils.path_exists(folder):
files_to_remove = [os.path.join(folder, x) for x in os.listdir(folder)
if x.startswith(prefix) and int(x[len(prefix):-len(extension)]) > automaton_counter]
utils.rm_files(files_to_remove)
def plot_random_walk_features():
experiment = "RW19GammaLambdaAlpha"
num_runs = 1
algorithms = ["TD", "ETD"]
trace_decays = [0.0, 0.4, 1]
metrics = {
"auc": {
"percent_metric": 1.0,
"ssa_ylim_bottom": 0.0,
"ssa_ylim_top": None,
"ssa_xlim_bottom": 0,
"ssa_xlim_top": 2,
}
}
data_path = Path(f"~/scratch/").expanduser()
save_path = path_exists(Path(__file__).parents[0] / "plots")
n_cols = 2
n_rows = 1
for metric, config in metrics.items():
fig, axs = plt.subplots(
n_rows,
n_cols,
figsize=(n_cols * 5, n_rows * 4),
squeeze=False,
sharex="col",
sharey="all",
dpi=120,
)
result = Result(experiment, data_path, num_runs)
cutoff = result.data[:, 0].mean()
for trace_decay in trace_decays:
color = lambdas.get(trace_decay)
for n_col, algorithm in enumerate(algorithms):
_config = {}
_config["trace_decay"] = trace_decay
_config["algorithm"] = algorithm
step_sizes = result.get_value_param("step_size", _config)
means, standard_errors = ssa(
result,
_config,
step_sizes,
metric,
percent=config.get("percent_metric"),
cutoff=cutoff,
)
lineplot(
axs[0, n_col],
step_sizes,
means,
standard_errors,
trace_decay,
n_std=2.5,
color=color,
show_legend=True,
xscale={
"value": "linear",
"base": 10
},
ylim={
"bottom": config.get("ssa_ylim_bottom"),
"top": config.get("ssa_ylim_top"),
},
xlim={
"bottom": config.get("ssa_xlim_bottom"),
"top": config.get("ssa_xlim_top"),
},
)
for row in range(n_rows):
for col in range(n_cols):
axs[row, col].spines["right"].set_visible(False)
axs[row, col].spines["top"].set_visible(False)
fig.tight_layout()
plt.savefig(save_path / f"{experiment}-{metric}")