def compute_metrics(model, summarise=True):
"""Compute metrics.
Args:
model (str): Name of the model folder.
summarise (bool, optional): Summarise the metrics rather than given the data
back. Defaults to `True`.
Returns:
union[None, tuple[:class:`np.array`, :class:`np.array`]]: The metrics if
`summarise` is `False`. Otherwise nothing.
"""
rmses, mlls = [], []
preds = wd_results.load(model, "preds.pickle")
for (y, mean, var) in preds:
rmses.append(metric.rmse(mean, y))
mlls.append(metric.mll(mean, var, y))
if summarise:
with out.Section(model.upper()):
for name, values in [("MLL", mlls), ("RMSE", rmses)]:
with out.Section(name):
out.kv("Mean", np.mean(values))
out.kv("Std", np.std(values) / len(values)**0.5)
else:
return mlls, rmses
def test_kv_newlines():
# Test values with newlines.
with Mock() as mock:
out.kv("a", "b\nc")
assert len(mock) == 2
assert mock[0] == "a:\n"
assert mock[1] == " b\n c\n"
def test_kv_dict_as_value():
# Test giving a key and a dictionary.
with Mock() as mock:
out.kv("dict", {1: 1})
assert len(mock) == 2
assert mock[0] == "dict:\n"
assert mock[1] == " 1: 1\n"
def test_kv_dict():
# Test giving a dictionary.
with Mock() as mock:
out.kv({"level1": {"level2": {1: 1}}})
assert len(mock) == 3
assert mock[0] == "level1:\n"
assert mock[1] == " level2:\n"
assert mock[2] == " 1: 1\n"
def test_kv():
with Mock() as mock:
out.kv("key", "value")
out.kv(1.0, 1.0)
out.kv(10.0, 10.0)
out.kv(1000.0, 1000.0)
out.kv(1000, 1000)
assert len(mock) == 5
assert mock[0] == "key: value\n"
assert mock[1] == "1.0: 1.0\n"
assert mock[2] == "10.0: 10.0\n"
assert mock[3] == "1.000e+03: 1.000e+03\n"
assert mock[4] == "1000: 1000\n"
def _sync_folder(source: str, ip: str, target: RemotePath):
with out.Section("Syncing to remote folder"):
out.kv("Source", source)
with out.Section("Target"):
out.kv("Host", target.remote.host)
out.kv("Path", target.path)
try:
ssh(
target.remote,
f'rsync -Pav -e "ssh -oStrictHostKeyChecking=no -i {config["ssh_key"]}"'
f' {config["ssh_user"]}@{ip}:{source} {target.path}',
)
except subprocess.CalledProcessError as e:
out.kv("Synchronisation error", str(e))
def _sync_folder(source: str, ip: str, target: LocalPath):
with out.Section("Syncing to local folder"):
out.kv("Source", source)
out.kv("Target", target.path)
try:
execute_command(
"rsync",
"-Pav",
"-e",
f'ssh -oStrictHostKeyChecking=no -i {config["ssh_key"]}',
f'{config["ssh_user"]}@{ip}:{source}',
target.path,
)
except subprocess.CalledProcessError as e:
out.kv("Synchronisation error", str(e))
def compare(model1, model2):
"""Compare two models.
Args:
model1 (str): Model folder of the first model to compare.
model2 (str): Model folder of the second model to compare.
"""
mlls1, rmses1 = compute_metrics(model1, summarise=False)
mlls2, rmses2 = compute_metrics(model2, summarise=False)
with out.Section(f"{model1.upper()} - {model2.upper()}"):
for name, values1, values2 in [("MLL", mlls1, mlls2),
("RMSE", rmses1, rmses2)]:
diff = [x - y for x, y in zip(values1, values2)]
with out.Section(name):
mean = np.mean(diff)
std = np.std(diff) / len(diff)**0.5
out.kv("Mean", mean)
out.kv("Std", std)
out.kv("p-value", st.norm.cdf(-abs(mean) / std))
y = (lik_noise + z_true).sample(100)
# Define likelihood.
def lik(x):
return B.sum((lik_noise + y).logpdf(x))
# Construct q distribution.
def construct_q(vs):
q = Normal(
Diagonal(vs.positive(shape=[d], name="q/var")),
vs.get(shape=[d, 1], name="q/mean"),
)
return q
# Construct objective.
def objective(vs):
q = construct_q(vs)
return -elbo(lik, prior, q, num_samples=1)
# Optimise.
objective_compiled = tf.function(objective, autograph=False)
minimise_adam(objective_compiled, vs, trace=True, iters=2000, rate=5e-2)
# Print result.
out.kv("True", z_true)
out.kv("Mean q", construct_q(vs).mean)
preds_f = {}
preds_f_test = {}
preds_k = {}
preds_psd = {}
for model in models:
preds_f[model.name] = wd.load(model.name.lower(), "pred_f.pickle")
preds_f_test[model.name] = wd.load(model.name.lower(),
"pred_f_test.pickle")
preds_k[model.name] = wd.load(model.name.lower(), "pred_k.pickle")
preds_psd[model.name] = wd.load(model.name.lower(), "pred_psd.pickle")
# Print performances.
for name in ["GPCM", "CGPCM", "RGPCM"]:
with out.Section(name):
t, mean, var = preds_f_test[name]
out.kv("RMSE", metric.rmse(mean, y_test))
out.kv("MLL", metric.mll(mean, var, y_test))
def plot_psd(name, y_label=True, style="pred", finish=True):
"""Plot prediction for the PSD."""
freqs, mean, lower, upper = preds_psd[name]
freqs -= freqs[0]
inds = freqs <= 0.2
freqs = freqs[inds]
mean = mean[inds]
lower = lower[inds]
upper = upper[inds]
if y_label:
k = wd_results.load(data, "data.pickle")["k"]
t, mean, var = wd_results.load(data, scheme, model, "k_pred.pickle")
inds = t <= until
if metric == "smll":
return smll(mean[inds], var[inds], k[inds])
elif metric == "rmse":
return rmse(mean[inds], k[inds])
else:
raise ValueError(f'Bad metric "{metric}".')
for model, kernel in [("gpcm", "eq"), ("cgpcm", "ceq-1"),
("rgpcm", "matern12")]:
with out.Section(model.upper()):
with out.Section("SMLL"):
out.kv("MF", kernel_analysis(kernel, "mean-field", model, "smll"))
out.kv("S", kernel_analysis(kernel, "structured", model, "smll"))
with out.Section("RMSE"):
out.kv("MF", kernel_analysis(kernel, "mean-field", model, "rmse"))
out.kv("S", kernel_analysis(kernel, "structured", model, "rmse"))
def plot_kernel_predictions(model, data_name, legend=True, first=False):
"""Plot the prediction for a kernel."""
k = wd_results.load(data_name, "data.pickle")["k"]
t, mean1, var1 = wd_results.load(data_name, "structured", model,
"k_pred.pickle")
t, mean2, var2 = wd_results.load(data_name, "mean-field", model,
"k_pred.pickle")
plt.plot(t, k, label="Truth", style="train")
plt.plot(t, mean1, label="Structured", style="pred")
def test_kv_unit():
with Mock() as mock:
out.kv("key", 1, unit="s")
assert len(mock) == 1
assert mock[0] == "key: 1 s\n"
with Measure() as prior:
# Construct a linear model.
slope = GP(1)
intercept = GP(5)
f = slope * (lambda x: x) + intercept
# Sample a slope, intercept, underlying function, and observations.
true_slope, true_intercept, f_true, y_obs = prior.sample(
slope(0), intercept(0), f(x), f(x_obs, 0.2)
)
# Condition on the observations to make predictions.
post = prior | (f(x_obs, 0.2), y_obs)
mean, lower, upper = post(f(x)).marginal_credible_bounds()
out.kv("True slope", true_slope[0, 0])
out.kv("Predicted slope", post(slope(0)).mean[0, 0])
out.kv("True intercept", true_intercept[0, 0])
out.kv("Predicted intercept", post(intercept(0)).mean[0, 0])
# Plot result.
plt.plot(x, f_true, label="True", style="test")
plt.scatter(x_obs, y_obs, label="Observations", style="train", s=20)
plt.plot(x, mean, label="Prediction", style="pred")
plt.fill_between(x, lower, upper, style="pred")
tweak()
plt.savefig("readme_example6_blr.png")
plt.show()
model.fit(t, y, iters=30_000)
k_pred_struc = extract(model.condition(t, y).predict_kernel(t_k))
psd_pred_struc = extract(model.condition(t, y).predict_psd())
wd.save((k_pred_mf, psd_pred_mf, k_pred_struc, psd_pred_struc),
"preds.pickle")
else:
k_pred_mf, psd_pred_mf, k_pred_struc, psd_pred_struc = wd.load(
"preds.pickle")
# Report metrics.
with out.Section("Structured"):
t, mean, var, _, _ = k_pred_struc
inds = t <= 3
out.kv("MLL", metric.mll(mean[inds], var[inds], k[inds]))
out.kv("RMSE", metric.rmse(mean[inds], k[inds]))
with out.Section("Mean field"):
t, mean, var, _, _ = k_pred_mf
inds = t <= 3
out.kv("MLL", metric.mll(mean[inds], var[inds], k[inds]))
out.kv("RMSE", metric.rmse(mean[inds], k[inds]))
plt.figure(figsize=(7.5, 3.75))
# Plot prediction for kernel.
plt.subplot(1, 2, 1)
plt.plot(t_k, k, label="Truth", style="train")
t, mean, var, err_95_lower, err_95_upper = k_pred_struc
plt.plot(t, mean, label="Structured", style="pred")
def manage_cluster(
commands: List[List[str]],
instance_type: str,
key_name: str,
security_group_id: str,
image_id: str,
sync_sources: List[str],
sync_target: Path,
monitor_aws_repo: str,
monitor_call: str,
monitor_delay: int,
):
"""Manage the cluster.
Args:
commands (list[list[str]]): One list of commands for every experiment.
image_id (str): Image ID.
instance_type (str): Type of the instance.
key_name (str): Name of the key pair.
security_group_id (str): Security group.
sync_sources (list[str]): List of sources to sync.
sync_target (:class:`.util.Path`): Directory to sync to.
monitor_aws_repo (str, optional): Path to the root of this repo. The repo
must consider the virtual environment "venv" which has the repo installed
in editable mode.
monitor_call (str): Call to start the monitor. See :mod:`.monitor`.
monitor_delay (int): Number of seconds to wait before starting the monitor.
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"--spawn",
type=int,
help="Spawn instances.",
)
parser.add_argument(
"--start",
action="store_true",
help="Start experiments.",
)
parser.add_argument(
"--terminate",
action="store_true",
help="Terminate all instances. This is a kill switch.",
)
parser.add_argument(
"--kill",
action="store_true",
help="Kill all running experiments, but keep the instances running.",
)
parser.add_argument(
"--stop",
action="store_true",
help="Stop all running instances",
)
parser.add_argument(
"--sync-stopped",
action="store_true",
help="Synchronise all stopped instances.",
)
parser.add_argument(
"--sync-sleep",
default=120,
type=int,
help="Number of seconds to sleep before syncing again.",
)
args = parser.parse_args()
if args.sync_stopped:
with out.Section("Syncing all stopped instances in five batches"):
for batch in np.array_split(get_state("stopped"), 5):
# Batches can be empty.
if len(batch) == 0:
continue
# Start the instances.
start(*batch)
try:
# Wait for the instances to have booted.
out.out(
"Waiting a minute for the instances to have booted...")
time.sleep(60)
# Refresh the instances to get the IPs.
instance_ids = [
instance["InstanceId"] for instance in batch
]
batch = get_instances(*instance_ids)
# Sync.
sync(
sync_sources,
sync_target,
ips=[
instance["PublicIpAddress"] for instance in batch
],
)
finally:
# Stop the instances again.
stop(*batch)
out.out("Syncing completed: not continuing execution of script.")
exit()
if args.spawn:
with out.Section("Starting all stopped instances"):
start_stopped()
with out.Section("Spawning instances"):
spawn(
image_id=image_id,
total_count=args.spawn,
instance_type=instance_type,
key_name=key_name,
security_group_id=security_group_id,
)
while not check_all_running():
out.out("Waiting for all instances to be running...")
time.sleep(5)
out.out("Waiting a minute for all instances to have booted...")
time.sleep(60)
if args.kill:
with out.Section("Killing all experiments"):
kill_all()
if args.stop:
with out.Section("Stopping all instances"):
stop_running()
if args.terminate:
with out.Section("Terminating all instances"):
terminate_all()
if args.start:
num_instances = len(get_running_ips())
pieces = np.array_split(commands, num_instances)
# Ensure that we have regular Python lists.
pieces = [piece.tolist() for piece in pieces]
with out.Section("Starting experiments"):
out.kv("Number of commands", len(commands))
out.kv("Number of instances", num_instances)
out.kv("Maximum runs per instance",
max([len(piece) for piece in pieces]))
ssh_map(
*[[
*config["setup_commands"],
*sum(piece, []),
*config["teardown_commands"],
] for piece in pieces],
start_experiment=True,
in_experiment=True,
start_monitor=True,
monitor_aws_repo=monitor_aws_repo,
monitor_delay=monitor_delay,
monitor_call=monitor_call,
)
while True:
out.kv("Instances still running", len(get_running_ips()))
sync(sync_sources, sync_target)
out.out(f"Sleeping for {args.sync_sleep} second(s)...")
time.sleep(args.sync_sleep)
post = f.measure | (f(x), f_true)
y_obs = post(f(x_obs)).sample()
def objective(vs):
f, y = model(vs)
evidence = y(x_obs).logpdf(y_obs)
return -evidence
# Learn hyperparameters.
minimise_l_bfgs_b(tf.function(objective, autograph=False), vs)
f, y = model(vs)
# Print the learned parameters.
out.kv("Prior", y.display(out.format))
vs.print()
# Condition on the observations to make predictions.
post = f.measure | (y(x_obs), y_obs)
mean, lower, upper = post(f(x)).marginals()
# Plot result.
plt.plot(x, B.squeeze(f_true), label="True", style="test")
plt.scatter(x_obs, B.squeeze(y_obs), label="Observations", style="train", s=20)
plt.plot(x, mean, label="Prediction", style="pred")
plt.fill_between(x, lower, upper, style="pred")
tweak()
plt.savefig("readme_example3_parametric.png")
plt.show()
def test_kv_fmt():
with Mock() as mock:
out.kv("key", 1, fmt=".4f")
assert len(mock) == 1
assert mock[0] == "key: 1.0000\n"
x = B.linspace(0, 10, 100)
x_obs = B.linspace(0, 7, 50_000)
x_ind = B.linspace(0, 10, 20)
# Construct a prior.
f = GP(EQ().periodic(2 * B.pi))
# Sample a true, underlying function and observations.
f_true = B.sin(x)
y_obs = B.sin(x_obs) + B.sqrt(0.5) * B.randn(*x_obs.shape)
# Compute a pseudo-point approximation of the posterior.
obs = PseudoObs(f(x_ind), (f(x_obs, 0.5), y_obs))
# Compute the ELBO.
out.kv("ELBO", obs.elbo(f.measure))
# Compute the approximate posterior.
f_post = f | obs
# Make predictions with the approximate posterior.
mean, lower, upper = f_post(x).marginal_credible_bounds()
# Plot result.
plt.plot(x, f_true, label="True", style="test")
plt.scatter(
x_obs,
y_obs,
label="Observations",
style="train",
c="tab:green",
# Setup script.
wd = WorkingDirectory("_experiments", "crude_oil_aggregate")
# Load all experiments and compute metrics.
names = ["GPCM", "CGPCM", "RGPCM"]
mlls = {name: [] for name in names}
rmses = {name: [] for name in names}
for year in range(2012, 2017 + 1):
wd_results = WorkingDirectory("_experiments", "crude_oil", str(year), observe=True)
t, y = wd_results.load("data.pickle")["test"]
for name in names:
_, mean, var = wd_results.load(name.lower(), "pred_f_test.pickle")
mlls[name].append(metric.mll(mean, var, y))
rmses[name].append(metric.rmse(mean, y))
# Print aggregate results.
for name in names:
with out.Section(name):
out.kv("MLL", np.mean(mlls[name]))
out.kv("MLL (std)", np.std(mlls[name]) / len(mlls[name]) ** 0.5)
out.kv("RMSE", np.mean(rmses[name]))
out.kv("RMSE (std)", np.std(rmses[name]) / len(rmses[name]) ** 0.5)
# Compare results.
for name1, name2 in [("RGPCM", "CGPCM"), ("RGPCM", "GPCM"), ("CGPCM", "GPCM")]:
with out.Section(f"{name1} - {name2}"):
out.kv("MLL", np.mean(mlls[name1]) - np.mean(mlls[name2]))
out.kv("MLL (p)", ttest_rel(mlls[name1], mlls[name2]).pvalue)
out.kv("RMSE", np.mean(rmses[name1]) - np.mean(rmses[name2]))
out.kv("RMSE (p)", ttest_rel(rmses[name1], rmses[name2]).pvalue)