def compute_metrics(model, summarise=True):
"""Compute metrics.
Args:
model (str): Name of the model folder.
summarise (bool, optional): Summarise the metrics rather than given the data
back. Defaults to `True`.
Returns:
union[None, tuple[:class:`np.array`, :class:`np.array`]]: The metrics if
`summarise` is `False`. Otherwise nothing.
"""
rmses, mlls = [], []
preds = wd_results.load(model, "preds.pickle")
for (y, mean, var) in preds:
rmses.append(metric.rmse(mean, y))
mlls.append(metric.mll(mean, var, y))
if summarise:
with out.Section(model.upper()):
for name, values in [("MLL", mlls), ("RMSE", rmses)]:
with out.Section(name):
out.kv("Mean", np.mean(values))
out.kv("Std", np.std(values) / len(values)**0.5)
else:
return mlls, rmses
def _sync_folder(source: str, ip: str, target: RemotePath):
with out.Section("Syncing to remote folder"):
out.kv("Source", source)
with out.Section("Target"):
out.kv("Host", target.remote.host)
out.kv("Path", target.path)
try:
ssh(
target.remote,
f'rsync -Pav -e "ssh -oStrictHostKeyChecking=no -i {config["ssh_key"]}"'
f' {config["ssh_user"]}@{ip}:{source} {target.path}',
)
except subprocess.CalledProcessError as e:
out.kv("Synchronisation error", str(e))
def _minimise_l_bfgs_b(f,
vs,
f_calls=10000,
iters=1000,
trace=False,
names=None,
jit=False):
names = _convert_and_validate_names(names)
# Run function once to ensure that all variables are initialised and
# available.
val_init = f(vs)
# SciPy doesn't perform zero iterations, so handle that edge case
# manually.
if iters == 0 or f_calls == 0:
return B.to_numpy(val_init)
# Extract initial value.
x0 = B.to_numpy(vs.get_latent_vector(*names))
# The optimiser expects to get `float64`s.
def _convert(*xs):
return [B.cast(np.float64, B.to_numpy(x)) for x in xs]
# Wrap the function and get the list of function evaluations.
f_vals, f_wrapped = wrap_f(vs, names, f, jit, _convert)
# Perform optimisation routine.
def perform_minimisation(callback_=lambda _: None):
return fmin_l_bfgs_b(
func=f_wrapped,
x0=x0,
maxiter=iters,
maxfun=f_calls,
callback=callback_,
disp=0,
)
if trace:
# Print progress during minimisation.
with out.Progress(name='Minimisation of "{}"'.format(f.__name__),
total=iters) as progress:
def callback(_):
progress({"Objective value": np.min(f_vals)})
x_opt, val_opt, info = perform_minimisation(callback)
with out.Section("Termination message"):
out.out(convert(info["task"], str))
else:
# Don't print progress; simply perform minimisation.
x_opt, val_opt, info = perform_minimisation()
vs.set_latent_vector(x_opt, *names) # Assign optimum.
return val_opt # Return optimal value.
def print_logs(path: str):
"""Display the tail of logs on all running instances.
Args:
path (str): Path to the log.
"""
for ip, log in ssh_map([f"tail -n100 {path}"], broadcast=True).items():
with out.Section(ip):
out.out(log)
def compare(model1, model2):
"""Compare two models.
Args:
model1 (str): Model folder of the first model to compare.
model2 (str): Model folder of the second model to compare.
"""
mlls1, rmses1 = compute_metrics(model1, summarise=False)
mlls2, rmses2 = compute_metrics(model2, summarise=False)
with out.Section(f"{model1.upper()} - {model2.upper()}"):
for name, values1, values2 in [("MLL", mlls1, mlls2),
("RMSE", rmses1, rmses2)]:
diff = [x - y for x, y in zip(values1, values2)]
with out.Section(name):
mean = np.mean(diff)
std = np.std(diff) / len(diff)**0.5
out.kv("Mean", mean)
out.kv("Std", std)
out.kv("p-value", st.norm.cdf(-abs(mean) / std))
def test_out_newlines():
# Test that newlines are correctly indented.
with Mock() as mock:
out.out("a\nb")
with out.Section():
out.out("c\nd")
assert len(mock) == 2
assert mock[0] == "a\nb\n"
assert mock[1] == " c\n d\n"
def test_section():
with Mock() as mock:
out.out("before")
with out.Section():
out.out("message1")
with out.Section("name"):
out.out("message2")
with out.Section():
out.out("message3")
out.out("after")
assert len(mock) == 6
assert mock[0] == "before\n"
assert mock[1] == " message1\n"
assert mock[2] == "name:\n"
assert mock[3] == " message2\n"
assert mock[4] == " message3\n"
assert mock[5] == "after\n"
def minimise_l_bfgs_b(f,
vs,
f_calls=10000,
iters=1000,
trace=False,
names=None):
names = [] if names is None else names
# Run function once to ensure that all variables are initialised and
# available.
val_init = f(vs)
# SciPy doesn't perform zero iterations, so handle that edge case
# manually.
if iters == 0 or f_calls == 0:
return B.to_numpy(val_init)
# Extract initial value.
x0 = B.to_numpy(vs.get_vector(*names))
# Wrap the function and get the list of function evaluations.
f_vals, f_wrapped = wrap_f(vs, names, f)
# Perform optimisation routine.
def perform_minimisation(callback_=lambda _: None):
return fmin_l_bfgs_b(func=f_wrapped,
x0=x0,
maxiter=iters,
maxfun=f_calls,
callback=callback_,
disp=0)
if trace:
# Print progress during minimisation.
with out.Progress(name='Minimisation of "{}"'.format(f.__name__),
total=iters) as progress:
def callback(_):
progress({'Objective value': np.min(f_vals)})
x_opt, val_opt, info = perform_minimisation(callback)
with out.Section('Termination message'):
out.out(info['task'].decode('utf-8'))
else:
# Don't print progress; simply perform minimisation.
x_opt, val_opt, info = perform_minimisation()
vs.set_vector(x_opt, *names) # Assign optimum.
return val_opt # Return optimal value.
def _sync_folder(source: str, ip: str, target: LocalPath):
with out.Section("Syncing to local folder"):
out.kv("Source", source)
out.kv("Target", target.path)
try:
execute_command(
"rsync",
"-Pav",
"-e",
f'ssh -oStrictHostKeyChecking=no -i {config["ssh_key"]}',
f'{config["ssh_user"]}@{ip}:{source}',
target.path,
)
except subprocess.CalledProcessError as e:
out.kv("Synchronisation error", str(e))
def sync(sources: List[str], target: Path, ips: List[str] = None):
"""Synchronise data.
Args:
sources (list[str]): List of sources to sync.
target (:class:`.util.Path`): Directory to sync to.
ips (list[str], optional): IPs to sync. Defaults to all running IPs.
"""
if ips is None:
ips = get_running_ips()
for ip in ips:
with out.Section(ip):
for source in sources:
_sync_folder(source, ip, target)
def exception(x, e):
"""In the case that an exception is raised during function evaluation,
print a warning and return NaN for the function value and gradient.
Args:
x (tensor): Current input.
e (:class:`Exception`): Caught exception.
Returns:
tuple: Tuple containing NaN and NaNs for the gradient.
"""
with out.Section("Caught exception during function evaluation"):
out.out(traceback.format_exc().strip())
grad_nan = np.empty(x.shape)
grad_nan[:] = np.nan
return np.nan, grad_nan
def kernel_analysis(data, scheme, model, metric, until=4):
"""Analyse the prediction for a kernel."""
k = wd_results.load(data, "data.pickle")["k"]
t, mean, var = wd_results.load(data, scheme, model, "k_pred.pickle")
inds = t <= until
if metric == "smll":
return smll(mean[inds], var[inds], k[inds])
elif metric == "rmse":
return rmse(mean[inds], k[inds])
else:
raise ValueError(f'Bad metric "{metric}".')
for model, kernel in [("gpcm", "eq"), ("cgpcm", "ceq-1"),
("rgpcm", "matern12")]:
with out.Section(model.upper()):
with out.Section("SMLL"):
out.kv("MF", kernel_analysis(kernel, "mean-field", model, "smll"))
out.kv("S", kernel_analysis(kernel, "structured", model, "smll"))
with out.Section("RMSE"):
out.kv("MF", kernel_analysis(kernel, "mean-field", model, "rmse"))
out.kv("S", kernel_analysis(kernel, "structured", model, "rmse"))
def plot_kernel_predictions(model, data_name, legend=True, first=False):
"""Plot the prediction for a kernel."""
k = wd_results.load(data_name, "data.pickle")["k"]
t, mean1, var1 = wd_results.load(data_name, "structured", model,
"k_pred.pickle")
t, mean2, var2 = wd_results.load(data_name, "mean-field", model,
"k_pred.pickle")
# Load predictions.
preds_f = {}
preds_f_test = {}
preds_k = {}
preds_psd = {}
for model in models:
preds_f[model.name] = wd.load(model.name.lower(), "pred_f.pickle")
preds_f_test[model.name] = wd.load(model.name.lower(),
"pred_f_test.pickle")
preds_k[model.name] = wd.load(model.name.lower(), "pred_k.pickle")
preds_psd[model.name] = wd.load(model.name.lower(), "pred_psd.pickle")
# Print performances.
for name in ["GPCM", "CGPCM", "RGPCM"]:
with out.Section(name):
t, mean, var = preds_f_test[name]
out.kv("RMSE", metric.rmse(mean, y_test))
out.kv("MLL", metric.mll(mean, var, y_test))
def plot_psd(name, y_label=True, style="pred", finish=True):
"""Plot prediction for the PSD."""
freqs, mean, lower, upper = preds_psd[name]
freqs -= freqs[0]
inds = freqs <= 0.2
freqs = freqs[inds]
mean = mean[inds]
lower = lower[inds]
upper = upper[inds]
n_z=n_z,
t=t,
)
model.fit(t, y, iters=30_000)
k_pred_struc = extract(model.condition(t, y).predict_kernel(t_k))
psd_pred_struc = extract(model.condition(t, y).predict_psd())
wd.save((k_pred_mf, psd_pred_mf, k_pred_struc, psd_pred_struc),
"preds.pickle")
else:
k_pred_mf, psd_pred_mf, k_pred_struc, psd_pred_struc = wd.load(
"preds.pickle")
# Report metrics.
with out.Section("Structured"):
t, mean, var, _, _ = k_pred_struc
inds = t <= 3
out.kv("MLL", metric.mll(mean[inds], var[inds], k[inds]))
out.kv("RMSE", metric.rmse(mean[inds], k[inds]))
with out.Section("Mean field"):
t, mean, var, _, _ = k_pred_mf
inds = t <= 3
out.kv("MLL", metric.mll(mean[inds], var[inds], k[inds]))
out.kv("RMSE", metric.rmse(mean[inds], k[inds]))
plt.figure(figsize=(7.5, 3.75))
# Plot prediction for kernel.
plt.subplot(1, 2, 1)
def manage_cluster(
commands: List[List[str]],
instance_type: str,
key_name: str,
security_group_id: str,
image_id: str,
sync_sources: List[str],
sync_target: Path,
monitor_aws_repo: str,
monitor_call: str,
monitor_delay: int,
):
"""Manage the cluster.
Args:
commands (list[list[str]]): One list of commands for every experiment.
image_id (str): Image ID.
instance_type (str): Type of the instance.
key_name (str): Name of the key pair.
security_group_id (str): Security group.
sync_sources (list[str]): List of sources to sync.
sync_target (:class:`.util.Path`): Directory to sync to.
monitor_aws_repo (str, optional): Path to the root of this repo. The repo
must consider the virtual environment "venv" which has the repo installed
in editable mode.
monitor_call (str): Call to start the monitor. See :mod:`.monitor`.
monitor_delay (int): Number of seconds to wait before starting the monitor.
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"--spawn",
type=int,
help="Spawn instances.",
)
parser.add_argument(
"--start",
action="store_true",
help="Start experiments.",
)
parser.add_argument(
"--terminate",
action="store_true",
help="Terminate all instances. This is a kill switch.",
)
parser.add_argument(
"--kill",
action="store_true",
help="Kill all running experiments, but keep the instances running.",
)
parser.add_argument(
"--stop",
action="store_true",
help="Stop all running instances",
)
parser.add_argument(
"--sync-stopped",
action="store_true",
help="Synchronise all stopped instances.",
)
parser.add_argument(
"--sync-sleep",
default=120,
type=int,
help="Number of seconds to sleep before syncing again.",
)
args = parser.parse_args()
if args.sync_stopped:
with out.Section("Syncing all stopped instances in five batches"):
for batch in np.array_split(get_state("stopped"), 5):
# Batches can be empty.
if len(batch) == 0:
continue
# Start the instances.
start(*batch)
try:
# Wait for the instances to have booted.
out.out(
"Waiting a minute for the instances to have booted...")
time.sleep(60)
# Refresh the instances to get the IPs.
instance_ids = [
instance["InstanceId"] for instance in batch
]
batch = get_instances(*instance_ids)
# Sync.
sync(
sync_sources,
sync_target,
ips=[
instance["PublicIpAddress"] for instance in batch
],
)
finally:
# Stop the instances again.
stop(*batch)
out.out("Syncing completed: not continuing execution of script.")
exit()
if args.spawn:
with out.Section("Starting all stopped instances"):
start_stopped()
with out.Section("Spawning instances"):
spawn(
image_id=image_id,
total_count=args.spawn,
instance_type=instance_type,
key_name=key_name,
security_group_id=security_group_id,
)
while not check_all_running():
out.out("Waiting for all instances to be running...")
time.sleep(5)
out.out("Waiting a minute for all instances to have booted...")
time.sleep(60)
if args.kill:
with out.Section("Killing all experiments"):
kill_all()
if args.stop:
with out.Section("Stopping all instances"):
stop_running()
if args.terminate:
with out.Section("Terminating all instances"):
terminate_all()
if args.start:
num_instances = len(get_running_ips())
pieces = np.array_split(commands, num_instances)
# Ensure that we have regular Python lists.
pieces = [piece.tolist() for piece in pieces]
with out.Section("Starting experiments"):
out.kv("Number of commands", len(commands))
out.kv("Number of instances", num_instances)
out.kv("Maximum runs per instance",
max([len(piece) for piece in pieces]))
ssh_map(
*[[
*config["setup_commands"],
*sum(piece, []),
*config["teardown_commands"],
] for piece in pieces],
start_experiment=True,
in_experiment=True,
start_monitor=True,
monitor_aws_repo=monitor_aws_repo,
monitor_delay=monitor_delay,
monitor_call=monitor_call,
)
while True:
out.kv("Instances still running", len(get_running_ips()))
sync(sync_sources, sync_target)
out.out(f"Sleeping for {args.sync_sleep} second(s)...")
time.sleep(args.sync_sleep)
# Setup script.
wd = WorkingDirectory("_experiments", "crude_oil_aggregate")
# Load all experiments and compute metrics.
names = ["GPCM", "CGPCM", "RGPCM"]
mlls = {name: [] for name in names}
rmses = {name: [] for name in names}
for year in range(2012, 2017 + 1):
wd_results = WorkingDirectory("_experiments", "crude_oil", str(year), observe=True)
t, y = wd_results.load("data.pickle")["test"]
for name in names:
_, mean, var = wd_results.load(name.lower(), "pred_f_test.pickle")
mlls[name].append(metric.mll(mean, var, y))
rmses[name].append(metric.rmse(mean, y))
# Print aggregate results.
for name in names:
with out.Section(name):
out.kv("MLL", np.mean(mlls[name]))
out.kv("MLL (std)", np.std(mlls[name]) / len(mlls[name]) ** 0.5)
out.kv("RMSE", np.mean(rmses[name]))
out.kv("RMSE (std)", np.std(rmses[name]) / len(rmses[name]) ** 0.5)
# Compare results.
for name1, name2 in [("RGPCM", "CGPCM"), ("RGPCM", "GPCM"), ("CGPCM", "GPCM")]:
with out.Section(f"{name1} - {name2}"):
out.kv("MLL", np.mean(mlls[name1]) - np.mean(mlls[name2]))
out.kv("MLL (p)", ttest_rel(mlls[name1], mlls[name2]).pvalue)
out.kv("RMSE", np.mean(rmses[name1]) - np.mean(rmses[name2]))
out.kv("RMSE (p)", ttest_rel(rmses[name1], rmses[name2]).pvalue)