def compute_metrics(model, summarise=True): """Compute metrics. Args: model (str): Name of the model folder. summarise (bool, optional): Summarise the metrics rather than given the data back. Defaults to `True`. Returns: union[None, tuple[:class:`np.array`, :class:`np.array`]]: The metrics if `summarise` is `False`. Otherwise nothing. """ rmses, mlls = [], [] preds = wd_results.load(model, "preds.pickle") for (y, mean, var) in preds: rmses.append(metric.rmse(mean, y)) mlls.append(metric.mll(mean, var, y)) if summarise: with out.Section(model.upper()): for name, values in [("MLL", mlls), ("RMSE", rmses)]: with out.Section(name): out.kv("Mean", np.mean(values)) out.kv("Std", np.std(values) / len(values)**0.5) else: return mlls, rmses
def test_kv_newlines(): # Test values with newlines. with Mock() as mock: out.kv("a", "b\nc") assert len(mock) == 2 assert mock[0] == "a:\n" assert mock[1] == " b\n c\n"
def test_kv_dict_as_value(): # Test giving a key and a dictionary. with Mock() as mock: out.kv("dict", {1: 1}) assert len(mock) == 2 assert mock[0] == "dict:\n" assert mock[1] == " 1: 1\n"
def test_kv_dict(): # Test giving a dictionary. with Mock() as mock: out.kv({"level1": {"level2": {1: 1}}}) assert len(mock) == 3 assert mock[0] == "level1:\n" assert mock[1] == " level2:\n" assert mock[2] == " 1: 1\n"
def test_kv(): with Mock() as mock: out.kv("key", "value") out.kv(1.0, 1.0) out.kv(10.0, 10.0) out.kv(1000.0, 1000.0) out.kv(1000, 1000) assert len(mock) == 5 assert mock[0] == "key: value\n" assert mock[1] == "1.0: 1.0\n" assert mock[2] == "10.0: 10.0\n" assert mock[3] == "1.000e+03: 1.000e+03\n" assert mock[4] == "1000: 1000\n"
def _sync_folder(source: str, ip: str, target: RemotePath): with out.Section("Syncing to remote folder"): out.kv("Source", source) with out.Section("Target"): out.kv("Host", target.remote.host) out.kv("Path", target.path) try: ssh( target.remote, f'rsync -Pav -e "ssh -oStrictHostKeyChecking=no -i {config["ssh_key"]}"' f' {config["ssh_user"]}@{ip}:{source} {target.path}', ) except subprocess.CalledProcessError as e: out.kv("Synchronisation error", str(e))
def _sync_folder(source: str, ip: str, target: LocalPath): with out.Section("Syncing to local folder"): out.kv("Source", source) out.kv("Target", target.path) try: execute_command( "rsync", "-Pav", "-e", f'ssh -oStrictHostKeyChecking=no -i {config["ssh_key"]}', f'{config["ssh_user"]}@{ip}:{source}', target.path, ) except subprocess.CalledProcessError as e: out.kv("Synchronisation error", str(e))
def compare(model1, model2): """Compare two models. Args: model1 (str): Model folder of the first model to compare. model2 (str): Model folder of the second model to compare. """ mlls1, rmses1 = compute_metrics(model1, summarise=False) mlls2, rmses2 = compute_metrics(model2, summarise=False) with out.Section(f"{model1.upper()} - {model2.upper()}"): for name, values1, values2 in [("MLL", mlls1, mlls2), ("RMSE", rmses1, rmses2)]: diff = [x - y for x, y in zip(values1, values2)] with out.Section(name): mean = np.mean(diff) std = np.std(diff) / len(diff)**0.5 out.kv("Mean", mean) out.kv("Std", std) out.kv("p-value", st.norm.cdf(-abs(mean) / std))
y = (lik_noise + z_true).sample(100) # Define likelihood. def lik(x): return B.sum((lik_noise + y).logpdf(x)) # Construct q distribution. def construct_q(vs): q = Normal( Diagonal(vs.positive(shape=[d], name="q/var")), vs.get(shape=[d, 1], name="q/mean"), ) return q # Construct objective. def objective(vs): q = construct_q(vs) return -elbo(lik, prior, q, num_samples=1) # Optimise. objective_compiled = tf.function(objective, autograph=False) minimise_adam(objective_compiled, vs, trace=True, iters=2000, rate=5e-2) # Print result. out.kv("True", z_true) out.kv("Mean q", construct_q(vs).mean)
preds_f = {} preds_f_test = {} preds_k = {} preds_psd = {} for model in models: preds_f[model.name] = wd.load(model.name.lower(), "pred_f.pickle") preds_f_test[model.name] = wd.load(model.name.lower(), "pred_f_test.pickle") preds_k[model.name] = wd.load(model.name.lower(), "pred_k.pickle") preds_psd[model.name] = wd.load(model.name.lower(), "pred_psd.pickle") # Print performances. for name in ["GPCM", "CGPCM", "RGPCM"]: with out.Section(name): t, mean, var = preds_f_test[name] out.kv("RMSE", metric.rmse(mean, y_test)) out.kv("MLL", metric.mll(mean, var, y_test)) def plot_psd(name, y_label=True, style="pred", finish=True): """Plot prediction for the PSD.""" freqs, mean, lower, upper = preds_psd[name] freqs -= freqs[0] inds = freqs <= 0.2 freqs = freqs[inds] mean = mean[inds] lower = lower[inds] upper = upper[inds] if y_label:
k = wd_results.load(data, "data.pickle")["k"] t, mean, var = wd_results.load(data, scheme, model, "k_pred.pickle") inds = t <= until if metric == "smll": return smll(mean[inds], var[inds], k[inds]) elif metric == "rmse": return rmse(mean[inds], k[inds]) else: raise ValueError(f'Bad metric "{metric}".') for model, kernel in [("gpcm", "eq"), ("cgpcm", "ceq-1"), ("rgpcm", "matern12")]: with out.Section(model.upper()): with out.Section("SMLL"): out.kv("MF", kernel_analysis(kernel, "mean-field", model, "smll")) out.kv("S", kernel_analysis(kernel, "structured", model, "smll")) with out.Section("RMSE"): out.kv("MF", kernel_analysis(kernel, "mean-field", model, "rmse")) out.kv("S", kernel_analysis(kernel, "structured", model, "rmse")) def plot_kernel_predictions(model, data_name, legend=True, first=False): """Plot the prediction for a kernel.""" k = wd_results.load(data_name, "data.pickle")["k"] t, mean1, var1 = wd_results.load(data_name, "structured", model, "k_pred.pickle") t, mean2, var2 = wd_results.load(data_name, "mean-field", model, "k_pred.pickle") plt.plot(t, k, label="Truth", style="train") plt.plot(t, mean1, label="Structured", style="pred")
def test_kv_unit(): with Mock() as mock: out.kv("key", 1, unit="s") assert len(mock) == 1 assert mock[0] == "key: 1 s\n"
with Measure() as prior: # Construct a linear model. slope = GP(1) intercept = GP(5) f = slope * (lambda x: x) + intercept # Sample a slope, intercept, underlying function, and observations. true_slope, true_intercept, f_true, y_obs = prior.sample( slope(0), intercept(0), f(x), f(x_obs, 0.2) ) # Condition on the observations to make predictions. post = prior | (f(x_obs, 0.2), y_obs) mean, lower, upper = post(f(x)).marginal_credible_bounds() out.kv("True slope", true_slope[0, 0]) out.kv("Predicted slope", post(slope(0)).mean[0, 0]) out.kv("True intercept", true_intercept[0, 0]) out.kv("Predicted intercept", post(intercept(0)).mean[0, 0]) # Plot result. plt.plot(x, f_true, label="True", style="test") plt.scatter(x_obs, y_obs, label="Observations", style="train", s=20) plt.plot(x, mean, label="Prediction", style="pred") plt.fill_between(x, lower, upper, style="pred") tweak() plt.savefig("readme_example6_blr.png") plt.show()
model.fit(t, y, iters=30_000) k_pred_struc = extract(model.condition(t, y).predict_kernel(t_k)) psd_pred_struc = extract(model.condition(t, y).predict_psd()) wd.save((k_pred_mf, psd_pred_mf, k_pred_struc, psd_pred_struc), "preds.pickle") else: k_pred_mf, psd_pred_mf, k_pred_struc, psd_pred_struc = wd.load( "preds.pickle") # Report metrics. with out.Section("Structured"): t, mean, var, _, _ = k_pred_struc inds = t <= 3 out.kv("MLL", metric.mll(mean[inds], var[inds], k[inds])) out.kv("RMSE", metric.rmse(mean[inds], k[inds])) with out.Section("Mean field"): t, mean, var, _, _ = k_pred_mf inds = t <= 3 out.kv("MLL", metric.mll(mean[inds], var[inds], k[inds])) out.kv("RMSE", metric.rmse(mean[inds], k[inds])) plt.figure(figsize=(7.5, 3.75)) # Plot prediction for kernel. plt.subplot(1, 2, 1) plt.plot(t_k, k, label="Truth", style="train") t, mean, var, err_95_lower, err_95_upper = k_pred_struc plt.plot(t, mean, label="Structured", style="pred")
def manage_cluster( commands: List[List[str]], instance_type: str, key_name: str, security_group_id: str, image_id: str, sync_sources: List[str], sync_target: Path, monitor_aws_repo: str, monitor_call: str, monitor_delay: int, ): """Manage the cluster. Args: commands (list[list[str]]): One list of commands for every experiment. image_id (str): Image ID. instance_type (str): Type of the instance. key_name (str): Name of the key pair. security_group_id (str): Security group. sync_sources (list[str]): List of sources to sync. sync_target (:class:`.util.Path`): Directory to sync to. monitor_aws_repo (str, optional): Path to the root of this repo. The repo must consider the virtual environment "venv" which has the repo installed in editable mode. monitor_call (str): Call to start the monitor. See :mod:`.monitor`. monitor_delay (int): Number of seconds to wait before starting the monitor. """ parser = argparse.ArgumentParser() parser.add_argument( "--spawn", type=int, help="Spawn instances.", ) parser.add_argument( "--start", action="store_true", help="Start experiments.", ) parser.add_argument( "--terminate", action="store_true", help="Terminate all instances. This is a kill switch.", ) parser.add_argument( "--kill", action="store_true", help="Kill all running experiments, but keep the instances running.", ) parser.add_argument( "--stop", action="store_true", help="Stop all running instances", ) parser.add_argument( "--sync-stopped", action="store_true", help="Synchronise all stopped instances.", ) parser.add_argument( "--sync-sleep", default=120, type=int, help="Number of seconds to sleep before syncing again.", ) args = parser.parse_args() if args.sync_stopped: with out.Section("Syncing all stopped instances in five batches"): for batch in np.array_split(get_state("stopped"), 5): # Batches can be empty. if len(batch) == 0: continue # Start the instances. start(*batch) try: # Wait for the instances to have booted. out.out( "Waiting a minute for the instances to have booted...") time.sleep(60) # Refresh the instances to get the IPs. instance_ids = [ instance["InstanceId"] for instance in batch ] batch = get_instances(*instance_ids) # Sync. sync( sync_sources, sync_target, ips=[ instance["PublicIpAddress"] for instance in batch ], ) finally: # Stop the instances again. stop(*batch) out.out("Syncing completed: not continuing execution of script.") exit() if args.spawn: with out.Section("Starting all stopped instances"): start_stopped() with out.Section("Spawning instances"): spawn( image_id=image_id, total_count=args.spawn, instance_type=instance_type, key_name=key_name, security_group_id=security_group_id, ) while not check_all_running(): out.out("Waiting for all instances to be running...") time.sleep(5) out.out("Waiting a minute for all instances to have booted...") time.sleep(60) if args.kill: with out.Section("Killing all experiments"): kill_all() if args.stop: with out.Section("Stopping all instances"): stop_running() if args.terminate: with out.Section("Terminating all instances"): terminate_all() if args.start: num_instances = len(get_running_ips()) pieces = np.array_split(commands, num_instances) # Ensure that we have regular Python lists. pieces = [piece.tolist() for piece in pieces] with out.Section("Starting experiments"): out.kv("Number of commands", len(commands)) out.kv("Number of instances", num_instances) out.kv("Maximum runs per instance", max([len(piece) for piece in pieces])) ssh_map( *[[ *config["setup_commands"], *sum(piece, []), *config["teardown_commands"], ] for piece in pieces], start_experiment=True, in_experiment=True, start_monitor=True, monitor_aws_repo=monitor_aws_repo, monitor_delay=monitor_delay, monitor_call=monitor_call, ) while True: out.kv("Instances still running", len(get_running_ips())) sync(sync_sources, sync_target) out.out(f"Sleeping for {args.sync_sleep} second(s)...") time.sleep(args.sync_sleep)
post = f.measure | (f(x), f_true) y_obs = post(f(x_obs)).sample() def objective(vs): f, y = model(vs) evidence = y(x_obs).logpdf(y_obs) return -evidence # Learn hyperparameters. minimise_l_bfgs_b(tf.function(objective, autograph=False), vs) f, y = model(vs) # Print the learned parameters. out.kv("Prior", y.display(out.format)) vs.print() # Condition on the observations to make predictions. post = f.measure | (y(x_obs), y_obs) mean, lower, upper = post(f(x)).marginals() # Plot result. plt.plot(x, B.squeeze(f_true), label="True", style="test") plt.scatter(x_obs, B.squeeze(y_obs), label="Observations", style="train", s=20) plt.plot(x, mean, label="Prediction", style="pred") plt.fill_between(x, lower, upper, style="pred") tweak() plt.savefig("readme_example3_parametric.png") plt.show()
def test_kv_fmt(): with Mock() as mock: out.kv("key", 1, fmt=".4f") assert len(mock) == 1 assert mock[0] == "key: 1.0000\n"
x = B.linspace(0, 10, 100) x_obs = B.linspace(0, 7, 50_000) x_ind = B.linspace(0, 10, 20) # Construct a prior. f = GP(EQ().periodic(2 * B.pi)) # Sample a true, underlying function and observations. f_true = B.sin(x) y_obs = B.sin(x_obs) + B.sqrt(0.5) * B.randn(*x_obs.shape) # Compute a pseudo-point approximation of the posterior. obs = PseudoObs(f(x_ind), (f(x_obs, 0.5), y_obs)) # Compute the ELBO. out.kv("ELBO", obs.elbo(f.measure)) # Compute the approximate posterior. f_post = f | obs # Make predictions with the approximate posterior. mean, lower, upper = f_post(x).marginal_credible_bounds() # Plot result. plt.plot(x, f_true, label="True", style="test") plt.scatter( x_obs, y_obs, label="Observations", style="train", c="tab:green",
# Setup script. wd = WorkingDirectory("_experiments", "crude_oil_aggregate") # Load all experiments and compute metrics. names = ["GPCM", "CGPCM", "RGPCM"] mlls = {name: [] for name in names} rmses = {name: [] for name in names} for year in range(2012, 2017 + 1): wd_results = WorkingDirectory("_experiments", "crude_oil", str(year), observe=True) t, y = wd_results.load("data.pickle")["test"] for name in names: _, mean, var = wd_results.load(name.lower(), "pred_f_test.pickle") mlls[name].append(metric.mll(mean, var, y)) rmses[name].append(metric.rmse(mean, y)) # Print aggregate results. for name in names: with out.Section(name): out.kv("MLL", np.mean(mlls[name])) out.kv("MLL (std)", np.std(mlls[name]) / len(mlls[name]) ** 0.5) out.kv("RMSE", np.mean(rmses[name])) out.kv("RMSE (std)", np.std(rmses[name]) / len(rmses[name]) ** 0.5) # Compare results. for name1, name2 in [("RGPCM", "CGPCM"), ("RGPCM", "GPCM"), ("CGPCM", "GPCM")]: with out.Section(f"{name1} - {name2}"): out.kv("MLL", np.mean(mlls[name1]) - np.mean(mlls[name2])) out.kv("MLL (p)", ttest_rel(mlls[name1], mlls[name2]).pvalue) out.kv("RMSE", np.mean(rmses[name1]) - np.mean(rmses[name2])) out.kv("RMSE (p)", ttest_rel(rmses[name1], rmses[name2]).pvalue)