def test_tsemo(test_num_improve_iter=2, save=False):
num_inputs = 2
num_objectives = 2
lab = VLMOP2()
strategy = TSEMO(lab.domain)
experiments = strategy.suggest_experiments(5 * num_inputs)
warnings.filterwarnings("ignore", category=RuntimeWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning)
num_improve_iter = 0
best_hv = None
pb = progress_bar(range(20))
for i in pb:
# Run experiments
experiments = lab.run_experiments(experiments)
# Get suggestions
experiments = strategy.suggest_experiments(1, experiments)
if save:
strategy.save("tsemo_settings.json")
y_pareto, _ = pareto_efficient(lab.data[["y_0", "y_1"]].to_numpy(),
maximize=False)
hv = hypervolume(y_pareto, [11, 11])
if best_hv == None:
best_hv = hv
elif hv > best_hv:
best_hv = hv
num_improve_iter += 1
pb.comment = f"Hypervolume: {hv}"
if num_improve_iter >= test_num_improve_iter:
print(
"Requirement to improve fbest in at least {} satisfied, test stopped."
.format(test_num_improve_iter))
break
def _create_param_df(self, reference=[-2957, 10.7]):
"""Create a parameters dictionary
Parameters
----------
reference : array-like, optional
Reference for the hypervolume calculatio
"""
records = []
for experiment_id, r in self.runners.items():
record = {}
record["experiment_id"] = experiment_id
# Transform
transform_name = r.strategy.transform.__class__.__name__
transform_params = r.strategy.transform.to_dict(
)["transform_params"]
record["transform_name"] = transform_name
if transform_name == "Chimera":
hierarchy = transform_params["hierarchy"]
for objective_name, v in hierarchy.items():
key = f"{objective_name}_tolerance"
record[key] = v["tolerance"]
elif transform_name == "MultitoSingleObjective":
record.update(transform_params)
# Strategy
record["strategy_name"] = r.strategy.__class__.__name__
# Batch size
record["batch_size"] = r.batch_size
# Number of initial experiments
try:
record["num_initial_experiments"] = r.n_init
except AttributeError:
pass
# Terminal hypervolume
data = r.experiment.data[["sty", "e_factor"]].to_numpy()
data[:, 0] *= -1 # make it a minimzation problem
y_front, _ = pareto_efficient(data[:self.trajectory_length, :],
maximize=False)
hv = hypervolume(y_front, ref=reference)
record["terminal_hypervolume"] = hv
# Computation time
time = (r.experiment.data["computation_t"].
iloc[0:self.trajectory_length].sum())
record["computation_t"] = time
record["noise_level"] = r.experiment.noise_level
records.append(record)
# Make pandas dataframe
self.df = pd.DataFrame.from_records(records)
return self.df
def run(self, **kwargs):
""" Run the closed loop experiment cycle
Parameters
----------
save_freq : int, optional
The frequency with which to checkpoint the state of the optimization. Defaults to None.
save_at_end : bool, optional
Save the state of the optimization at the end of a run, even if it is stopped early.
Default is True.
save_dir : str, optional
The directory to save checkpoints locally. Defaults to `~/.summit/runner`.
"""
# Set parameters
prev_res = None
self.restarts = 0
n_objs = len(self.experiment.domain.output_variables)
fbest_old = np.zeros(n_objs)
fbest = np.zeros(n_objs)
# Serialization
save_freq = kwargs.get("save_freq")
save_dir = kwargs.get("save_dir", str(get_summit_config_path()))
self.uuid_val = uuid.uuid4()
save_dir = pathlib.Path(save_dir) / "runner" / str(self.uuid_val)
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
save_at_end = kwargs.get("save_at_end", True)
# Create neptune experiment
if self.neptune_exp is None:
session = Session(backend=HostedNeptuneBackend())
proj = session.get_project(self.neptune_project)
neptune_exp = proj.create_experiment(
name=self.neptune_experiment_name,
description=self.neptune_description,
upload_source_files=self.neptune_files,
logger=self.logger,
tags=self.neptune_tags,
)
else:
neptune_exp = self.neptune_exp
# Run optimization loop
for i in progress_bar(range(self.max_iterations)):
# Get experiment suggestions
if i == 0:
k = self.n_init if self.n_init is not None else self.batch_size
next_experiments = self.strategy.suggest_experiments(
num_experiments=k)
else:
next_experiments = self.strategy.suggest_experiments(
num_experiments=self.batch_size, prev_res=prev_res)
prev_res = self.experiment.run_experiments(next_experiments)
# Send best objective values to Neptune
for j, v in enumerate(self.experiment.domain.output_variables):
if i > 0:
fbest_old[j] = fbest[j]
if v.maximize:
fbest[j] = self.experiment.data[v.name].max()
elif not v.maximize:
fbest[j] = self.experiment.data[v.name].min()
neptune_exp.send_metric(v.name + "_best", fbest[j])
# Send hypervolume for multiobjective experiments
if n_objs > 1:
output_names = [
v.name for v in self.experiment.domain.output_variables
]
data = self.experiment.data[output_names].copy()
for v in self.experiment.domain.output_variables:
if v.maximize:
data[(v.name, "DATA")] = -1.0 * data[v.name]
y_pareto, _ = pareto_efficient(data.to_numpy(), maximize=False)
hv = hypervolume(y_pareto, self.ref)
neptune_exp.send_metric("hypervolume", hv)
# Save state
if save_freq is not None:
file = save_dir / f"iteration_{i}.json"
if i % save_freq == 0:
self.save(file)
neptune_exp.send_artifact(str(file))
if not save_dir:
os.remove(file)
# Stop if no improvement
compare = np.abs(fbest - fbest_old) > self.f_tol
if all(compare) or i <= 1:
nstop = 0
else:
nstop += 1
if self.max_same is not None:
if nstop >= self.max_same and self.restarts >= self.max_restarts:
self.logger.info(
f"{self.strategy.__class__.__name__} stopped after {i+1} iterations and {self.restarts} restarts."
)
break
elif nstop >= self.max_same:
nstop = 0
prev_res = None
self.strategy.reset()
self.restarts += 1
# Save at end
if save_at_end:
file = save_dir / f"iteration_{i}.json"
self.save(file)
neptune_exp.send_artifact(str(file))
if not save_dir:
os.remove(file)
# Stop the neptune experiment
neptune_exp.stop()
def pareto_plot(self, objectives=None, colorbar=False, ax=None):
"""Make a 2D pareto plot of the experiments thus far
Parameters
----------
objectives: array-like, optional
List of names of objectives to plot.
By default picks the first two objectives
ax: `matplotlib.pyplot.axes`, optional
An existing axis to apply the plot to
Returns
-------
if ax is None returns a tuple with the first component
as the a new figure and the second component the axis
if ax is a matplotlib axis, returns only the axis
Raises
------
ValueError
If the number of objectives is not equal to two
"""
if objectives is None:
objectives = [
v.name for v in self.domain.variables if v.is_objective
]
objectives = objectives[0:2]
if len(objectives) != 2:
raise ValueError("Can only plot 2 objectives")
data = self._data[objectives].copy()
# Handle minimize objectives
for objective in objectives:
if not self.domain[objective].maximize:
data[objective] = -1.0 * data[objective]
values, indices = pareto_efficient(data.to_numpy(), maximize=True)
if ax is None:
fig, ax = plt.subplots(1)
return_fig = True
else:
return_fig = False
# Plot all data
if len(self.data) > 0:
strategies = pd.unique(self.data["strategy"])
markers = ["o", "x"]
for strategy, marker in zip(strategies, markers):
strat_data = self.data[self.data["strategy"] == strategy]
c = strat_data.index.values if colorbar else "k"
cmap = ListedColormap(COLORS[:len(c)])
im = ax.scatter(
strat_data[objectives[0]],
strat_data[objectives[1]],
cmap=cmap,
c=c,
alpha=1 if colorbar else 0.5,
marker=marker,
s=100,
label=strategy,
)
# Sort data so get nice pareto plot
self.pareto_data = self.data.iloc[indices].copy()
self.pareto_data = self.pareto_data.sort_values(by=objectives[0])
if len(self.pareto_data) > 2:
ax.plot(
self.pareto_data[objectives[0]],
self.pareto_data[objectives[1]],
c=(165 / 256, 0, 38 / 256),
label="Pareto Front",
linewidth=3,
)
ax.set_xlabel(objectives[0])
ax.set_ylabel(objectives[1])
if return_fig and colorbar:
fig.colorbar(im)
ax.tick_params(direction="in")
ax.legend()
if return_fig:
return fig, ax
elif return_fig and colorbar:
return fig, ax, im
elif not return_fig and colorbar:
return ax, im
else:
return ax
def plot_hv_trajectories(
self,
reference=[-2957, 10.7],
plot_type="matplotlib",
include_experiment_ids=False,
min_terminal_hv_avg=0,
ax=None,
):
""" Plot the hypervolume trajectories with repeats as 95% confidence interval
Parameters
----------
reference : array-like, optional
Reference for the hypervolume calculation. Defaults to -2957, 10.7
plot_type : str, optional
Plotting backend to use: matplotlib or plotly. Defaults to matplotlib.
include_experiment_ids : bool, optional
Whether to include experiment ids in the plot labels
min_terminal_hv_avg : float, optional`
Minimum terminal average hypervolume cutoff for inclusion in the plot. Defaults to 0.
"""
# Create figure
if plot_type == "matplotlib":
if ax is not None:
fig = None
else:
fig, ax = plt.subplots(1)
elif plot_type == "plotly":
fig = go.Figure()
else:
raise ValueError(
f"{plot_type} is not a valid plot type. Must be matplotlib or plotly."
)
# Group experiment repeats
df = self.df.copy()
df = df.set_index("experiment_id")
df = df.drop(columns=["terminal_hypervolume", "computation_t"])
uniques = df.drop_duplicates(keep="last") # This actually groups them
df_new = self.df.copy()
if plot_type == "plotly":
colors = px.colors.qualitative.Plotly
else:
colors = COLORS
cycle = len(colors)
c_num = 0
self.hv = {}
for index, unique in uniques.iterrows():
# Find number of matching rows to this unique row
temp_df = df_new.merge(unique.to_frame().transpose(), how="inner")
ids = temp_df["experiment_id"].values
# Calculate hypervolume trajectories
ids = ids if len(ids) < self.num_repeats else ids[:self.
num_repeats]
hv_trajectories = np.zeros([self.trajectory_length, len(ids)])
for j, experiment_id in enumerate(ids):
r = self.runners[experiment_id]
data = r.experiment.data[["sty", "e_factor"]].to_numpy()
data[:, 0] *= -1 # make it a minimzation problem
for i in range(self.trajectory_length):
y_front, _ = pareto_efficient(data[0:i + 1, :],
maximize=False)
hv_trajectories[i, j] = hypervolume(y_front, ref=reference)
# Mean and standard deviation
hv_mean_trajectory = np.mean(hv_trajectories, axis=1)
hv_std_trajectory = np.std(hv_trajectories, axis=1)
if hv_mean_trajectory[-1] < min_terminal_hv_avg:
continue
# Update plot
t = np.arange(1, self.trajectory_length + 1)
# label = self._create_label(unique)
transform = unique["transform_name"]
if transform == "MultitoSingleObjective":
transform = "Custom"
label = (f"""{unique["strategy_name"]} ({transform})"""
if transform is not "Transform" else
f"""{unique["strategy_name"]}""")
if include_experiment_ids:
label += f" ({ids[0]}-{ids[-1]})"
lower = hv_mean_trajectory - 1.96 * hv_std_trajectory
lower = np.clip(lower, 0, None)
upper = hv_mean_trajectory + 1.96 * hv_std_trajectory
if plot_type == "matplotlib":
ax.plot(t,
hv_mean_trajectory,
label=label,
color=colors[c_num])
ax.fill_between(t,
lower,
upper,
alpha=0.1,
color=colors[c_num])
elif plot_type == "plotly":
r, g, b = hex_to_rgb(colors[c_num])
color = lambda alpha: f"rgba({r},{g},{b},{alpha})"
fig.add_trace(
go.Scatter(
x=t,
y=hv_mean_trajectory,
mode="lines",
name=label,
line=dict(color=color(1)),
legendgroup=label,
))
fig.add_trace(
go.Scatter(
x=t,
y=lower,
mode="lines",
fill="tonexty",
line=dict(width=0),
fillcolor=color(0.1),
showlegend=False,
legendgroup=label,
))
fig.add_trace(
go.Scatter(
x=t,
y=upper,
mode="lines",
fill="tozeroy",
line=dict(width=0),
fillcolor=color(0.1),
showlegend=False,
legendgroup=label,
))
if cycle == c_num + 1:
c_num = 0
elif plot_type == "plotly":
c_num += 1
elif plot_type == "matplotlib":
c_num += 2
# Plot formattting
if plot_type == "matplotlib":
ax.set_xlabel("Experiments")
ax.set_ylabel("Hypervolume")
legend = ax.legend(loc="upper left")
ax.tick_params(direction="in")
ax.set_xlim(1, self.trajectory_length)
if fig is None:
return ax, legend
else:
return fig, ax, legend
elif plot_type == "plotly":
fig.update_layout(xaxis=dict(title="Experiments"),
yaxis=dict(title="Hypervolume"))
fig.show()
return fig
def _select_max_hvi(self, y, samples, num_evals=1):
"""Returns the point(s) that maximimize hypervolume improvement
Parameters
----------
samples: np.ndarray
The samples on which hypervolume improvement is calculated
num_evals: `int`
The number of points to return (with top hypervolume improvement)
Returns
-------
hv_imp, index
Returns a tuple with lists of the best hypervolume improvement
and the indices of the corresponding points in samples
"""
samples_original = samples.copy()
samples = samples.copy()
y = y.copy()
# Set up maximization and minimization
for v in self.domain.variables:
if v.is_objective and v.maximize:
y[v.name] = -1 * y[v.name]
samples[v.name] = -1 * samples[v.name]
# samples, mean, std = samples.standardize(return_mean=True, return_std=True)
samples = samples.data_to_numpy()
Ynew = y.data_to_numpy()
# Reference
Yfront, _ = pareto_efficient(Ynew, maximize=False)
r = np.max(
Yfront,
axis=0) + 0.01 * (np.max(Yfront, axis=0) - np.min(Yfront, axis=0))
indices = []
n = samples.shape[1]
mask = np.ones(samples.shape[0], dtype=bool)
samples_indices = np.arange(0, samples.shape[0])
for i in range(num_evals):
masked_samples = samples[mask, :]
Yfront, _ = pareto_efficient(Ynew, maximize=False)
if len(Yfront) == 0:
raise ValueError("Pareto front length too short")
hv_improvement = []
hvY = hypervolume(Yfront, r)
# Determine hypervolume improvement by including
# each point from samples (masking previously selected poonts)
for sample in masked_samples:
sample = sample.reshape(1, n)
A = np.append(Ynew, sample, axis=0)
Afront, _ = pareto_efficient(A, maximize=False)
hv = hypervolume(Afront, r)
hv_improvement.append(hv - hvY)
hvY0 = hvY if i == 0 else hvY0
hv_improvement = np.array(hv_improvement)
masked_index = np.argmax(hv_improvement)
# Housekeeping: find the max HvI point and mask out for next round
original_index = samples_indices[mask][masked_index]
new_point = samples[original_index, :].reshape(1, n)
Ynew = np.append(Ynew, new_point, axis=0)
mask[original_index] = False
indices.append(original_index)
# Append current estimate of the pareto front to sample_paretos
samples_copy = samples_original.copy()
samples_copy = samples_copy * self.output_std + self.output_mean
samples_copy[("hvi", "DATA")] = hv_improvement
self.samples.append(samples_copy)
if len(hv_improvement) == 0:
hv_imp = 0
elif len(indices) == 0:
indices = []
hv_imp = 0
else:
# Total hypervolume improvement
# Includes all points added to batch (hvY + last hv_improvement)
# Subtracts hypervolume without any points added (hvY0)
hv_imp = hv_improvement[masked_index] + hvY - hvY0
return hv_imp, indices