def test_ml():
logger.info("Loading instances...")
train_instances = load("%s/train_instances.bin" % basepath)
test_instances = load("%s/test_instances.bin" % basepath)
solvers = {
"ml-exact":
LearningSolver(
time_limit=time_limit,
solver=internal_solver,
),
"ml-heuristic":
LearningSolver(
time_limit=time_limit,
solver=internal_solver,
mode="heuristic",
),
}
benchmark = BenchmarkRunner(solvers)
logger.info("Loading results...")
benchmark.load_results("%s/benchmark_baseline.csv" % basepath)
logger.info("Fitting...")
benchmark.fit(train_instances)
logger.info("Solving...")
benchmark.parallel_solve(test_instances, n_jobs=n_jobs)
benchmark.save_results("%s/benchmark_ml.csv" % basepath)
def test_baseline():
test_instances = load("%s/test_instances.bin" % basepath)
solvers = {
"baseline": LearningSolver(
time_limit=test_time_limit,
solver=internal_solver,
),
}
benchmark = BenchmarkRunner(solvers)
benchmark.parallel_solve(test_instances, n_jobs=n_jobs)
benchmark.save_results("%s/benchmark_baseline.csv" % basepath)
def test_benchmark():
# Generate training and test instances
train_instances = MaxWeightStableSetGenerator(
n=randint(low=25, high=26)).generate(5)
test_instances = MaxWeightStableSetGenerator(
n=randint(low=25, high=26)).generate(3)
# Training phase...
training_solver = LearningSolver()
training_solver.parallel_solve(train_instances, n_jobs=10)
# Test phase...
test_solvers = {
"Strategy A": LearningSolver(),
"Strategy B": LearningSolver(),
}
benchmark = BenchmarkRunner(test_solvers)
benchmark.fit(train_instances)
benchmark.parallel_solve(test_instances, n_jobs=2, n_trials=2)
assert benchmark.raw_results().values.shape == (12, 16)
benchmark.save_results("/tmp/benchmark.csv")
assert os.path.isfile("/tmp/benchmark.csv")
benchmark = BenchmarkRunner(test_solvers)
benchmark.load_results("/tmp/benchmark.csv")
assert benchmark.raw_results().values.shape == (12, 16)
def charts():
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")
sns.set_palette("Blues_r")
benchmark = BenchmarkRunner({})
benchmark.load_results("%s/benchmark_ml.csv" % basepath)
results = benchmark.raw_results()
results["Gap (%)"] = results["Gap"] * 100.0
sense = results.loc[0, "Sense"]
if sense == "min":
primal_column = "Relative Upper Bound"
obj_column = "Upper Bound"
predicted_obj_column = "Predicted UB"
else:
primal_column = "Relative Lower Bound"
obj_column = "Lower Bound"
predicted_obj_column = "Predicted LB"
palette = {
"baseline": "#9b59b6",
"ml-exact": "#3498db",
"ml-heuristic": "#95a5a6"
}
fig, (ax1, ax2, ax3, ax4) = plt.subplots(
nrows=1,
ncols=4,
figsize=(12, 4),
gridspec_kw={'width_ratios': [2, 1, 1, 2]},
)
sns.stripplot(
x="Solver",
y="Wallclock Time",
data=results,
ax=ax1,
jitter=0.25,
palette=palette,
size=4.0,
)
sns.barplot(
x="Solver",
y="Wallclock Time",
data=results,
ax=ax1,
errwidth=0.,
alpha=0.4,
palette=palette,
estimator=median,
)
ax1.set(ylabel='Wallclock Time (s)')
ax2.set_ylim(-0.5, 5.5)
sns.stripplot(
x="Solver",
y="Gap (%)",
jitter=0.25,
data=results[results["Solver"] != "ml-heuristic"],
ax=ax2,
palette=palette,
size=4.0,
)
ax3.set_ylim(0.95, 1.05)
sns.stripplot(
x="Solver",
y=primal_column,
jitter=0.25,
data=results[results["Solver"] == "ml-heuristic"],
ax=ax3,
palette=palette,
)
sns.scatterplot(
x=obj_column,
y=predicted_obj_column,
hue="Solver",
data=results[results["Solver"] == "ml-exact"],
ax=ax4,
palette=palette,
)
xlim, ylim = ax4.get_xlim(), ax4.get_ylim()
ax4.plot([-1e10, 1e10], [-1e10, 1e10], ls='-', color="#cccccc")
ax4.set_xlim(xlim)
ax4.set_ylim(ylim)
ax4.get_legend().remove()
fig.tight_layout()
plt.savefig("%s/performance.png" % basepath, bbox_inches='tight', dpi=150)