def test_paired_permutation(self):
""" Testing paired permutation test. """
rng = np.random.RandomState(42)
a, b = rng.normal(loc=0, size=100), rng.normal(loc=0, size=100)
result = paired_permutation(a, a, rng, 100, self.logger)
self.assertGreater(result, 0.9999)
result = paired_permutation(a, b, rng, 100, self.logger)
self.assertGreater(result, 0.3)
a, b = rng.normal(loc=-1, size=100), rng.normal(loc=1, size=100)
result = paired_permutation(a, b, rng, 1000, self.logger)
self.assertLess(result, 0.001)
def _permutation_test(self,
epm_rh,
default,
incumbent,
num_permutations,
par=1):
if par != 1 and not self.scenario.cutoff:
return np.nan
cutoff = self.scenario.cutoff
def_cost = get_cost_dict_for_config(epm_rh,
default,
par=par,
cutoff=cutoff)
inc_cost = get_cost_dict_for_config(epm_rh,
incumbent,
par=par,
cutoff=cutoff)
data1, data2 = zip(*[(def_cost[i], inc_cost[i])
for i in def_cost.keys()])
p = paired_permutation(data1,
data2,
self.rng,
num_permutations=num_permutations,
logger=self.logger)
self.logger.debug(
"p-value for def/inc-difference: %f (permutation test "
"with %d permutations and par %d)", p, num_permutations, par)
return p
def create_performance_table(self, default, incumbent, epm_rh, oracle):
"""Create table, compare default against incumbent on train-,
test- and combined instances. Listing PAR10, PAR1 and timeouts.
Distinguishes between train and test, if available."""
self.logger.info("... create performance table")
cost_dict_def = get_cost_dict_for_config(epm_rh, default)
cost_dict_inc = get_cost_dict_for_config(epm_rh, incumbent)
def_par1, inc_par1 = self.get_parX(cost_dict_def,
1), self.get_parX(cost_dict_inc, 1)
def_par10, inc_par10 = self.get_parX(cost_dict_def, 10), self.get_parX(
cost_dict_inc, 10)
ora_par1, ora_par10 = self.get_parX(oracle,
1), self.get_parX(oracle, 10)
def_timeouts = get_timeout(epm_rh, default, self.scenario.cutoff)
inc_timeouts = get_timeout(epm_rh, incumbent, self.scenario.cutoff)
def_timeouts_tuple = self.timeouts_to_tuple(def_timeouts)
inc_timeouts_tuple = self.timeouts_to_tuple(inc_timeouts)
if self.scenario.cutoff:
ora_timeout = self.timeouts_to_tuple(
{i: c < self.scenario.cutoff
for i, c in oracle.items()})
data1, data2 = zip(*[(int(def_timeouts[i]), int(inc_timeouts[i]))
for i in def_timeouts.keys()])
p_value_timeouts = "%.5f" % paired_permutation(
data1,
data2,
self.rng,
num_permutations=10000,
logger=self.logger)
else:
ora_timeout = self.timeouts_to_tuple({})
p_value_timeouts = "N/A"
# p-values (paired permutation)
p_value_par10 = self._permutation_test(epm_rh, default, incumbent,
10000, 10)
p_value_par10 = "%.5f" % p_value_par10 if np.isfinite(
p_value_par10) else 'N/A'
p_value_par1 = self._permutation_test(epm_rh, default, incumbent,
10000, 1)
p_value_par1 = "%.5f" % p_value_par1 if np.isfinite(
p_value_par1) else 'N/A'
dec_place = 3
metrics = []
if self.scenario.run_obj == 'runtime':
metrics.append('PAR10')
metrics.append('PAR1')
else:
metrics.append('Quality')
if self.scenario.cutoff:
metrics.append('Timeouts')
train, test = len(self.scenario.train_insts) > 1, len(
self.scenario.test_insts) > 1
oracle = train or test # oracle only makes sense with instances
# Create table
array = []
if 'PAR10' in metrics:
if train and test:
values = [
def_par10[0], inc_par10[0], ora_par10[0], def_par10[1],
inc_par10[1], ora_par10[1]
]
elif oracle:
values = [def_par10, inc_par10,
ora_par10] # oracle only with instances
else:
values = [def_par10, inc_par10]
values = [
round(value, dec_place) if np.isfinite(value) else 'N/A'
for value in values
]
if train or test:
values.append(p_value_par10)
array.append(values)
if 'PAR1' in metrics or 'Quality' in metrics:
if train and test:
values = [
def_par1[0], inc_par1[0], ora_par1[0], def_par1[1],
inc_par1[1], ora_par1[1]
]
elif oracle:
values = [def_par1, inc_par1,
ora_par1] # oracle only with instances
else:
values = [def_par1, inc_par1]
values = [
round(value, dec_place) if np.isfinite(value) else 'N/A'
for value in values
]
if train or test:
values.append(p_value_par1)
array.append(values)
if 'Timeouts' in metrics:
if train and test:
values = [
"{}/{}".format(def_timeouts_tuple[0][0],
def_timeouts_tuple[0][1]),
"{}/{}".format(inc_timeouts_tuple[0][0],
inc_timeouts_tuple[0][1]),
"{}/{}".format(ora_timeout[0][0], ora_timeout[0][1]),
"{}/{}".format(def_timeouts_tuple[1][0],
def_timeouts_tuple[1][1]),
"{}/{}".format(inc_timeouts_tuple[1][0],
inc_timeouts_tuple[1][1]),
"{}/{}".format(ora_timeout[1][0], ora_timeout[1][1]),
]
elif oracle:
values = [
"{}/{}".format(def_timeouts_tuple[0],
def_timeouts_tuple[1]),
"{}/{}".format(inc_timeouts_tuple[0],
inc_timeouts_tuple[1]),
"{}/{}".format(ora_timeout[0], ora_timeout[1])
]
else:
values = [
"{}/{}".format(def_timeouts_tuple[0],
def_timeouts_tuple[1]),
"{}/{}".format(inc_timeouts_tuple[0],
inc_timeouts_tuple[1]),
]
if train or test:
values.append(p_value_timeouts)
array.append(values)
array = np.array(array)
columns = ['Default', 'Incumbent']
if oracle:
columns.append('Oracle')
if train and test:
columns = columns + columns
if train or test:
columns.append('p-value')
self.logger.debug(array)
self.logger.debug(columns)
df = DataFrame(data=array, index=metrics, columns=columns)
table = df.to_html()
if train and test:
# Insert two-column-header
table = table.split(sep='</thead>', maxsplit=1)[1]
new_table = "<table border=\"3\" class=\"dataframe\">\n"\
" <col>\n"\
" <colgroup span=\"2\"></colgroup>\n"\
" <colgroup span=\"2\"></colgroup>\n"\
" <thead>\n"\
" <tr>\n"\
" <td rowspan=\"2\"></td>\n"\
" <th colspan=\"3\" scope=\"colgroup\">Train</th>\n"\
" <th colspan=\"3\" scope=\"colgroup\">Test</th>\n"\
" <th colspan=\"1\" scope=\"colgroup\">p-value</th>\n"\
" </tr>\n"\
" <tr>\n"\
" <th scope=\"col\">Default</th>\n"\
" <th scope=\"col\">Incumbent</th>\n"\
" <th scope=\"col\">Oracle</th>\n"\
" <th scope=\"col\">Default</th>\n"\
" <th scope=\"col\">Incumbent</th>\n"\
" <th scope=\"col\">Oracle</th>\n"\
" </tr>\n"\
"</thead>\n"
table = new_table + table
self.table = table
self.dataframe = df
return df