def get_tag_ac_4_p(acs, nc, cache=None):
"""
:param acs: all configurations.
acs can either be a list of strings or list of NT_ModelResults.
*********************************************************
acs should already be sorted by MTRY, i.e,
the default configuration (MTRY=1.0) MUST BE LAST
*********************************************************
:param nc: the nominal confidence
:param cache: this function gets called a lot, so we cache all the NT_ModeResults.
:return: scenario tag
"""
nc_ = nc / 100
if isinstance(acs[0], str):
c_ptrs_ = acs[:]
if cache is None:
cache = {}
if not cache:
default_ptr = c_ptrs_.pop(-1)
default_cfg = load_pickled_resource(default_ptr)
cfgs = [load_pickled_resource(c_ptr) for c_ptr in c_ptrs_]
for c_ptr, cfg in zip(acs, cfgs + [default_cfg]):
cache[c_ptr] = cfg
else:
default_ptr = c_ptrs_.pop(-1)
default_cfg = cache[default_ptr]
cfgs = [cache[c_ptr] for c_ptr in c_ptrs_]
else:
default_cfg = acs[-1]
cfgs = acs[:-1]
d_cov = pi.cov(default_cfg, nc)
d_mpiw = pi.mw(default_cfg, nc, precision=8)
coverages = arr([pi.cov(cfg, nc) for cfg in cfgs])
candidates = np.where(coverages >= nc_)[0]
cf_cov = cf_mpiw = p = None
if candidates.size:
mpiws = arr([pi.mw(cfgs[i], nc, precision=8) for i in candidates])
selected = mpiws.argmin()
z, p = Stats.wilcoxon_signed_rank_test(
pi.w(default_cfg, nc), pi.w(cfgs[candidates[selected]], nc))
cf_cov = coverages[candidates[selected]]
cf_mpiw = mpiws[selected]
return tag_cf_4_p(nc, d_cov, d_mpiw, cf_cov, cf_mpiw, p)
def coverage_acf_anc_4_p(acs, ncs):
res = []
acs_ = _get_configs(acs)
for c in ncs:
res.append([pi.cov(cf, c, precision=2) for cf in acs_])
return np.transpose(res)
def meta_tags_4_anc_4_p(mt_ptrs,
t_ptrs,
m_dcf_ptr,
dc_ptr,
selectors,
ncs,
itag=False):
cr = choose_m_4_anc(mt_ptrs, m_dcf_ptr, selectors, ncs)
dcf = load_pickled_resource(dc_ptr)
d_c, d_mw, d_w = _coverage_mw_widths(dcf, ncs)
tags = []
for i, nc in enumerate(ncs):
cf = choose_min(cr, selectors, t_ptrs, i, nc)
cov = mw = p = None
if cf:
cov = pi.cov(cf, nc)
mw = pi.mw(cf, nc)
_, p = Stats.wilcoxon_signed_rank_test(pi.w(cf, nc), d_w[i])
if p != p:
p = 1.0
idx, tag = tag_cf_4_p(nc, d_c[i], d_mw[i], cov, mw, p)
if itag:
tags.append(idx)
else:
tags.append(tag)
return tags
def clf_accuracy_4_at_4_anc(at_ptrs, cf_ptrs, ncs):
truth_4_nc = []
cfs = [load_pickled_resource(cfp) for cfp in cf_ptrs]
for nc in ncs:
truth_4_nc.append(
arr([1 if pi.cov(cf, nc) >= nc / 100 else 0 for cf in cfs]))
del cfs
at_cfs = [_load_cfs_4_t_4_p(t_ptrs) for t_ptrs in at_ptrs]
te_scores = []
f1_scores_bs = []
for i, nc in enumerate(ncs):
truth = truth_4_nc[i]
predictions = []
for cfs in at_cfs:
predictions.append(
arr([1 if pi.cov(cf, nc) >= nc / 100 else 0 for cf in cfs]))
# get the base scores
te_scores.append(
[precision_recall_f1(truth, preds) for preds in predictions])
# now bootstrap
f1_scores = []
for _ in range(100):
s = np.random.choice(np.arange(truth.size), truth.size)
# f1 for each technique
f1 = []
for preds in predictions:
f1.append(precision_recall_f1(truth[s], preds[s])[2])
f1_scores.append(f1)
f1_scores_bs.append(f1_scores)
del at_cfs
return np.round(te_scores, 4), np.round(f1_scores_bs, 4)
def m_tags_4_anc(at_ptrs, dcf_ptr, selectors, ncs, dcf=None, itag=True):
if dcf is None:
dcf = load_pickled_resource(dcf_ptr)
d_c, d_mw, d_w = _coverage_mw_widths(dcf, ncs)
del dcf
else:
d_c, d_mw, d_w = _coverage_mw_widths(dcf, ncs)
data = []
at_cfs = [_load_cfs_4_t_4_p(t_ptrs) for t_ptrs in at_ptrs]
for i, nc in enumerate(ncs):
t_data = []
for j, t_cfs in enumerate(at_cfs):
s_cfs = [
selector.pick(at_ptrs[j], t_cfs, nc) for selector in selectors
]
data_ = []
for s_cf, s_id in s_cfs:
cov = mw = p = None
if s_cf:
cov = pi.cov(s_cf, nc)
mw = pi.mw(s_cf, nc)
_, p = Stats.wilcoxon_signed_rank_test(
pi.w(s_cf, nc), d_w[i])
if p != p:
p = 1.0
i_tag, tag = tag_cf_4_p(nc, d_c[i], d_mw[i], cov, mw, p)
if itag:
data_.append(i_tag)
else:
data_.append(tag)
t_data.extend(data_)
data.append(t_data)
del at_cfs
return arr(data)
def get_coverages(techniques, nc):
return arr([pi.cov(te, nc, precision=2) for te in techniques])