def get_tag_ac_4_p(acs, nc, cache=None): """ :param acs: all configurations. acs can either be a list of strings or list of NT_ModelResults. ********************************************************* acs should already be sorted by MTRY, i.e, the default configuration (MTRY=1.0) MUST BE LAST ********************************************************* :param nc: the nominal confidence :param cache: this function gets called a lot, so we cache all the NT_ModeResults. :return: scenario tag """ nc_ = nc / 100 if isinstance(acs[0], str): c_ptrs_ = acs[:] if cache is None: cache = {} if not cache: default_ptr = c_ptrs_.pop(-1) default_cfg = load_pickled_resource(default_ptr) cfgs = [load_pickled_resource(c_ptr) for c_ptr in c_ptrs_] for c_ptr, cfg in zip(acs, cfgs + [default_cfg]): cache[c_ptr] = cfg else: default_ptr = c_ptrs_.pop(-1) default_cfg = cache[default_ptr] cfgs = [cache[c_ptr] for c_ptr in c_ptrs_] else: default_cfg = acs[-1] cfgs = acs[:-1] d_cov = pi.cov(default_cfg, nc) d_mpiw = pi.mw(default_cfg, nc, precision=8) coverages = arr([pi.cov(cfg, nc) for cfg in cfgs]) candidates = np.where(coverages >= nc_)[0] cf_cov = cf_mpiw = p = None if candidates.size: mpiws = arr([pi.mw(cfgs[i], nc, precision=8) for i in candidates]) selected = mpiws.argmin() z, p = Stats.wilcoxon_signed_rank_test( pi.w(default_cfg, nc), pi.w(cfgs[candidates[selected]], nc)) cf_cov = coverages[candidates[selected]] cf_mpiw = mpiws[selected] return tag_cf_4_p(nc, d_cov, d_mpiw, cf_cov, cf_mpiw, p)
def coverage_acf_anc_4_p(acs, ncs): res = [] acs_ = _get_configs(acs) for c in ncs: res.append([pi.cov(cf, c, precision=2) for cf in acs_]) return np.transpose(res)
def meta_tags_4_anc_4_p(mt_ptrs, t_ptrs, m_dcf_ptr, dc_ptr, selectors, ncs, itag=False): cr = choose_m_4_anc(mt_ptrs, m_dcf_ptr, selectors, ncs) dcf = load_pickled_resource(dc_ptr) d_c, d_mw, d_w = _coverage_mw_widths(dcf, ncs) tags = [] for i, nc in enumerate(ncs): cf = choose_min(cr, selectors, t_ptrs, i, nc) cov = mw = p = None if cf: cov = pi.cov(cf, nc) mw = pi.mw(cf, nc) _, p = Stats.wilcoxon_signed_rank_test(pi.w(cf, nc), d_w[i]) if p != p: p = 1.0 idx, tag = tag_cf_4_p(nc, d_c[i], d_mw[i], cov, mw, p) if itag: tags.append(idx) else: tags.append(tag) return tags
def clf_accuracy_4_at_4_anc(at_ptrs, cf_ptrs, ncs): truth_4_nc = [] cfs = [load_pickled_resource(cfp) for cfp in cf_ptrs] for nc in ncs: truth_4_nc.append( arr([1 if pi.cov(cf, nc) >= nc / 100 else 0 for cf in cfs])) del cfs at_cfs = [_load_cfs_4_t_4_p(t_ptrs) for t_ptrs in at_ptrs] te_scores = [] f1_scores_bs = [] for i, nc in enumerate(ncs): truth = truth_4_nc[i] predictions = [] for cfs in at_cfs: predictions.append( arr([1 if pi.cov(cf, nc) >= nc / 100 else 0 for cf in cfs])) # get the base scores te_scores.append( [precision_recall_f1(truth, preds) for preds in predictions]) # now bootstrap f1_scores = [] for _ in range(100): s = np.random.choice(np.arange(truth.size), truth.size) # f1 for each technique f1 = [] for preds in predictions: f1.append(precision_recall_f1(truth[s], preds[s])[2]) f1_scores.append(f1) f1_scores_bs.append(f1_scores) del at_cfs return np.round(te_scores, 4), np.round(f1_scores_bs, 4)
def m_tags_4_anc(at_ptrs, dcf_ptr, selectors, ncs, dcf=None, itag=True): if dcf is None: dcf = load_pickled_resource(dcf_ptr) d_c, d_mw, d_w = _coverage_mw_widths(dcf, ncs) del dcf else: d_c, d_mw, d_w = _coverage_mw_widths(dcf, ncs) data = [] at_cfs = [_load_cfs_4_t_4_p(t_ptrs) for t_ptrs in at_ptrs] for i, nc in enumerate(ncs): t_data = [] for j, t_cfs in enumerate(at_cfs): s_cfs = [ selector.pick(at_ptrs[j], t_cfs, nc) for selector in selectors ] data_ = [] for s_cf, s_id in s_cfs: cov = mw = p = None if s_cf: cov = pi.cov(s_cf, nc) mw = pi.mw(s_cf, nc) _, p = Stats.wilcoxon_signed_rank_test( pi.w(s_cf, nc), d_w[i]) if p != p: p = 1.0 i_tag, tag = tag_cf_4_p(nc, d_c[i], d_mw[i], cov, mw, p) if itag: data_.append(i_tag) else: data_.append(tag) t_data.extend(data_) data.append(t_data) del at_cfs return arr(data)
def get_coverages(techniques, nc): return arr([pi.cov(te, nc, precision=2) for te in techniques])