def run_test(seed, data, pclearning): random.seed(seed) # Separate learning data and test data pt_learning, pt_test = data.pt.split(2, [pclearning, 100 - pclearning]) aa_learning = data.aa.get_subset(pt_learning.keys()) aa_test = data.aa.get_subset(pt_test.keys()) # Initialize ELECTRE-TRI BM model cat_profiles = generate_categories_profiles(data.cats) worst = data.pt.get_worst(data.c) best = data.pt.get_best(data.c) b = generate_alternatives(len(data.cats) - 1, 'b') bpt = None cvs = None lbda = None model = MRSort(data.c, cvs, bpt, lbda, cat_profiles) # Run the linear program t1 = time.time() mip = mip_mrsort(model, pt_learning, aa_learning) obj = mip.solve() t_total = time.time() - t1 # CA learning set aa_learning2 = model.pessimist(pt_learning) ca_learning = compute_ca(aa_learning, aa_learning2) auc_learning = model.auc(aa_learning, pt_learning) diff_learning = compute_confusion_matrix(aa_learning, aa_learning2, model.categories) # Compute CA of test setting if len(aa_test) > 0: aa_test2 = model.pessimist(pt_test) ca_test = compute_ca(aa_test, aa_test2) auc_test = model.auc(aa_test, pt_test) diff_test = compute_confusion_matrix(aa_test, aa_test2, model.categories) else: ca_test = 0 auc_test = 0 ncat = len(data.cats) diff_test = OrderedDict([((a, b), 0) for a in model.categories \ for b in model.categories]) # Compute CA of whole set aa2 = model.pessimist(data.pt) ca = compute_ca(data.aa, aa2) auc = model.auc(data.aa, data.pt) diff_all = compute_confusion_matrix(data.aa, aa2, model.categories) t = test_result("%s-%d-%d" % (data.name, seed, pclearning)) model.id = 'learned' aa_learning.id, aa_test.id = 'learning_set', 'test_set' pt_learning.id, pt_test.id = 'learning_set', 'test_set' save_to_xmcda("%s/%s.bz2" % (directory, t.test_name), model, aa_learning, aa_test, pt_learning, pt_test) t['seed'] = seed t['na'] = len(data.a) t['nc'] = len(data.c) t['ncat'] = len(data.cats) t['pclearning'] = pclearning t['na_learning'] = len(aa_learning) t['na_test'] = len(aa_test) t['obj'] = obj t['ca_learning'] = ca_learning t['ca_test'] = ca_test t['ca_all'] = ca t['auc_learning'] = auc_learning t['auc_test'] = auc_test t['auc_all'] = auc for k, v in diff_learning.items(): t['learn_%s_%s' % (k[0], k[1])] = v for k, v in diff_test.items(): t['test_%s_%s' % (k[0], k[1])] = v for k, v in diff_all.items(): t['all_%s_%s' % (k[0], k[1])] = v t['t_total'] = t_total return t
buf = "" for c in win: buf += "%s, " % criteria_names[c] print('[%s]' % buf[:-2], file=fcoalitions) fcoalitions.close() aa_learned = m.pessimist(pt_learning) fca = open('%s-ca.dat' % bname, 'w+') ca = compute_ca(aa_learning, aa_learned) print("%.4f" % ca, end='', file=fca) fca.close() fauc = open('%s-auc.dat' % bname, 'w+') auc = m.auc(aa_learning, pt_learning) print("%.4f" % auc, end='', file=fauc) fauc.close() fmisclassified = open('%s-misclassified.dat' % bname, 'w+') print("{Alternative} ", file=fmisclassified, end='') print("{Original assignment} ", file=fmisclassified, end='') print("{Model assignment}", file=fmisclassified, end='') for c in criteria: print(" {%s}" % criteria_names[c], file=fmisclassified, end='') print("\n", file=fmisclassified, end='') misclassified_aids = [] for aid in aa_learning.keys(): aa1 = aa_learning[aid].category_id aa2 = aa_learned[aid].category_id
buf += "%s, " % criteria_names[c] print('[%s]' % buf[:-2], file=fcoalitions) fcoalitions.close() pt_learning = PerformanceTable().from_xmcda(root, 'learning_set') aa_learning = AlternativesAssignments().from_xmcda(root, 'learning_set') aa_learned = m.pessimist(pt_learning) fca = open('%s-ca_learning.dat' % bname, 'w+') ca = compute_ca(aa_learning, aa_learned) print("%.4f" % ca, end='', file=fca) fca.close() fauc = open('%s-auc_learning.dat' % bname, 'w+') auc = m.auc(aa_learning, pt_learning) print("%.4f" % auc, end='', file=fauc) fauc.close() ca_learning.append(ca) auc_learning.append(auc) pt_test = PerformanceTable().from_xmcda(root, 'test_set') aa_test = AlternativesAssignments().from_xmcda(root, 'test_set') aa_test2 = m.pessimist(pt_test) fca = open('%s-ca_test.dat' % bname, 'w+') ca = compute_ca(aa_test, aa_test2) print("%.4f" % ca, end='', file=fca) fca.close()
aa_learning = AlternativesAssignments().from_xmcda(root, 'learning_set') aa_test = AlternativesAssignments().from_xmcda(root, 'test_set') aa_learning_m2 = m.pessimist(pt_learning) aa_test_m2 = m.pessimist(pt_test) # Compute classification accuracy ca_learning = compute_ca(aa_learning, aa_learning_m2) ca_test = compute_ca(aa_test, aa_test_m2) table_ca_learning.append(ca_learning) table_ca_test.append(ca_test) # Compute area under the curve auc_learning = m.auc(aa_learning, pt_learning) auc_test = m.auc(aa_test, pt_test) table_auc_learning.append(auc_learning) table_auc_test.append(auc_test) if m.veto_lbda is not None: nveto += 1 # Compute confusion matrices for a in aa_learning.keys(): key = (aa_learning[a].category_id, aa_learning_m2[a].category_id) if key in cmatrix_learning: cmatrix_learning[key] += 1 else: cmatrix_learning[key] = 1
def run_test(seed, data, pclearning, nloop, nmodels, nmeta): random.seed(seed) global aaa global allm global fct_ca global LOO # Separate learning data and test data if LOO: pt_learning, pt_test = data.pt.split_LOO(seed) else: pt_learning, pt_test = data.pt.split(2, [pclearning, 100 - pclearning]) aa_learning = data.aa.get_subset(pt_learning.keys()) aa_test = data.aa.get_subset(pt_test.keys()) #import pdb; pdb.set_trace() # Initialize a random model cat_profiles = generate_categories_profiles(data.cats) worst = data.pt.get_worst(data.c) best = data.pt.get_best(data.c) b = generate_alternatives(len(data.cats) - 1, 'b') bpt = None cvs = None lbda = None model = MRSort(data.c, cvs, bpt, lbda, cat_profiles) # if LOO: # print(data.c, cvs, bpt, lbda, cat_profiles) # print(model.categories_profiles.to_categories()) # print(model.categories) # import pdb; pdb.set_trace() # Run the metaheuristic t1 = time.time() pt_sorted = SortedPerformanceTable(pt_learning) # Algorithm meta = meta_mrsort(nmodels, model.criteria, model.categories_profiles.to_categories(), pt_sorted, aa_learning, seed = seed * 100) # if LOO: # print(nmodels, model.criteria, # model.categories_profiles.to_categories(), # pt_sorted, aa_learning) #import pdb; pdb.set_trace() #lp_weights = lp_weights, #heur_profiles = heur_profiles, #lp_veto_weights = lp_veto_weights, #heur_veto_profiles = heur_veto_profiles, for i in range(0, nloop): model, ca_learning, all_models = meta.optimize(nmeta, fct_ca) #import pdb; pdb.set_trace() if ca_learning == 1: break t_total = time.time() - t1 aa_learning2 = model.pessimist(pt_learning) ca_learning = compute_ca(aa_learning, aa_learning2) ca_learning_good = compute_ca_good(aa_learning, aa_learning2) #import pdb; pdb.set_trace() auc_learning = model.auc(aa_learning, pt_learning) diff_learning = compute_confusion_matrix(aa_learning, aa_learning2, model.categories) # Compute CA of test setting if len(aa_test) > 0: aa_test2 = model.pessimist(pt_test) ca_test = compute_ca(aa_test, aa_test2) ca_test_good = compute_ca_good(aa_test, aa_test2) auc_test = model.auc(aa_test, pt_test) diff_test = compute_confusion_matrix(aa_test, aa_test2, model.categories) #import pdb; pdb.set_trace() else: ca_test = 0 auc_test = 0 ncat = len(data.cats) diff_test = OrderedDict([((a, b), 0) for a in model.categories \ for b in model.categories]) # Compute CA of whole set aa2 = model.pessimist(data.pt) ca = compute_ca(data.aa, aa2) ca_good = compute_ca_good(data.aa, aa2) auc = model.auc(data.aa, data.pt) diff_all = compute_confusion_matrix(data.aa, aa2, model.categories) t = test_result("%s-%d-%d-%d-%d-%d" % (data.name, seed, nloop, nmodels, nmeta, pclearning)) model.id = 'learned' aa_learning.id, aa_test.id = 'learning_set', 'test_set' pt_learning.id, pt_test.id = 'learning_set', 'test_set' save_to_xmcda("%s/%s.bz2" % (directory, t.test_name), model, aa_learning, aa_test, pt_learning, pt_test) t['seed'] = seed t['na'] = len(data.a) t['nc'] = len(data.c) t['ncat'] = len(data.cats) t['pclearning'] = pclearning t['nloop'] = nloop t['nmodels'] = nmodels t['nmeta'] = nmeta t['na_learning'] = len(aa_learning) t['na_test'] = len(aa_test) t['ca_learning'] = ca_learning t['ca_test'] = ca_test t['ca_all'] = ca t['ca_learning_good'] = ca_learning_good t['ca_test_good'] = ca_test_good t['ca_all_good'] = ca_good t['auc_learning'] = auc_learning t['auc_test'] = auc_test t['auc_all'] = auc # import pdb; pdb.set_trace() aaa[seed]=dict() aaa[seed]['id'] = seed aaa[seed]['learning_asgmt_id'] = [i.id for i in aa_learning] aaa[seed]['learning_asgmt'] = [i.category_id for i in aa_learning] aaa[seed]['learning_asgmt2'] = [i.category_id for i in aa_learning2] aaa[seed]['test_asgmt_id'] = [i.id for i in aa_test] aaa[seed]['test_asgmt'] = [i.category_id for i in aa_test] aaa[seed]['test_asgmt2'] = [i.category_id for i in aa_test2] aaa[seed]['criteria'] = [i for i,j in model.criteria.items()] aaa[seed]['criteria_weights'] = [str(i.value) for i in model.cv.values()] aaa[seed]['profiles_values'] = [str(model.bpt['b1'].performances[i]) for i,j in model.criteria.items()] aaa[seed]['lambda'] = model.lbda #[model.bpt['b1'].performances[i] for i,j in model.criteria.items()] allm[seed]=dict() allm[seed]['id'] = seed current_model = 0 allm[seed]['mresults'] = dict() for all_model in list(all_models)[1:]: current_model += 1 # skipping the 1rst model already treated allm[seed]['mresults'][current_model] = ["",""] aa_learning2_allm = all_model.model.pessimist(pt_learning) ca_learning_allm = compute_ca(aa_learning, aa_learning2_allm) ca_learning_good_allm = compute_ca_good(aa_learning, aa_learning2_allm) auc_learning_allm = all_model.model.auc(aa_learning, pt_learning) # diff_learning_allm = compute_confusion_matrix(aa_learning, aa_learning2_allm, # all_model.model.categories) # Compute CA of test setting if len(aa_test) > 0: aa_test2_allm = all_model.model.pessimist(pt_test) ca_test_allm = compute_ca(aa_test, aa_test2_allm) ca_test_good_allm = compute_ca_good(aa_test, aa_test2_allm) auc_test_allm = all_model.model.auc(aa_test, pt_test) # diff_test_allm = compute_confusion_matrix(aa_test, aa_test2_allm, # all_model.categories) else: ca_test_allm = 0 auc_test_allm = 0 ncat_allm = len(data.cats) # diff_test_allm = OrderedDict([((a, b), 0) for a in all_model.categories \ # for b in all_model.model.categories]) # Compute CA of whole set aa2_allm = all_model.model.pessimist(data.pt) ca_allm = compute_ca(data.aa, aa2_allm) ca_good_allm = compute_ca_good(data.aa, aa2_allm) auc_allm = all_model.model.auc(data.aa, data.pt) #diff_all_allm = compute_confusion_matrix(data.aa, aa2_allm, all_model.model.categories) allm[seed]['mresults'][current_model][0] = 'na_learning,na_test,ca_learning,ca_test,ca_all,ca_learning_good,ca_test_good,ca_all_good,auc_learning,auc_test,auc_all' allm[seed]['mresults'][current_model][1] = str(len(aa_learning)) + "," + str(len(aa_test)) + "," + str(ca_learning_allm) + "," + str(ca_test_allm) + "," + str(ca_allm) + "," + str(ca_learning_good_allm) + "," + str(ca_test_good_allm) + "," + str(ca_good_allm) + "," + str(auc_learning_allm) + "," + str(auc_test_allm) + "," + str(auc_allm) #allm[seed]['mresults'][current_model][1] = #all_model.model.bpt['b1'].performances #all_model.model.cv.values() #import pdb; pdb.set_trace() # allm[seed][current_model]['na_learning'] = len(aa_learning) # allm[seed][current_model]['na_test'] = len(na_test) # allm[seed][current_model]['ca_learning'] = ca_learning_allm # allm[seed][current_model]['ca_test'] = ca_test_allm # allm[seed][current_model]['ca_all'] = ca_allm # allm[seed][current_model]['ca_learning_good'] = ca_learning_good_allm # allm[seed][current_model]['ca_test_good'] = ca_test_good_allm # allm[seed][current_model]['ca_all_good'] = ca_good_allm # allm[seed][current_model]['auc_learning'] = auc_learning_allm # allm[seed][current_model]['auc_test'] = auc_test_allm # allm[seed][current_model]['auc_all'] = auc_allm for k, v in diff_learning.items(): t['learn_%s_%s' % (k[0], k[1])] = v for k, v in diff_test.items(): t['test_%s_%s' % (k[0], k[1])] = v for k, v in diff_all.items(): t['all_%s_%s' % (k[0], k[1])] = v t['t_total'] = t_total return t
print('[%s]' % buf[:-2], file=fcoalitions) fcoalitions.close() pt_learning = PerformanceTable().from_xmcda(root, 'learning_set') aa_learning = AlternativesAssignments().from_xmcda(root, 'learning_set') aa_learned = m.pessimist(pt_learning) fca = open('%s-ca_learning.dat' % bname, 'w+') ca = compute_ca(aa_learning, aa_learned) print("%.4f" % ca, end = '', file=fca) fca.close() fauc = open('%s-auc_learning.dat' % bname, 'w+') auc = m.auc(aa_learning, pt_learning) print("%.4f" % auc, end = '', file=fauc) fauc.close() ca_learning.append(ca) auc_learning.append(auc) pt_test = PerformanceTable().from_xmcda(root, 'test_set') aa_test = AlternativesAssignments().from_xmcda(root, 'test_set') aa_test2 = m.pessimist(pt_test) fca = open('%s-ca_test.dat' % bname, 'w+') ca = compute_ca(aa_test, aa_test2) print("%.4f" % ca, end = '', file=fca) fca.close()
def run_test(seed, data, pclearning, nloop, nmodels, nmeta): random.seed(seed) # Separate learning data and test data pt_learning, pt_test = data.pt.split(2, [pclearning, 100 - pclearning]) aa_learning = data.aa.get_subset(pt_learning.keys()) aa_test = data.aa.get_subset(pt_test.keys()) # Initialize a random model cat_profiles = generate_categories_profiles(data.cats) worst = data.pt.get_worst(data.c) best = data.pt.get_best(data.c) b = generate_alternatives(len(data.cats) - 1, 'b') bpt = None cvs = None lbda = None model = MRSort(data.c, cvs, bpt, lbda, cat_profiles) # Run the metaheuristic t1 = time.time() pt_sorted = SortedPerformanceTable(pt_learning) # Algorithm meta = meta_mrsort(nmodels, model.criteria, model.categories_profiles.to_categories(), pt_sorted, aa_learning, seed=seed * 100) #lp_weights = lp_weights, #heur_profiles = heur_profiles, #lp_veto_weights = lp_veto_weights, #heur_veto_profiles = heur_veto_profiles, for i in range(0, nloop): model, ca_learning = meta.optimize(nmeta) if ca_learning == 1: break t_total = time.time() - t1 aa_learning2 = model.pessimist(pt_learning) ca_learning = compute_ca(aa_learning, aa_learning2) auc_learning = model.auc(aa_learning, pt_learning) diff_learning = compute_confusion_matrix(aa_learning, aa_learning2, model.categories) # Compute CA of test setting if len(aa_test) > 0: aa_test2 = model.pessimist(pt_test) ca_test = compute_ca(aa_test, aa_test2) auc_test = model.auc(aa_test, pt_test) diff_test = compute_confusion_matrix(aa_test, aa_test2, model.categories) else: ca_test = 0 auc_test = 0 ncat = len(data.cats) diff_test = OrderedDict([((a, b), 0) for a in model.categories \ for b in model.categories]) # Compute CA of whole set aa2 = model.pessimist(data.pt) ca = compute_ca(data.aa, aa2) auc = model.auc(data.aa, data.pt) diff_all = compute_confusion_matrix(data.aa, aa2, model.categories) t = test_result("%s-%d-%d-%d-%d-%d" % (data.name, seed, nloop, nmodels, nmeta, pclearning)) model.id = 'learned' aa_learning.id, aa_test.id = 'learning_set', 'test_set' pt_learning.id, pt_test.id = 'learning_set', 'test_set' save_to_xmcda("%s/%s.bz2" % (directory, t.test_name), model, aa_learning, aa_test, pt_learning, pt_test) t['seed'] = seed t['na'] = len(data.a) t['nc'] = len(data.c) t['ncat'] = len(data.cats) t['pclearning'] = pclearning t['nloop'] = nloop t['nmodels'] = nmodels t['nmeta'] = nmeta t['na_learning'] = len(aa_learning) t['na_test'] = len(aa_test) t['ca_learning'] = ca_learning t['ca_test'] = ca_test t['ca_all'] = ca t['auc_learning'] = auc_learning t['auc_test'] = auc_test t['auc_all'] = auc for k, v in diff_learning.items(): t['learn_%s_%s' % (k[0], k[1])] = v for k, v in diff_test.items(): t['test_%s_%s' % (k[0], k[1])] = v for k, v in diff_all.items(): t['all_%s_%s' % (k[0], k[1])] = v t['t_total'] = t_total return t
'learning_set') aa_test = AlternativesAssignments().from_xmcda(root, 'test_set') aa_learning_m2 = m.pessimist(pt_learning) aa_test_m2 = m.pessimist(pt_test) # Compute classification accuracy ca_learning = compute_ca(aa_learning, aa_learning_m2) ca_test = compute_ca(aa_test, aa_test_m2) table_ca_learning.append(ca_learning) table_ca_test.append(ca_test) # Compute area under the curve auc_learning = m.auc(aa_learning, pt_learning) auc_test = m.auc(aa_test, pt_test) table_auc_learning.append(auc_learning) table_auc_test.append(auc_test) if m.veto_lbda is not None: nveto += 1 # Compute confusion matrices for a in aa_learning.keys(): key = (aa_learning[a].category_id, aa_learning_m2[a].category_id) if key in cmatrix_learning: cmatrix_learning[key] += 1 else: cmatrix_learning[key] = 1
print_pt_and_assignments(aids, None, [aa_learning_m1, aa_learning_m2], pt_learning, au) # for i in range(1, len(pt_learning) + 1): # aid = "a%d" % i # uti = m2.global_utility(pt_learning["a%d" % i]) # if aa_learning_m2[aid].category_id != aa_learning_m1[aid].category_id: # print("%s %g %s %s" % (aid, uti.value, aa_learning_m2[aid].category_id, aa_learning_m1[aid].category_id)) # print_pt_and_assignments(anok, c, [aa_learning_m1, aa_learning_m2], pt_learning) if pt_test is not None: aa_test_m2 = m2.get_assignments(pt_test) def compute_auc_histo(aa): pass if aa_learning_m1 is not None: ca_learning = compute_ca(aa_learning_m1, aa_learning_m2) auc_learning = m2.auc(aa_learning_m1, pt_learning) print("Learning set") print("============") print("CA : %g" % ca_learning) print("AUC: %g" % auc_learning) print("Confusion table:") matrix = compute_confusion_matrix(aa_learning_m1, aa_learning_m2, m2.categories) print_confusion_matrix(matrix, m2.categories) aids = [a.id for a in aa_learning_m1 \ if aa_learning_m1[a.id].category_id != aa_learning_m2[a.id].category_id] if len(aids) > 0: print("List of alternatives wrongly assigned:") print_pt_and_assignments(aids, None, [aa_learning_m1, aa_learning_m2], pt_learning)
sys.exit(1) t_total = time.time() - t1 model.id = 'learned' data.pt.id = 'learning_set' data.aa.id = 'learning_set' dt = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") save_to_xmcda("%s/%s-all-%s-%s.bz2" % (DATADIR, algo, data.name, dt), data.aa, data.pt, model) aa2 = model.get_assignments(data.pt) ca = compute_ca(data.aa, aa2) auc = model.auc(data.aa, data.pt) anok = [] for a in data.a: if data.aa[a.id].category_id != aa2[a.id].category_id: anok.append(a) if len(anok) > 0: print("Alternatives wrongly assigned:") print_pt_and_assignments(anok.keys(), data.c.keys(), [data.aa, aa2], data.pt) print("Model parameters:") cids = model.criteria.keys() if model_type == 'mrsort': print(model.bpt)
sys.exit(1) t_total = time.time() - t1 model.id = 'learned' data.pt.id = 'learning_set' data.aa.id = 'learning_set' dt = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") save_to_xmcda("%s/%s-all-%s-%s.bz2" % (DATADIR, algo, data.name, dt), data.aa, data.pt, model) aa2 = model.get_assignments(data.pt) ca = compute_ca(data.aa, aa2) auc = model.auc(data.aa, data.pt) anok = [] for a in data.a: if data.aa[a.id].category_id != aa2[a.id].category_id: anok.append(a) if len(anok) > 0: print("Alternatives wrongly assigned:") print_pt_and_assignments(anok.keys(), data.c.keys(), [data.aa, aa2], data.pt) print("Model parameters:") cids = model.criteria.keys() if model_type == 'mrsort': print(model.bpt) print(model.cv)
def run_test(seed, data, pclearning, nloop, nmodels, nmeta): random.seed(seed) # Separate learning data and test data pt_learning, pt_test = data.pt.split(2, [pclearning, 100 - pclearning]) aa_learning = data.aa.get_subset(pt_learning.keys()) aa_test = data.aa.get_subset(pt_test.keys()) # Initialize a random model cat_profiles = generate_categories_profiles(data.cats) worst = data.pt.get_worst(data.c) best = data.pt.get_best(data.c) b = generate_alternatives(len(data.cats) - 1, 'b') bpt = None cvs = None lbda = None model = MRSort(data.c, cvs, bpt, lbda, cat_profiles) # Run the metaheuristic t1 = time.time() pt_sorted = SortedPerformanceTable(pt_learning) # Algorithm meta = meta_mrsort(nmodels, model.criteria, model.categories_profiles.to_categories(), pt_sorted, aa_learning, seed = seed * 100) #lp_weights = lp_weights, #heur_profiles = heur_profiles, #lp_veto_weights = lp_veto_weights, #heur_veto_profiles = heur_veto_profiles, for i in range(0, nloop): model, ca_learning = meta.optimize(nmeta) if ca_learning == 1: break t_total = time.time() - t1 aa_learning2 = model.pessimist(pt_learning) ca_learning = compute_ca(aa_learning, aa_learning2) auc_learning = model.auc(aa_learning, pt_learning) diff_learning = compute_confusion_matrix(aa_learning, aa_learning2, model.categories) # Compute CA of test setting if len(aa_test) > 0: aa_test2 = model.pessimist(pt_test) ca_test = compute_ca(aa_test, aa_test2) auc_test = model.auc(aa_test, pt_test) diff_test = compute_confusion_matrix(aa_test, aa_test2, model.categories) else: ca_test = 0 auc_test = 0 ncat = len(data.cats) diff_test = OrderedDict([((a, b), 0) for a in model.categories \ for b in model.categories]) # Compute CA of whole set aa2 = model.pessimist(data.pt) ca = compute_ca(data.aa, aa2) auc = model.auc(data.aa, data.pt) diff_all = compute_confusion_matrix(data.aa, aa2, model.categories) t = test_result("%s-%d-%d-%d-%d-%d" % (data.name, seed, nloop, nmodels, nmeta, pclearning)) model.id = 'learned' aa_learning.id, aa_test.id = 'learning_set', 'test_set' pt_learning.id, pt_test.id = 'learning_set', 'test_set' save_to_xmcda("%s/%s.bz2" % (directory, t.test_name), model, aa_learning, aa_test, pt_learning, pt_test) t['seed'] = seed t['na'] = len(data.a) t['nc'] = len(data.c) t['ncat'] = len(data.cats) t['pclearning'] = pclearning t['nloop'] = nloop t['nmodels'] = nmodels t['nmeta'] = nmeta t['na_learning'] = len(aa_learning) t['na_test'] = len(aa_test) t['ca_learning'] = ca_learning t['ca_test'] = ca_test t['ca_all'] = ca t['auc_learning'] = auc_learning t['auc_test'] = auc_test t['auc_all'] = auc for k, v in diff_learning.items(): t['learn_%s_%s' % (k[0], k[1])] = v for k, v in diff_test.items(): t['test_%s_%s' % (k[0], k[1])] = v for k, v in diff_all.items(): t['all_%s_%s' % (k[0], k[1])] = v t['t_total'] = t_total return t