def sub_just_confusion(saveinfo, runinfo, verbose, ordering): model = runinfo["model"] cv_dir = runinfo["cv_dir"] scorer = runinfo["scorer"] resultdir = saveinfo["resultdir"] fheader = os.path.join(cv_dir, model+"_"+scorer) resultfiles = [os.path.split(ss)[-1] for ss in glob(fheader+"*.npz")] Ncv = len(resultfiles) #:wqprint resultfiles SAVE_CONFUS = [] for icv, fh in enumerate(resultfiles): readfile = os.path.join(cv_dir, fh) data = np.load(readfile) confus = data['econfusion'] # normalize over rows msum = confus.sum(axis=1)[:, None] msum[msum==0.0] = 1.0 confus /= msum SAVE_CONFUS.append(confus) sum_confus = np.array(SAVE_CONFUS).mean(axis=0) sum_confus = sum_confus[np.ix_(ordering, ordering)] # re-order confile = os.path.join(resultdir, model+"_"+scorer+"_confusion") plot_confusion(sum_confus, confile)
def get_single(saveinfo, runinfo, verbose): if verbose: print "get CV results...\nmodel=%s, scoring=%s"%(runinfo["model"], runinfo["scorer"]) T = time.time() # return data for this model if verbose: print "running cv" mod_scores, mod_losses, mean_W, sum_confus = read_cv(runinfo, saveinfo, verbose) #mod_results = mod_score, mod_loss, mean_W # return data for shuffle model if verbose: print "running shuffle" model = runinfo['model'] scorer = runinfo['scorer'] if model.endswith("majority"): copy_run = copy.copy(runinfo) copy_run["model"] = "*majority*" else: copy_run = runinfo shuf_scores, shuf_losses, shuf_W = read_shuffle(copy_run, saveinfo, verbose) get_w = False #if "majority" in model else True if get_w: dims = float(np.prod(mean_W.shape)) sparse = (np.abs(mean_W)>1E-3).sum()/dims sparse_s = np.logical_or( mean_W < shuf_W[:,:, 0], mean_W > shuf_W[:,:, 1]).sum()/dims sparse_ss = np.logical_or( mean_W < shuf_W[:,:, 2], mean_W > shuf_W[:,:, 3]).sum()/dims sparse_max = np.logical_or( mean_W < shuf_W[:,:, 4], mean_W > shuf_W[:,:, 5]).sum()/dims sparsity = 1.0 - np.array([sparse, sparse_s, sparse_ss, sparse_max ]) else: sparsity = np.ones(4) # save relevant results to disk resultdir = saveinfo["resultdir"] npfile = os.path.join(resultdir, model+"_"+scorer) np.savez(npfile, scores=scores, losses= losses, mod_scores=mod_scores, mod_losses=mod_losses,\ mean_W=mean_W, shuf_scores=shuf_scores, shuf_losses=shuf_losses, shuf_W=shuf_W, \ sum_confus=sum_confus, sparsity=sparsity) # create and save confusion matrix image confile = os.path.join(resultdir, model+"_"+scorer+"_confusion") plot_confusion(sum_confus, confile) if verbose: print "CV results time = %s"%(time.time()-T,) return mod_scores, mod_losses, shuf_scores, shuf_losses, sparsity