def sub_just_confusion(saveinfo, runinfo, verbose, ordering):
model = runinfo["model"]
cv_dir = runinfo["cv_dir"]
scorer = runinfo["scorer"]
resultdir = saveinfo["resultdir"]
fheader = os.path.join(cv_dir, model+"_"+scorer)
resultfiles = [os.path.split(ss)[-1] for ss in glob(fheader+"*.npz")]
Ncv = len(resultfiles)
#:wqprint resultfiles
SAVE_CONFUS = []
for icv, fh in enumerate(resultfiles):
readfile = os.path.join(cv_dir, fh)
data = np.load(readfile)
confus = data['econfusion']
# normalize over rows
msum = confus.sum(axis=1)[:, None]
msum[msum==0.0] = 1.0
confus /= msum
SAVE_CONFUS.append(confus)
sum_confus = np.array(SAVE_CONFUS).mean(axis=0)
sum_confus = sum_confus[np.ix_(ordering, ordering)] # re-order
confile = os.path.join(resultdir, model+"_"+scorer+"_confusion")
plot_confusion(sum_confus, confile)
def get_single(saveinfo, runinfo, verbose):
if verbose:
print "get CV results...\nmodel=%s, scoring=%s"%(runinfo["model"], runinfo["scorer"])
T = time.time()
# return data for this model
if verbose:
print "running cv"
mod_scores, mod_losses, mean_W, sum_confus = read_cv(runinfo, saveinfo, verbose)
#mod_results = mod_score, mod_loss, mean_W
# return data for shuffle model
if verbose:
print "running shuffle"
model = runinfo['model']
scorer = runinfo['scorer']
if model.endswith("majority"):
copy_run = copy.copy(runinfo)
copy_run["model"] = "*majority*"
else:
copy_run = runinfo
shuf_scores, shuf_losses, shuf_W = read_shuffle(copy_run, saveinfo, verbose)
get_w = False #if "majority" in model else True
if get_w:
dims = float(np.prod(mean_W.shape))
sparse = (np.abs(mean_W)>1E-3).sum()/dims
sparse_s = np.logical_or( mean_W < shuf_W[:,:, 0], mean_W > shuf_W[:,:, 1]).sum()/dims
sparse_ss = np.logical_or( mean_W < shuf_W[:,:, 2], mean_W > shuf_W[:,:, 3]).sum()/dims
sparse_max = np.logical_or( mean_W < shuf_W[:,:, 4], mean_W > shuf_W[:,:, 5]).sum()/dims
sparsity = 1.0 - np.array([sparse, sparse_s, sparse_ss, sparse_max ])
else:
sparsity = np.ones(4)
# save relevant results to disk
resultdir = saveinfo["resultdir"]
npfile = os.path.join(resultdir, model+"_"+scorer)
np.savez(npfile, scores=scores, losses= losses, mod_scores=mod_scores, mod_losses=mod_losses,\
mean_W=mean_W, shuf_scores=shuf_scores, shuf_losses=shuf_losses, shuf_W=shuf_W, \
sum_confus=sum_confus, sparsity=sparsity)
# create and save confusion matrix image
confile = os.path.join(resultdir, model+"_"+scorer+"_confusion")
plot_confusion(sum_confus, confile)
if verbose: print "CV results time = %s"%(time.time()-T,)
return mod_scores, mod_losses, shuf_scores, shuf_losses, sparsity
