def gen_labels(wdir, kc):
print('\ngenerating labels for %s\n' % wdir)
load_config('%s/input.py' % wdir)
istep = core.get_istep()
if istep not in kc.nc:
print("istep not in kmeanconf.py")
sys.exit()
nc = kc.nc[istep]
hook = kc.hooks[istep]
istep = core.get_istep()
labels = {}
labels['chi2dof'] = _gen_labels_by_chi2(wdir, istep)
labels['echi2dof'] = _gen_labels_by_echi2(wdir, istep)
labels['cluster'] = _gen_labels_by_cluster(wdir, istep, nc, hook)
clusters = sorted(list(set(labels['cluster'])))
chi_2_clusters = [0.0 for _ in clusters]
length_clusters = [0.0 for _ in clusters]
for i in range(len(labels['cluster'])):
for j in range(len(clusters)):
if labels['cluster'][i] == clusters[j]:
chi_2_clusters[j] += labels['chi2dof'][i]
length_clusters[j] += 1.0
labels['cluster_average_chi2'] = np.array(chi_2_clusters) / np.array(
length_clusters)
checkdir('%s/data' % wdir)
save(labels, '%s/data/labels-%d.dat' % (wdir, istep))
def plot_chi2_dist_per_exp(wdir, istep):
#--needs revisions NS (09/02/19)
labels = load('%s/data/labels-%d.dat' % (wdir, istep))
cluster, colors, nc, order = get_clusters(wdir, istep)
chi2 = labels['chi2dof']
data = load('%s/data/predictions-%d.dat' % (wdir, istep))
for reaction in data['reactions']:
for idx in data['reactions'][reaction]:
if reaction != 'sidis': continue
if idx != 2006: continue
res = np.array(data['reactions'][reaction][idx]['residuals-rep'])
chi2 = np.array([np.sum(r**2) / len(r) for r in res])
#chi2=np.array([_ for _ in chi2 if _<5])
Mean = np.average(chi2)
dchi = np.std(chi2)
chi2min = Mean - 2 * dchi
chi2max = Mean + 2 * dchi
R = (chi2min, chi2max)
nrows = 1
ncols = 1
#--plot labeled residuals
ax = py.subplot(nrows, ncols, 1)
ax.hist(chi2,
bins=30,
range=R,
histtype='step',
label='size=%d' % len(chi2))
for j in range(nc):
_chi2 = [chi2[i] for i in range(len(chi2)) if cluster[i] == j]
c = colors[j]
print c, np.mean(_chi2), reaction, idx
label = 'cluster id=%d size=%d' % (j, len(_chi2))
ax.hist(_chi2,
bins=30,
range=R,
histtype='step',
label=label,
color=c)
ax.set_xlabel('chi2/npts')
ax.set_ylabel('yiled')
ax.legend()
ax.text(0.1, 0.8, 'step %d' % istep, transform=ax.transAxes)
py.tight_layout()
checkdir('%s/gallery' % wdir)
py.savefig('%s/gallery/chi2-dist-%d-%s-%d.jpg' %
(wdir, istep, reaction, idx))
py.close()
def gen_melltab_hybrid(self):
path2dytab=conf['path2dytab-hybrid']
for k in conf['dy-pion tabs']:
path2dytabK='%s/%d'%(path2dytab,k)
checkdir(path2dytabK)
for i in range(len(conf['dy-pion tabs'][k]['idx'])):
print('%s: %i/%i'%(k,i,len(conf['dy-pion tabs'][k]['idx'])))
idx=conf['dy-pion tabs'][k]['idx'][i]
S = conf['dy-pion tabs'][k]['S'][i]
Y = conf['dy-pion tabs'][k]['Y'][i]
Q2 = conf['dy-pion tabs'][k]['Q2'][i]
muF2=Q2
data=self._gen_melltab_hybrid(Q2,S,Y,muF2)
fname='%s/%d.melltab'%(path2dytabK,idx)
np.save(fname,data)
def gen_melltab(self):
path2dytab=conf['path2dytab']
for k in conf['dy tabs']:
path2dytabK='%s/%d'%(path2dytab,k)
checkdir(path2dytabK)
npts=len(conf['dy tabs'][k]['value'])
for i in range(npts):
S = conf['dy tabs'][k]['S'][i]
Y = conf['dy tabs'][k]['Y'][i]
Q2 = conf['dy tabs'][k]['Q2'][i]
idx= conf['dy tabs'][k]['idx'][i]
muF2=Q2
data=self._gen_melltab(Q2,S,Y,muF2)
fname='%s/%d.melltab'%(path2dytabK,idx)
save(data,fname)
def gen_samples(wdir,nsamples=100,nsize=10):
load_config('%s/input.py'%wdir)
conf['bootstrap']=False
istep=core.get_istep()
core.mod_conf(istep) #--set conf as specified in istep
resman=RESMAN(nworkers=5,parallel=True,datasets=True)
parman=resman.parman
order=parman.order
obsres={}
if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres
if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres
if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres
if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres
if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres
if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares
print('\ngen ml samples using the setup as in %s\n'%wdir)
checkdir('%s/mlsamples'%wdir)
data=gen_dataframe(parman.order,obsres,nsize)
for _ in range(nsamples):
lprint('progress: %d/%d'%(_,nsamples))
par=parman.gen_flat(setup=True)
res,rres,nres=resman.get_residuals(par)
#--fill dataframe
i=data['size']
data['params'][i]=par
for reaction in obsres:
for idx in data['reactions'][reaction]:
prediction=copy.copy(obsres[reaction].tabs[idx]['prediction'])
data['reactions'][reaction][idx][i]=prediction
data['size']+=1
if data['size']==nsize:
save(data,'%s/mlsamples/%s'%(wdir,id_generator(12)))
data=gen_dataframe(parman.order,obsres,nsize)
print
#--close resman
resman.shutdown()
def plot_params(wdir,dist,kc):
load_config('%s/input.py'%wdir)
istep=core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep,replicas[0])
clusters,colors,nc,cluster_order = classifier.get_clusters(wdir,istep,kc)
_order = replicas[0]['order'][istep]
#--get correct order from dist
order = []
idx = []
for i in range(len(_order)):
if _order[i][1] != dist: continue
order.append(_order[i][2])
idx.append(i)
#--get correct params from dist
params = np.zeros((len(order),len(replicas)))
for i in range(len(order)):
for j in range(len(replicas)):
params[i][j] = replicas[j]['params'][istep][idx[i]]
#--create plot with enough space for # of parameters
nrows, ncols = np.ceil(len(order)/5.0), 5
fig = py.figure(figsize=(ncols*7,nrows*4))
X = np.linspace(1,len(replicas),len(replicas))
#--create plot
for i in range(len(order)):
ax = py.subplot(nrows,ncols, i+1)
ax.set_title('%s'%(order[i]), size=20)
#params = [replicas[j]['params'][istep][i] for j in range(len(replicas))]
color = [colors[clusters[j]] for j in range(len(replicas))]
ax.scatter(X,params[i],color=color)
ax.plot(X,np.ones(len(X))*np.average(params[i]),'k--',alpha=0.5)
filename='%s/gallery/%s-params.png'%(wdir,dist)
checkdir('%s/gallery'%wdir)
py.savefig(filename)
print 'Saving figure to %s'%filename
def plot_chi2_dist(wdir, istep):
#--needs revisions NS (09/02/19)
labels = load('%s/data/labels-%d.dat' % (wdir, istep))
chi2 = labels['chi2dof']
cluster, colors, nc, order = get_clusters(wdir, istep)
nrows = 1
ncols = 1
#--plot labeled residuals
ax = py.subplot(nrows, ncols, 1)
chi2 = [_ for _ in chi2 if _ < 2]
Mean = np.average(chi2)
dchi = np.std(chi2)
chi2min = Mean - 2 * dchi
chi2max = Mean + 2 * dchi
R = (chi2min, chi2max)
ax.hist(chi2,
bins=60,
range=R,
histtype='step',
label='size=%d' % len(chi2))
for j in range(nc):
chi2_ = [chi2[i] for i in range(len(chi2)) if cluster[i] == j]
c = colors[j]
print c, np.mean(chi2_), np.mean(chi2)
label = 'cluster id=%d size=%d' % (j, len(chi2_))
ax.hist(chi2_, bins=30, range=R, histtype='step', label=label, color=c)
mean = np.average(chi2_)
ax.axvline(mean, color=c, ls=':')
ax.set_xlabel('chi2/npts')
ax.set_ylabel('yiled')
ax.legend()
ax.text(0.1, 0.8, 'step %d' % istep, transform=ax.transAxes)
py.tight_layout()
checkdir('%s/gallery' % wdir)
py.savefig('%s/gallery/chi2-dist-%d.jpg' % (wdir, istep))
py.close()
