def gen_labels(wdir, kc): print('\ngenerating labels for %s\n' % wdir) load_config('%s/input.py' % wdir) istep = core.get_istep() if istep not in kc.nc: print("istep not in kmeanconf.py") sys.exit() nc = kc.nc[istep] hook = kc.hooks[istep] istep = core.get_istep() labels = {} labels['chi2dof'] = _gen_labels_by_chi2(wdir, istep) labels['echi2dof'] = _gen_labels_by_echi2(wdir, istep) labels['cluster'] = _gen_labels_by_cluster(wdir, istep, nc, hook) clusters = sorted(list(set(labels['cluster']))) chi_2_clusters = [0.0 for _ in clusters] length_clusters = [0.0 for _ in clusters] for i in range(len(labels['cluster'])): for j in range(len(clusters)): if labels['cluster'][i] == clusters[j]: chi_2_clusters[j] += labels['chi2dof'][i] length_clusters[j] += 1.0 labels['cluster_average_chi2'] = np.array(chi_2_clusters) / np.array( length_clusters) checkdir('%s/data' % wdir) save(labels, '%s/data/labels-%d.dat' % (wdir, istep))
def plot_chi2_dist_per_exp(wdir, istep): #--needs revisions NS (09/02/19) labels = load('%s/data/labels-%d.dat' % (wdir, istep)) cluster, colors, nc, order = get_clusters(wdir, istep) chi2 = labels['chi2dof'] data = load('%s/data/predictions-%d.dat' % (wdir, istep)) for reaction in data['reactions']: for idx in data['reactions'][reaction]: if reaction != 'sidis': continue if idx != 2006: continue res = np.array(data['reactions'][reaction][idx]['residuals-rep']) chi2 = np.array([np.sum(r**2) / len(r) for r in res]) #chi2=np.array([_ for _ in chi2 if _<5]) Mean = np.average(chi2) dchi = np.std(chi2) chi2min = Mean - 2 * dchi chi2max = Mean + 2 * dchi R = (chi2min, chi2max) nrows = 1 ncols = 1 #--plot labeled residuals ax = py.subplot(nrows, ncols, 1) ax.hist(chi2, bins=30, range=R, histtype='step', label='size=%d' % len(chi2)) for j in range(nc): _chi2 = [chi2[i] for i in range(len(chi2)) if cluster[i] == j] c = colors[j] print c, np.mean(_chi2), reaction, idx label = 'cluster id=%d size=%d' % (j, len(_chi2)) ax.hist(_chi2, bins=30, range=R, histtype='step', label=label, color=c) ax.set_xlabel('chi2/npts') ax.set_ylabel('yiled') ax.legend() ax.text(0.1, 0.8, 'step %d' % istep, transform=ax.transAxes) py.tight_layout() checkdir('%s/gallery' % wdir) py.savefig('%s/gallery/chi2-dist-%d-%s-%d.jpg' % (wdir, istep, reaction, idx)) py.close()
def gen_melltab_hybrid(self): path2dytab=conf['path2dytab-hybrid'] for k in conf['dy-pion tabs']: path2dytabK='%s/%d'%(path2dytab,k) checkdir(path2dytabK) for i in range(len(conf['dy-pion tabs'][k]['idx'])): print('%s: %i/%i'%(k,i,len(conf['dy-pion tabs'][k]['idx']))) idx=conf['dy-pion tabs'][k]['idx'][i] S = conf['dy-pion tabs'][k]['S'][i] Y = conf['dy-pion tabs'][k]['Y'][i] Q2 = conf['dy-pion tabs'][k]['Q2'][i] muF2=Q2 data=self._gen_melltab_hybrid(Q2,S,Y,muF2) fname='%s/%d.melltab'%(path2dytabK,idx) np.save(fname,data)
def gen_melltab(self): path2dytab=conf['path2dytab'] for k in conf['dy tabs']: path2dytabK='%s/%d'%(path2dytab,k) checkdir(path2dytabK) npts=len(conf['dy tabs'][k]['value']) for i in range(npts): S = conf['dy tabs'][k]['S'][i] Y = conf['dy tabs'][k]['Y'][i] Q2 = conf['dy tabs'][k]['Q2'][i] idx= conf['dy tabs'][k]['idx'][i] muF2=Q2 data=self._gen_melltab(Q2,S,Y,muF2) fname='%s/%d.melltab'%(path2dytabK,idx) save(data,fname)
def gen_samples(wdir,nsamples=100,nsize=10): load_config('%s/input.py'%wdir) conf['bootstrap']=False istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=5,parallel=True,datasets=True) parman=resman.parman order=parman.order obsres={} if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares print('\ngen ml samples using the setup as in %s\n'%wdir) checkdir('%s/mlsamples'%wdir) data=gen_dataframe(parman.order,obsres,nsize) for _ in range(nsamples): lprint('progress: %d/%d'%(_,nsamples)) par=parman.gen_flat(setup=True) res,rres,nres=resman.get_residuals(par) #--fill dataframe i=data['size'] data['params'][i]=par for reaction in obsres: for idx in data['reactions'][reaction]: prediction=copy.copy(obsres[reaction].tabs[idx]['prediction']) data['reactions'][reaction][idx][i]=prediction data['size']+=1 if data['size']==nsize: save(data,'%s/mlsamples/%s'%(wdir,id_generator(12))) data=gen_dataframe(parman.order,obsres,nsize) print #--close resman resman.shutdown()
def plot_params(wdir,dist,kc): load_config('%s/input.py'%wdir) istep=core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep,replicas[0]) clusters,colors,nc,cluster_order = classifier.get_clusters(wdir,istep,kc) _order = replicas[0]['order'][istep] #--get correct order from dist order = [] idx = [] for i in range(len(_order)): if _order[i][1] != dist: continue order.append(_order[i][2]) idx.append(i) #--get correct params from dist params = np.zeros((len(order),len(replicas))) for i in range(len(order)): for j in range(len(replicas)): params[i][j] = replicas[j]['params'][istep][idx[i]] #--create plot with enough space for # of parameters nrows, ncols = np.ceil(len(order)/5.0), 5 fig = py.figure(figsize=(ncols*7,nrows*4)) X = np.linspace(1,len(replicas),len(replicas)) #--create plot for i in range(len(order)): ax = py.subplot(nrows,ncols, i+1) ax.set_title('%s'%(order[i]), size=20) #params = [replicas[j]['params'][istep][i] for j in range(len(replicas))] color = [colors[clusters[j]] for j in range(len(replicas))] ax.scatter(X,params[i],color=color) ax.plot(X,np.ones(len(X))*np.average(params[i]),'k--',alpha=0.5) filename='%s/gallery/%s-params.png'%(wdir,dist) checkdir('%s/gallery'%wdir) py.savefig(filename) print 'Saving figure to %s'%filename
def plot_chi2_dist(wdir, istep): #--needs revisions NS (09/02/19) labels = load('%s/data/labels-%d.dat' % (wdir, istep)) chi2 = labels['chi2dof'] cluster, colors, nc, order = get_clusters(wdir, istep) nrows = 1 ncols = 1 #--plot labeled residuals ax = py.subplot(nrows, ncols, 1) chi2 = [_ for _ in chi2 if _ < 2] Mean = np.average(chi2) dchi = np.std(chi2) chi2min = Mean - 2 * dchi chi2max = Mean + 2 * dchi R = (chi2min, chi2max) ax.hist(chi2, bins=60, range=R, histtype='step', label='size=%d' % len(chi2)) for j in range(nc): chi2_ = [chi2[i] for i in range(len(chi2)) if cluster[i] == j] c = colors[j] print c, np.mean(chi2_), np.mean(chi2) label = 'cluster id=%d size=%d' % (j, len(chi2_)) ax.hist(chi2_, bins=30, range=R, histtype='step', label=label, color=c) mean = np.average(chi2_) ax.axvline(mean, color=c, ls=':') ax.set_xlabel('chi2/npts') ax.set_ylabel('yiled') ax.legend() ax.text(0.1, 0.8, 'step %d' % istep, transform=ax.transAxes) py.tight_layout() checkdir('%s/gallery' % wdir) py.savefig('%s/gallery/chi2-dist-%d.jpg' % (wdir, istep)) py.close()