def _run(self): # construct workname workname = conf['args'].config.split('/')[-1].replace('.py', '') outputdir = '%s/outputs/%s' % (conf['args'].outdir, workname) checkdir(outputdir) # cp the inputfile to outputdir os.system('cp %s %s/' % (conf['args'].config, outputdir)) self.npar = len(conf['parman'].par) conf['nll'] = self.nll conf['par lims'] = self.get_par_lims() if tol not in conf: conf['tol'] = 1e-10 conf['num points'] = int(self.npar * 2) conf['method'] = 'cov' conf['kappa'] = 1 conf['sample size'] = 1000 checkdir('mcdata') par = conf['parman'].par res = self.get_residuals(par) conf['data size'] = len(res) results = nest.NEST().run() save(results, '%s/nest%d' % (outputdir, conf['args'].idx))
def collect_exp_data(wdir): load_config('%s/input.py'%wdir) conf['bootstrap']=False istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=1,parallel=False,datasets=True) obsres={} if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares print('\ncollecting exp data using the setup as in %s\n'%wdir) data={} for reaction in obsres: tabs={} for idx in obsres[reaction].tabs: tabs[idx]={} tabs[idx]['value']=obsres[reaction].tabs[idx]['value'] tabs[idx]['alpha']=obsres[reaction].tabs[idx]['alpha'] data[reaction]=tabs save(data,'%s/expdata.dat'%(wdir)) #--close resman resman.shutdown()
def generate_xf(wdir, groups, flavors, Q2=None): load_config('%s/input.py' % wdir) ## setup kinematics xs = np.logspace(-2.1, -0.04, 100) if Q2 == None: Q2 = conf['Q20'] print('\ngenerating pdf-proton from LHAPDF at Q2 = %f' % Q2) ## compute xf for all replicas xfs = {} for group in groups: xfs[group] = lhapdf_xfs(group, xs, flavors, Q2) if group == 'JAM19PDF_proton_nlo': xfs[group]['color'] = 'g' elif group == 'NNPDF31_nlo_as_0118': xfs[group]['color'] = 'm' elif group == 'CJ15nlo': xfs[group]['color'] = 'k' elif group == 'MMHT2014nlo68cl': xfs[group]['color'] = 'Yellow' elif group == 'CSKK_nnlo_EIG': xfs[group]['color'] = 'chocolate' elif group == 'ABMP16_3_nlo': xfs[group]['color'] = 'b' print save({ 'X': xs, 'Q2': Q2, 'XF': xfs, 'groups': groups }, '%s/analysis/qpdlib/lhapdf-%f.dat' % (os.environ['FITPACK'], Q2))
def gen_labels(wdir, kc): print('\ngenerating labels for %s\n' % wdir) load_config('%s/input.py' % wdir) istep = core.get_istep() if istep not in kc.nc: print("istep not in kmeanconf.py") sys.exit() nc = kc.nc[istep] hook = kc.hooks[istep] istep = core.get_istep() labels = {} labels['chi2dof'] = _gen_labels_by_chi2(wdir, istep) labels['echi2dof'] = _gen_labels_by_echi2(wdir, istep) labels['cluster'] = _gen_labels_by_cluster(wdir, istep, nc, hook) clusters = sorted(list(set(labels['cluster']))) chi_2_clusters = [0.0 for _ in clusters] length_clusters = [0.0 for _ in clusters] for i in range(len(labels['cluster'])): for j in range(len(clusters)): if labels['cluster'][i] == clusters[j]: chi_2_clusters[j] += labels['chi2dof'][i] length_clusters[j] += 1.0 labels['cluster_average_chi2'] = np.array(chi_2_clusters) / np.array( length_clusters) checkdir('%s/data' % wdir) save(labels, '%s/data/labels-%d.dat' % (wdir, istep))
def gen_jar_file(wdir, kc): print('\ngen jar file using (best cluster) %s\n' % wdir) replicas = core.get_replicas(wdir) conf = {} load_config('%s/input.py' % wdir) istep = core.get_istep() core.mod_conf(istep, replicas[0]) #--set conf as specified in istep cluster, colors, nc, cluster_order = classifier.get_clusters( wdir, istep, kc) best_cluster = cluster_order[0] conf['order'] = replicas[0]['order'][istep] conf['replicas'] = [] cnt = 0 for i in range(len(replicas)): replica = replicas[i] if cluster[i] != best_cluster: continue cnt += 1 conf['replicas'].append(replica['params'][istep]) print('number of replicas = %d' % cnt) checkdir('%s/data' % wdir) save(conf, '%s/data/jar-%d.dat' % (wdir, istep))
def rap_fits(self): checkdir('maxlike') dy = [1.5, 2.0, 2.5, 3.0, 3.5] PAR = {} for _dy in dy: print '#' * 10 conf['datasets']['sidis']['filters'][0]['filter'] = "z<0.6 and Q2>1.69 and pT>0.2 and pT<0.9 and dy>%f" % _dy conf['resman'].setup() par = self.run_leastsq(gen_output=False) PAR[_dy] = par save(PAR, 'maxlike/rap_fits.dat')
def check_snapshot(self): self.npar = len(conf['parman'].par) conf = {} conf['nll'] = self.nll conf['par lims'] = self.get_par_lims() conf['num points'] = self.npar * conf['num points factor'] conf['snapshot'] = conf['args'].snapshot nest = NEST(conf) par = nest.samples_p[-1] self.get_residuals(par) report = self.conf['resman'].gen_report(verb=1, level=1) save(report, 'report')
def gen_samples(wdir,nsamples=100,nsize=10): load_config('%s/input.py'%wdir) conf['bootstrap']=False istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=5,parallel=True,datasets=True) parman=resman.parman order=parman.order obsres={} if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares print('\ngen ml samples using the setup as in %s\n'%wdir) checkdir('%s/mlsamples'%wdir) data=gen_dataframe(parman.order,obsres,nsize) for _ in range(nsamples): lprint('progress: %d/%d'%(_,nsamples)) par=parman.gen_flat(setup=True) res,rres,nres=resman.get_residuals(par) #--fill dataframe i=data['size'] data['params'][i]=par for reaction in obsres: for idx in data['reactions'][reaction]: prediction=copy.copy(obsres[reaction].tabs[idx]['prediction']) data['reactions'][reaction][idx][i]=prediction data['size']+=1 if data['size']==nsize: save(data,'%s/mlsamples/%s'%(wdir,id_generator(12))) data=gen_dataframe(parman.order,obsres,nsize) print #--close resman resman.shutdown()
def single_run(self, path, factor, tol=1e-10): self.npar = len(conf['parman'].par) conf['nll'] = self.nll conf['par lims'] = self.get_par_lims() conf['tol'] = tol conf['num points'] = int(self.npar * factor) conf['method'] = 'cov' conf['kappa'] = 1 conf['sample size'] = 1000 par = conf['parman'].par res = self.get_residuals(par) conf['data size'] = len(res) results = nest.NEST().run() save(results, path)
def gen_dist_from_fitpack(wdir,dist): print('\ngenerating %s tables for benchmark using %s'%(dist,wdir)) load_config('%s/input.py'%wdir) istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=1,parallel=False,datasets=False) parman=resman.parman jar=load('%s/data/jar-%d.dat'%(wdir,istep)) replicas=jar['replicas'] X,Q2=gen_grid(dist) qpd=conf[dist] nx=len(X) nQ2=len(Q2) flavs=[-5,-4,-3,-2,-1,1,2,3,4,5,21] fmap={-5:'bb',-4:'cb',-3:'sb',-2:'ub',-1:'db'} fmap.update({5:'b',4:'c',3:'s',2:'u',1:'d'}) fmap.update({21:'g'}) #--gen qpd data per replica checkdir('%s/benchmark'%wdir) cnt=0 for par in replicas: lprint('progress: %d/%d'%(cnt+1,len(replicas))) parman.set_new_params(par) data={} for _ in Q2: data[_]={} for flav in flavs: data[_][flav]=np.array([qpd.get_xF(x,_,fmap[flav]) for x in X]) save(data,'%s/benchmark/%s-%d.dat'%(wdir,dist,cnt)) cnt+=1 print
def gen_report(self): inputfile = conf['args'].config nestfile = conf['args'].fname nestout = load(nestfile) weight = nestout['weights'][0] q = nestout['samples'][0] self.get_residuals(q) report = conf['resman'].gen_report(verb=1, level=1) save(report, 'report') L = open(inputfile).readlines() for i in range(len(L)): if L[i].startswith('#'): continue if 'params' in L[i] and '<<' in L[i]: l = L[i].split('=')[0].replace('conf', '').replace("'", '') k, kk = l.replace('][', '@').replace('[', '').replace(']', '').split('@')[1:] left = L[i].split('<<')[0] right = L[i].split('>>')[1] L[i] = left + '<<%30.20e>>' % conf['params'][k.strip()][ kk.strip()]['value'] + right if 'norm' in L[i] and '<<' in L[i]: l = L[i].split('=')[0].replace('conf', '').replace("'", '') dum1, k, dum2, kk = l.replace('][', '@').replace('[', '').replace( ']', '').split('@') left = L[i].split('<<')[0] right = L[i].split('>>')[1] value = conf['datasets'][k.strip()]['norm'][int(kk)]['value'] L[i] = left + '<<%30.20e>>' % value + right F = open(inputfile, 'w') F.writelines(L) F.close()
def gen_priors(wdir, kc, nsamples): load_config('%s/input.py' % wdir) istep = core.get_istep() #cluster,colors,nc,cluster_order = core.get_clusters(wdir,istep) cluster, colors, nc, cluster_order = classifier.get_clusters( wdir, istep, kc) best_cluster = cluster_order[0] #--get data replicas = core.get_replicas(wdir) step = conf['steps'][istep] samples = [] for i in range(len(replicas)): if cluster[i] != best_cluster: continue replica = replicas[i] order = replica['order'][istep] params = replica['params'][istep] samples.append(params) samples = np.array(samples) pmin = [np.amin(p) for p in samples.T] pmax = [np.amax(p) for p in samples.T] new = [] for i in range(len(pmin)): new.append(np.random.uniform(low=pmin[i], high=pmax[i], size=nsamples)) new = np.transpose(new) checkdir('%s/msr-opt-priors' % wdir) replica = replicas[0] #--template replica for i in range(nsamples): lprint('%d/%d' % (i + 1, nsamples)) replica['params'][istep] = new[i] fname = '%s/msr-opt-priors/%s.msr' % (wdir, id_generator(12)) save(replica, fname) print
def gen_dist_from_lhapdf(wdir,dist,dirname): print('\ngenerating %s tables for benchmark using %s from lhapdf'%(dist,wdir)) replicas=lhapdf.mkPDFs(dirname) X,Q2=gen_grid(dist) flavs=[-5,-4,-3,-2,-1,1,2,3,4,5,21] #--gen qpd data per replica checkdir('%s/benchmark'%wdir) cnt=0 for i in range(len(replicas)): lprint('progress: %d/%d'%(cnt+1,len(replicas))) replica=replicas[i] data={} for _ in Q2: data[_]={} for flav in flavs: data[_][flav]=np.array([replica.xfxQ2(flav,x,_) for x in X]) save(data,'%s/benchmark/%s-%d-lha.dat'%(wdir,dist,cnt)) cnt+=1 print
def results(self, fname, cmd=None): ########################################## # dx=0.5*(np.array(self.samples_x[:-1])-np.array(self.samples_x[1:])) # log_l=np.log(self.samples_l[1:]) # log_w=np.log(dx*self.jac)+log_l # weights=np.exp(log_w) # weights/=np.sum(weights) # data={} # data['samples']=self.samples_p[::-1] # data['x']=self.samples_x[1:][::-1] # data['l']=self.samples_l[1:][::-1] # data['nll']=self.samples_nll[::-1] # data['logz']=self.logz # data['weights']=weights[::-1] #data['active p']=self.active_p #data['active nll']=self.active_nll # return data ########################################## data = {} data['nll'] = copy.copy(self.samples_nll) data['samples'] = copy.copy(self.samples_p) data['num active points'] = conf['num points'] self.samples_nll = [] self.samples_p = [] save(data, fname) self.block_cnt += 1 self.block_size = 0 self.status = 'ready' if cmd != None: # print #print fname #print cmd.replace('fname',fname) os.system(cmd.replace('fname', fname))
def run(self): global conf load_config(self.inputfile) #--modify confs if 'hooks' not in conf: conf['hooks'] = {} if self.msrhook != None: conf['hooks']['msr'] = self.msrhook if self.verbose == False: conf['verbose'] = 1 else: conf['verbose'] = self.verbose #--backup conf(after mods) conf_bkp = copy.deepcopy(conf) #--decide if the input should be modified output = True if 'bootstrap' in conf and conf['bootstrap'] == True: output = False if 'steps' in conf: isteps = sorted(conf['steps']) self.order = {} self.params = {} chi2 = {} if self.prior != None: prior = load(self.prior) self.order = prior['order'] self.params = prior['params'] chi2 = prior['chi2'] prior_steps = sorted(self.order.keys()) isteps = [i for i in isteps if i not in prior_steps] for i in isteps: step = conf['steps'][i] conf.update(copy.deepcopy(conf_bkp)) conf.update(self.get_conf(conf_bkp, step)) print msg = '--step %d: ' % i msg += 'npQCD objects =' for _ in conf['params'].keys(): msg += _ + ' ' msg += 'datasets =' for _ in conf['datasets'].keys(): msg += _ + ' ' print msg par = self.get_par(i, step['dep']) self.order[i] = self.parman.order[:] self.params[i] = par[:] chi2[i] = self.resman.get_chi2() if output: self.gen_output(i) #--store the results from steps data = {'order': self.order, 'params': self.params, 'chi2': chi2} if self.prior == None: fname = '%s.msr' % id_generator(size=12) else: fname = self.prior.split('/')[-1] save(data, fname) #--run hooks if 'msr' in conf['hooks']: #--ms==multi steps cmd = conf['hooks']['msr'].replace('<<fname>>', fname) os.system(cmd) else: par = self.get_par(0) if output: self.gen_output(0) #--run remining hooks if available for _ in conf['hooks']: if _ == 'msr': continue os.system(conf['hooks'][_])
def get_predictions(wdir, cores=2, mod_conf=None, pre_flags={}): ## 'pre_flags' can be defined for example like the following ## pre_flags = {'regenerate_same': False, 'regenerate_part': False, 'regenerate_mismatch': True} if mod_conf == None: load_config('%s/input.py' % wdir) else: config.conf = conf conf['bootstrap'] = False istep = core.get_istep() current_replica_names = sorted(os.listdir('%s/msr-inspected' % wdir)) previous_replica_names_path = '%s/data/predicted-replicas-%d.dat' % (wdir, istep) previous_predictions_path = '%s/data/predictions-%d.dat' % (wdir, istep) part_flag = False if os.path.exists(previous_predictions_path) and os.path.exists( previous_replica_names_path): previous_replica_names = load(previous_replica_names_path) if current_replica_names == previous_replica_names: print( 'The replicas names are the same since predictions were generated last time,' ) print( 'do you wish to regenrate and overwrite previous predictions?') if 'regenerate_same' in pre_flags: if pre_flags['regenerate_same']: print( 'regenerating everything based on predifined flag...') part_flag = False else: print( 'exit generating predictions based on predifined flag.' ) return else: if sys.version_info[0] == 2: while True: same_flag = raw_input( 'y (regenerate)/n (skip generation of predictions): ' ) if same_flag == '': print('please type "y" or "n"') elif same_flag.lower()[0] == 'n': return elif same_flag.lower()[0] == 'y': part_flag = False break else: print('please type "y" or "n"') elif sys.version_info[0] == 3: while True: same_flag = input( 'y (regenerate)/n (skip generation of predictions): ' ) if same_flag == '': print('please type "y" or "n"') elif same_flag.lower()[0] == 'n': return elif same_flag.lower()[0] == 'y': part_flag = False break else: print('please type "y" or "n"') elif all(_ in current_replica_names for _ in previous_replica_names): print('Based on the replica names,') print( 'part of the predictions you are trying to generate has already been generated last time,' ) print( 'do you wish to regenrate with all replicas or only generate with new replicas?' ) if 'regenerate_part' in pre_flags: if pre_flags['regenerate_part']: print( 'regenerating with all replicas based on predifined flag...' ) part_flag = False else: print( 'generating only on new replicas based on predifined flag...' ) part_flag = True else: if sys.version_info[0] == 2: while True: part_flag = raw_input( 'w (regenerate with all replicas)/p (generate only with new replicas): ' ) if part_flag == '': print('please type "w" or "p"') elif part_flag.lower()[0] == 'w': part_flag = False break elif part_flag.lower()[0] == 'p': part_flag = True break else: print('please type "y" or "n"') elif sys.version_info[0] == 3: while True: part_flag = input( 'w (regenerate with all replicas)/p (generate only with new replicas): ' ) if part_flag == '': print('please type "w" or "p"') elif part_flag.lower()[0] == 'w': part_flag = False break elif part_flag.lower()[0] == 'p': part_flag = True break else: print('please type "y" or "n"') else: part_flag = False else: pass if part_flag: replicas_to_load = np.setdiff1d(current_replica_names, previous_replica_names) replicas = [ load('%s/msr-inspected/%s' % (wdir, _)) for _ in replicas_to_load ] else: replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) #--set conf as specified in istep resman = RESMAN(nworkers=cores, parallel=True, datasets=True) parman = resman.parman order = replicas[0]['order'][istep] ## order and parameters are updated later for each replicas individually # parman.order = order # order = parman.order if part_flag: previous_data = load(previous_predictions_path) if order != previous_data['order']: print( 'Previous replicas have different parameter orders from new replicas,' ) print('previously: ', previous_data['order']) print('currently: ', order) print( 'do you wish to overwrite the previous predictions or exit to check?' ) if 'regenerate_mismatch' in pre_flags: if pre_flags['regenerate_mismatch']: print( 'regenerating everything based on predifined flag...') part_flag = False exit_flag = False else: print( 'exit generating predictions based on predifined flag') exit_flag = True else: part_flag, exit_flag = ask_replica_mismatch() if exit_flag: sys.exit() obsres = {} if 'idis' in conf['datasets']: obsres['idis'] = resman.idis_res if 'pidis' in conf['datasets']: obsres['pidis'] = resman.pidis_res if 'sidis' in conf['datasets']: obsres['sidis'] = resman.sidis_res if 'psidis' in conf['datasets']: obsres['psidis'] = resman.psidis_res if 'dy' in conf['datasets']: obsres['dy'] = resman.dy_res if 'sia' in conf['datasets']: obsres['sia'] = resman.sia_res if 'qpdf' in conf['datasets']: obsres['qpdf'] = resman.qpdf_res if 'dy-pion' in conf['datasets']: obsres['dy-pion'] = resman.dy_pion_res if 'pion_qT' in conf['datasets']: obsres['pion_qT'] = resman.pion_qTres if 'ln' in conf['datasets']: obsres['ln'] = resman.ln_res if 'jet' in conf['datasets']: obsres['jet'] = resman.jet_res if 'pjet' in conf['datasets']: obsres['pjet'] = resman.pjet_res if part_flag: # previous_data = load(previous_predictions_path) if obsres.keys() != previous_data['reactions'].keys(): print( 'Previous replicas have different observables from new replicas,' ) print('previously: ', previous_data['reactions'].keys()) print('currently: ', obsres.keys()) print( 'do you wish to overwrite the previous predictions or exit to check?' ) if 'regenerate_mismatch' in pre_flags: if pre_flags['regenerate_mismatch']: print( 'regenerating everything based on predifined flag...') part_flag = False exit_flag = False else: print( 'exit generating predictions based on predifined flag') exit_flag = True else: part_flag, exit_flag = ask_replica_mismatch() if exit_flag: sys.exit() if part_flag: for _ in obsres.keys(): if obsres[_].tabs.keys() != previous_data['reactions'][_].keys( ): print( 'Previous replicas have different datasets from new replicas in observable %s,' % _) print('previously: ', previous_data['reactions'][_].keys()) print('currently: ', obsres[_].tabs.keys()) print( 'do you wish to overwrite the previous predictions or exit to check?' ) if 'regenerate_mismatch' in pre_flags: if pre_flags['regenerate_mismatch']: print( 'regenerating everything based on predifined flag...' ) part_flag = False exit_flag = False else: print( 'exit generating predictions based on predifined flag' ) exit_flag = True else: part_flag, exit_flag = ask_replica_mismatch() if exit_flag: sys.exit() #--setup big table to store all we want data = {} data['order'] = order data['params'] = [] data['reactions'] = {} data['res'] = [] data['rres'] = [] data['nres'] = [] for _ in obsres: tabs = copy.copy(obsres[_].tabs) #--create a space to store all the predictions from replicas for idx in tabs: tabs[idx]['prediction-rep'] = [] tabs[idx]['residuals-rep'] = [] data['reactions'][_] = tabs print('\ngen predictions using %s\n' % wdir) cnt = 0 if part_flag: n_all_replicas = len(current_replica_names) n_available_replicas = len(previous_replica_names) cnt += n_available_replicas else: replicas = core.get_replicas(wdir) for replica in replicas: cnt += 1 if part_flag: lprint('progress: %d/%d' % (cnt, n_all_replicas)) else: lprint('progress: %d/%d' % (cnt, len(replicas))) core.mod_conf(istep, replica) #--will update passive dist parman.par = copy.copy(replica['params'][istep]) parman.order = copy.copy(replica['order'][istep]) data['params'] = np.append(data['params'], parman.par) #for i in range(len(parman.par)): # print parman.order[i],parman.par[i],replica['order'][istep][i] #--compute residuals (==theory) res, rres, nres = resman.get_residuals(parman.par, initial=True) data['res'].append(res) data['rres'].append(rres) data['nres'].append(nres) #--save predictions of the current step and current replica at data for _ in obsres: for idx in data['reactions'][_]: prediction = copy.copy(obsres[_].tabs[idx]['prediction']) residuals = copy.copy(obsres[_].tabs[idx]['residuals']) data['reactions'][_][idx]['prediction-rep'].append(prediction) data['reactions'][_][idx]['residuals-rep'].append(residuals) print #--close resman resman.shutdown() ## append newly generated parts if 'part_flag' is True if part_flag: data['params'] = np.append(previous_data['params'], data['params']) data['res'] = np.concatenate((previous_data['res'], data['res'])) data['rres'] = np.concatenate((previous_data['rres'], data['rres'])) data['nres'] = np.concatenate((previous_data['nres'], data['nres'])) for _ in data['reactions']: for dataset in data['reactions'][_]: data['reactions'][_][dataset][ 'prediction-rep'] = previous_data['reactions'][_][dataset][ 'prediction-rep'] + data['reactions'][_][dataset][ 'prediction-rep'] data['reactions'][_][dataset]['residuals-rep'] = previous_data[ 'reactions'][_][dataset]['prediction-rep'] + data[ 'reactions'][_][dataset]['residuals-rep'] #--convert tables to numpy array before saving for _ in ['res', 'rres', 'nres']: data[_] = np.array(data[_]) checkdir('%s/data' % wdir) if mod_conf == None: save(data, '%s/data/predictions-%d.dat' % (wdir, istep)) save(current_replica_names, '%s/data/predicted-replicas-%d.dat' % (wdir, istep)) else: save(data, '%s/data/predictions-%d-sim.dat' % (wdir, istep))
def get_summary(self, wdir, istep): #--needs revision NS (09/02/19) data = load('%s/data/predictions-%d.dat' % (wdir, istep)) summary = [] dic = {} global_chi2 = 0 global_npts = 0 for reaction in data['reactions']: dic[reaction] = {} for idx in data['reactions'][reaction]: dic[reaction][idx] = {} tab = data['reactions'][reaction][idx] value = tab['value'] alpha = tab['alpha'] CHI2 = [] for i in range(len(tab['predictions'])): chi2tot = np.sum(data['res'][i]**2) / data['res'][i].size #if chi2tot>2: continue prediction = tab['predictions'][i] CHI2.append(np.sum(((value - prediction) / alpha)**2)) npts = len(value) col = tab['col'][0] chi2min = np.amin(CHI2) / npts chi2max = np.amax(CHI2) / npts chi2ave = np.mean(CHI2) / npts global_chi2 += np.amin(CHI2) global_npts += npts msg = 'reaction: %8s' msg += ' idx: %7d' msg += ' col: %10s' msg += ' chi2/npts (min,ave,max): %6.2f %6.2f %6.2f' msg += ' npts: %5d' msg = msg % (reaction, idx, col[:10], chi2min, chi2ave, chi2max, npts) dic[reaction][idx]['col'] = col dic[reaction][idx]['chi2'] = np.amin(CHI2) dic[reaction][idx]['chi2/npts'] = np.amin(CHI2) / npts dic[reaction][idx]['npts'] = npts if reaction == 'dy': msg += ' rea: %s' % tab['reaction'][0] if reaction == 'sia': msg += ' had: %s' % tab['hadron'][0] if reaction == 'idis': msg += ' tar: %s' % tab['target'][0] msg += ' obs: %s' % tab['obs'][0] if reaction == 'sidis': msg += ' tar: %1s' % tab['target'][0][0] msg += ' had: %5s' % tab['hadron'][0] msg += ' obs: %s' % tab['obs'][0] if reaction == 'pidis': msg += ' tar: %s' % tab['target'][0] msg += ' obs: %s' % tab['obs'][0] if reaction == 'psidis': msg += ' tar: %1s' % tab['target'][0][0] msg += ' had: %5s' % tab['hadron'][0] msg += ' obs: %s' % tab['obs'][0] if reaction == 'jet': msg += ' rea: %s' % tab['reaction'][0] msg += ' col: %s' % tab['col'][0] if reaction == 'pjet': msg += ' rea: %s' % tab['reaction'][0] msg += ' col: %s' % tab['col'][0] #print msg summary.append(msg) #print tab.keys() print 'global summary' print 'chi2/npts =', global_chi2 / global_npts print 'npts =', global_npts summary = [_ + '\n' for _ in summary] F = open('%s/data/summary-%d.txt' % (wdir, istep), 'w') F.writelines(summary) F.close() save(dic, '%s/data/summary-%d.dat' % (wdir, istep))
def gen_xf(wdir, flavors=['g', 'u', 'ub', 'd', 'db', 's', 'sb'], Q2=None): load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) ## 'conf' will be modified for each replica individually later in the loop over 'replicas' ## the reason for doing this is that 'fix parameters' has to be set correctly for each replica if 'pdf' not in conf['steps'][istep]['active distributions']: print('pdf-proton not in active distribution') return resman = RESMAN(nworkers=1, parallel=False, datasets=False) parman = resman.parman parman.order = replicas[0]['order'][istep] ## make sure 'parman' uses the same order for active distributions as all the replicas do # print parman.order pdf = conf['pdf'] ## setup kinematics xs = 10.0**np.linspace(-3, -1, 100) xs = np.append(xs, np.linspace(0.1, 0.99, 100)) if Q2 == None: Q2 = conf['Q20'] print('\ngenerating pdf-proton from %s at Q2 = %.2f' % (wdir, Q2)) ## compute xf for all replicas xfs = {} n_replicas = len(replicas) for i in range(n_replicas): lprint('%d/%d' % (i + 1, n_replicas)) ## filter #flag=False #params=replica['params'][istep] #order=replica['order'][istep] #for i in range(len(order)): # if order[i][0]!=1:continue # if order[i][1]!='pdf':continue # #if order[i][2]=='s1 a': # # if params[i]<-0.9: flag=True #if flag: continue parman.order = copy.copy(replicas[i]['order'][istep]) core.mod_conf(istep, replicas[i]) parman.set_new_params(replicas[i]['params'][istep], initial=True) for flavor in flavors: if flavor not in xfs: xfs[flavor] = [] if flavor == 'rs': func = lambda x: (pdf.get_xF(x, Q2, 's') + pdf.get_xF( x, Q2, 'sb')) / (pdf.get_xF(x, Q2, 'db') + pdf.get_xF( x, Q2, 'ub')) elif flavor == 'uv': func = lambda x: pdf.get_xF(x, Q2, 'u') - pdf.get_xF( x, Q2, 'ub') elif flavor == 'dv': func = lambda x: pdf.get_xF(x, Q2, 'd') - pdf.get_xF( x, Q2, 'db') elif flavor == 'd/u': func = lambda x: pdf.get_xF(x, Q2, 'd') / pdf.get_xF( x, Q2, 'u') elif flavor == 'db+ub': func = lambda x: pdf.get_xF(x, Q2, 'db') + pdf.get_xF( x, Q2, 'ub') elif flavor == 'db-ub': func = lambda x: pdf.get_xF(x, Q2, 'db') - pdf.get_xF( x, Q2, 'ub') elif flavor == 's+sb': func = lambda x: pdf.get_xF(x, Q2, 's') + pdf.get_xF( x, Q2, 'sb') else: func = lambda x: pdf.get_xF(x, Q2, flavor) xfs[flavor].append(np.array([func(x) for x in xs])) print checkdir('%s/data' % wdir) if Q2 == conf['Q20']: save({ 'X': xs, 'Q2': Q2, 'XF': xfs }, '%s/data/pdf-%d.dat' % (wdir, istep)) else: save({ 'X': xs, 'Q2': Q2, 'XF': xfs }, '%s/data/pdf-%d-%f.dat' % (wdir, istep, Q2))
def soupCrawl(target_file, area_code, page): """ crawling the information from file ファイルから情報をクロールする :param target_file: str file that want to crawl クロールしたいファイル :param area_code: str the area code エリアコード :param page: str page ページ :return: """ # web url using response # レスポンスを使ったウェブURL #response = requests.get(target_url) # local file use open # ローカルファイル使用オープン soup = BeautifulSoup(open(target_file, encoding="utf-8"), "html.parser") area_name = soup.find("span", "list-sidebar__item-title").text.strip() # list # リスト nameList = [] ekiList = [] ratingList = [] commentList = [] dinnerMinList = [] dinnerMaxList = [] lunchMinList = [] lunchMaxList = [] dinnerAvgList = [] lunchAvgList = [] nomihodaiList = [] tabehodaiList = [] # extract restaurant information # レストラン情報の抽出 restaurants = soup.find_all("div", "list-rst__wrap js-open-new-window") for r in restaurants: # name # 名前 name = r.find("a", class_="list-rst__rst-name-target cpy-rst-name") if (name is not None): nameList.append(name.text.replace("\t", " ")) else: nameList.append("-") # eki # 駅の名前 eki = r.find("span", class_="list-rst__area-genre cpy-area-genre") if (eki is not None): ekiList.append(eki.text.split()[0]) else: ekiList.append("-") # rating # 評価 rating = r.find( "span", class_="c-rating__val c-rating__val--strong list-rst__rating-val") if (rating is not None): ratingList.append(rating.text) else: ratingList.append("-") # comment # コメント comment = r.find("em", class_="list-rst__rvw-count-num cpy-review-count") if (comment is not None): commentList.append(comment.text) else: commentList.append("0") # dinner # ディナー dinner = r.find( "span", class_="c-rating__val list-rst__budget-val cpy-dinner-budget-val") if (dinner is not None): dinnerInfo = convertPrice(dinner.text) dinnerMinList.append(dinnerInfo[0]) dinnerMaxList.append(dinnerInfo[1]) dinnerAvgList.append(dinnerInfo[2]) else: dinnerMinList.append("-") dinnerMaxList.append("-") dinnerAvgList.append("-") # lunch # ランチ lunch = r.find( "span", class_="c-rating__val list-rst__budget-val cpy-lunch-budget-val") if (lunch is not None): lunchInfo = convertPrice(lunch.text) lunchMinList.append(lunchInfo[0]) lunchMaxList.append(lunchInfo[1]) lunchAvgList.append(lunchInfo[2]) else: lunchMinList.append("-") lunchMaxList.append("-") lunchAvgList.append("-") # nomihodai & tabehodai # 飲み放題と食べ放題 hodai = r.find_all("li", class_="list-rst__search-word-item") hodaiResult = convertHodai(hodai) nomihodaiList.append(hodaiResult[0]) tabehodaiList.append(hodaiResult[1]) # print total information # 合計情報を印刷する result = "" for i in range(0, len(nameList)): text=area_name+"\t"+area_code+"\t"+str(page)+"\t"+nameList[i]+"\t"+\ ekiList[i]+"\t"+ratingList[i]+"\t"+commentList[i]+"\t"+\ dinnerMinList[i]+"\t"+dinnerMaxList[i]+"\t"+\ lunchMinList[i]+"\t"+lunchMaxList[i]+"\t"+\ dinnerAvgList[i]+"\t"+lunchAvgList[i]+"\t"+\ nomihodaiList[i]+"\t"+tabehodaiList[i]+"\n" print(text) result += text # remove last \n # 最後に\nを削除 #result=result[:-1] #print(result) # save file # ファイルを保存 tools.save(config.config['CRAWL']['FILENAME'], result)
def get_moments(wdir, flavors, x_1, x_2, Q2=None): load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) ## 'conf' will be modified for each replica individually later in the loop over 'replicas' ## the reason for doing this is that 'fix parameters' has to be set correctly for each replica if 'ppdf_smx' not in conf['steps'][istep]['active distributions']: print('ppdf-smx-proton not in active distribution') return resman = RESMAN(nworkers=1, parallel=False, datasets=False) parman = resman.parman parman.order = replicas[0]['order'][istep] ## make sure 'parman' uses the same order for active distributions as all the replicas do # print parman.order ppdf = conf['ppdf_smx'] ## setup kinematics # X = 10.0 ** np.linspace(-3, -1, 100) # X = np.append(X, np.linspace(0.1, 0.99, 100)) if Q2 == None: Q2 = conf['Q20'] print( '\ngenerating momentus for polarized pdf_smx-proton from %s at Q2 = %f' % (wdir, Q2)) ## compute moments for all replicas moments = {} statistics = {} n_replicas = len(replicas) for i in range(n_replicas): lprint('%d/%d' % (i + 1, n_replicas)) parman.order = copy.copy(replicas[i]['order'][istep]) core.mod_conf(istep, replicas[i]) parman.set_new_params(replicas[i]['params'][istep], initial=True) for flavor in flavors: if flavor not in moments: moments[flavor] = [] if flavor not in statistics: statistics[flavor] = {} if flavor == 'up': func = lambda x: (ppdf.get_xF(x, Q2, 'u') + ppdf.get_xF( x, Q2, 'ub')) / x elif flavor == 'dp': func = lambda x: (ppdf.get_xF(x, Q2, 'd') + ppdf.get_xF( x, Q2, 'sb')) / x elif flavor == 'sp': func = lambda x: (ppdf.get_xF(x, Q2, 's') + ppdf.get_xF( x, Q2, 'sb')) / x else: func = lambda x: ppdf.get_xF(x, Q2, flavor) / x moments[flavor].append(quad(func, x_1, x_2)[0]) for flavor in flavors: statistics[flavor]['mean'] = np.mean(moments[flavor]) statistics[flavor]['standard_deviation'] = np.std(moments[flavor]) print for flavor in flavors: print flavor, ': ', statistics[flavor] checkdir('%s/data' % wdir) if Q2 == conf['Q20']: save({ 'Q2': Q2, 'moments': moments, 'statistics': statistics }, '%s/data/ppdf_smx-moments-%s-to-%s-%d.dat' % (wdir, x_1, x_2, istep)) else: save({ 'Q2': Q2, 'moments': moments, 'statistics': statistics }, '%s/data/ppdf_smx-moments-%s-to-%s-%d-%f.dat' % (wdir, x_1, x_2, istep, Q2))
def gen_xf(wdir, flavors, Q2=None): load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) ## 'conf' will be modified for each replica individually later in the loop over 'replicas' ## the reason for doing this is that 'fix parameters' has to be set correctly for each replica if 'ppdf_smx' not in conf['steps'][istep]['active distributions']: print('ppdf_smx-proton not in active distribution') return resman = RESMAN(nworkers=1, parallel=False, datasets=False) parman = resman.parman parman.order = replicas[0]['order'][istep] ## make sure 'parman' uses the same order for active distributions as all the replicas do # print parman.order ppdf = conf['ppdf_smx'] ppdfJAM = conf['ppdf'] ## setup kinematics xs = 10.0**np.linspace(-5, -1, 100) xs = np.append(xs, np.linspace(0.1, 0.3, 100)) #xs = np.append(xs, np.linspace(0.1, 0.99, 100)) if Q2 == None: Q2 = conf['Q20'] print('\ngenerating polarized pdf-proton from %s at Q2 = %f' % (wdir, Q2)) ## compute xf for all replicas xfs = {} n_replicas = len(replicas) for i in range(n_replicas): lprint('%d/%d' % (i + 1, n_replicas)) ## filter #flag=False #params=replica['params'][istep] #order=replica['order'][istep] #for i in range(len(order)): # if order[i][0]!=1:continue # if order[i][1]!='ppdf':continue # #if order[i][2]=='s1 a': # # if params[i]<-0.9: flag=True #if flag: continue parman.order = copy.copy(replicas[i]['order'][istep]) core.mod_conf(istep, replicas[i]) parman.set_new_params(replicas[i]['params'][istep], initial=True) x0 = conf['smx']['x0'] xsep = conf['smx']['xsep'] for flavor in flavors: if flavor not in xfs: xfs[flavor] = [] if flavor == 'up': #func = lambda x: x*ppdf.get_C(x, Q2)[1] #func = lambda x: np.heaviside(xsep-x,1)*(x*ppdf.get_C(x, Q2)[1] + ppdfJAM.get_xF(xsep,xsep*Q2/x,flavor)) + np.heaviside(x-xsep,0)*ppdfJAM.get_xF(x,Q2,flavor) func = lambda x: np.heaviside(xsep - x, 1) * (x * ppdf.get_C( x, Q2)[1] + x * ppdf.get_C_NOevo(xsep, xsep * Q2 / x)[ 1]) + np.heaviside(x - xsep, 0) * x * ppdf.get_C_NOevo( x, Q2)[1] elif flavor == 'dp': #func = lambda x: x*ppdf.get_C(x, Q2)[2] #func = lambda x: np.heaviside(xsep-x,1)*(x*ppdf.get_C(x, Q2)[2] + ppdfJAM.get_xF(xsep,xsep*Q2/x,flavor)) + np.heaviside(x-xsep,0)*ppdfJAM.get_xF(x,Q2,flavor) func = lambda x: np.heaviside(xsep - x, 1) * (x * ppdf.get_C( x, Q2)[2] + x * ppdf.get_C_NOevo(xsep, xsep * Q2 / x)[ 2]) + np.heaviside(x - xsep, 0) * x * ppdf.get_C_NOevo( x, Q2)[2] elif flavor == 'sp': #func = lambda x: x*ppdf.get_C(x, Q2)[3] #func = lambda x: np.heaviside(xsep-x,1)*(x*ppdf.get_C(x, Q2)[3] + ppdfJAM.get_xF(xsep,xsep*Q2/x,flavor)) + np.heaviside(x-xsep,0)*ppdfJAM.get_xF(x,Q2,flavor) func = lambda x: np.heaviside(xsep - x, 1) * (x * ppdf.get_C( x, Q2)[3] + x * ppdf.get_C_NOevo(xsep, xsep * Q2 / x)[ 3]) + np.heaviside(x - xsep, 0) * x * ppdf.get_C_NOevo( x, Q2)[3] elif flavor == 'g': #func = lambda x: x*ppdf.get_C(x, Q2)[0] #func = lambda x: np.heaviside(xsep-x,1)*(x*ppdf.get_C(x, Q2)[0] + ppdfJAM.get_xF(xsep,xsep*Q2/x,flavor)) + np.heaviside(x-xsep,0)*ppdfJAM.get_xF(x,Q2,flavor) func = lambda x: np.heaviside(xsep - x, 1) * (x * ppdf.get_C( x, Q2)[0] + x * ppdf.get_C_NOevo(xsep, xsep * Q2 / x)[ 0]) + np.heaviside(x - xsep, 0) * x * ppdf.get_C_NOevo( x, Q2)[0] xfs[flavor].append([func(x) for x in xs]) print checkdir('%s/data' % wdir) if Q2 == conf['Q20']: save({ 'X': xs, 'Q2': Q2, 'XF': xfs }, '%s/data/ppdf_smx-%d.dat' % (wdir, istep)) else: save({ 'X': xs, 'Q2': Q2, 'XF': xfs }, '%s/data/ppdf_smx-%d-%f.dat' % (wdir, istep, Q2))
def get_parameters(wdir, Q2=None): load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) ## 'conf' will be modified for each replica individually later in the loop over 'replicas' ## the reason for doing this is that 'fix parameters' has to be set correctly for each replica if 'ppdf_smx' not in conf['steps'][istep]['active distributions']: print('ppdf_smx-proton not in active distribution') return resman = RESMAN(nworkers=1, parallel=False, datasets=False) parman = resman.parman parman.order = replicas[0]['order'][ istep] ## make sure 'parman' uses the same order as all the replicas do # print parman.order ppdf = conf['ppdf_smx'] ## setup kinematics if Q2 == None: Q2 = conf['Q20'] ## get parameters for all flavors of PDF print('\ngetting ppdf_smx-parameters from %s at Q2 = %.2f' % (wdir, Q2)) ## check if any of the shapes are fixed shape_1 = [] shape_2 = [] shape_3 = [] for parameter in conf['params']['ppdf_smx']: if '1' in parameter: shape_1.append(conf['params']['ppdf_smx'][parameter]['fixed']) elif '2' in parameter: shape_2.append(conf['params']['ppdf_smx'][parameter]['fixed']) elif '3' in parameter: shape_3.append(conf['params']['ppdf_smx'][parameter]['fixed']) else: print('there seems to be more than three shapes') print(parameter) sys.exit('please update script %s' % __file__) fixed_shape_1 = False fixed_shape_2 = False fixed_shape_3 = False if (len(shape_1) != 0) and all([_ == True for _ in shape_1]): fixed_shape_1 = True elif len(shape_1) == 0: fixed_shape_1 = True if (len(shape_2) != 0) and all([_ == True for _ in shape_2]): fixed_shape_2 = True elif len(shape_2) == 0: fixed_shape_2 = True if (len(shape_3) != 0) and all([_ == True for _ in shape_3]): fixed_shape_3 = True elif len(shape_3) == 0: fixed_shape_3 = True parameters = {} if not fixed_shape_1: parameters[1] = {} if not fixed_shape_2: parameters[2] = {} if not fixed_shape_3: parameters[3] = {} n_replicas = len(replicas) for i in range(n_replicas): lprint('%d/%d' % (i + 1, n_replicas)) parman.order = copy.copy(replicas[i]['order'][istep]) core.mod_conf(istep, replicas[i]) parman.set_new_params(replicas[i]['params'][istep], initial=True) for flavor, value in ppdf.params.iteritems(): for j in parameters: if str(j) in flavor: flavor_key = flavor.replace(str(j), '') if flavor_key not in parameters[j]: parameters[j][flavor_key] = { 'n': [], 'a': [], 'b': [], 'c': [], 'd': [] } parameters[j][flavor_key]['n'].append(value[0]) parameters[j][flavor_key]['a'].append(value[1]) parameters[j][flavor_key]['b'].append(value[2]) parameters[j][flavor_key]['c'].append(value[3]) parameters[j][flavor_key]['d'].append(value[4]) else: pass ## remove c or d parameters if fixed in all flavors and shapes ## this is to avoid producing empty figures shapes_fixed_c = [] shapes_fixed_d = [] for shape in parameters: flavors_fixed_c = [] flavors_fixed_d = [] for flavor in parameters[shape]: cs = parameters[shape][flavor]['c'] ds = parameters[shape][flavor]['d'] if all([_ == cs[0] for _ in cs]): flavors_fixed_c.append(True) else: flavors_fixed_c.append(False) if all([_ == ds[0] for _ in ds]): flavors_fixed_d.append(True) else: flavors_fixed_d.append(False) if all(flavors_fixed_c): shapes_fixed_c.append(True) else: shapes_fixed_c.append(False) if all(flavors_fixed_d): shapes_fixed_d.append(True) else: shapes_fixed_d.append(False) if all(shapes_fixed_c): for shape in parameters: for flavor in parameters[shape]: del parameters[shape][flavor]['c'] print('parameter c is fixed for shape %s' % shape) if all(shapes_fixed_d): for shape in parameters: for flavor in parameters[shape]: del parameters[shape][flavor]['d'] print('parameter d is fixed for shape %s' % shape) print checkdir('%s/data' % wdir) if Q2 == conf['Q20']: save(parameters, '%s/data/ppdf_smx-parameters-%d.dat' % (wdir, istep)) else: save(parameters, '%s/data/ppdf_smx-parameters-%d-%f.dat' % (wdir, istep, Q2))
def gen_xf(wdir, had, flavors=['g', 'u', 'ub', 'd', 'db', 's', 'sb'], Q2=None): fflabel = 'ff%s' % had print('\ngenerating ff-%s from %s' % (had, wdir)) load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) ## set conf as specified in istep if fflabel not in conf['steps'][istep]['active distributions']: print('ff-%s not in active distribution' % had) return resman = RESMAN(nworkers=1, parallel=False, datasets=False) parman = resman.parman jar = load('%s/data/jar-%d.dat' % (wdir, istep)) ## 'jar' contains parameters and their orders replicas = jar['replicas'] ## 'jar['replicas']' is a list, in which each element is a list of parameters obeying 'jar['order']' ff = conf[fflabel] ## setup kinematics X = np.linspace(0.01, 0.99, 100) if Q2 == None: Q2 = conf['Q20'] ## compute XF for all replicas XF = {} cnt = 0 for par in replicas: cnt += 1 lprint('%d/%d' % (cnt, len(replicas))) ## filter #flag=False #params=replica['params'][istep] #order=replica['order'][istep] #for i in range(len(order)): # if order[i][0]!=1:continue # if order[i][1]!='pdf':continue # #if order[i][2]=='s1 a': # # if params[i]<-0.9: flag=True #if flag: continue parman.set_new_params(par, initial=True) #print #print conf['ffpion'].get_xF(0.5,10.0,'u') #print conf['ffkaon'].get_xF(0.5,10.0,'u') #print ff.get_xF(0.5,10.0,'u') for flavor in flavors: if flavor not in XF: XF[flavor] = [] if flavor == 'c' or flavor == 'cb' or flavor == 'c+cb': _Q2 = conf['aux'].mc2 + 1 elif flavor == 'b' or flavor == 'bb' or flavor == 'b+bb': _Q2 = conf['aux'].mb2 + 1 else: _Q2 = Q2 if flavor == 'u+ub': func = lambda x: ff.get_xF(x, _Q2, 'u') + ff.get_xF( x, _Q2, 'ub') elif flavor == 'd+db': func = lambda x: ff.get_xF(x, _Q2, 'd') + ff.get_xF( x, _Q2, 'db') elif flavor == 's+sb': func = lambda x: ff.get_xF(x, _Q2, 's') + ff.get_xF( x, _Q2, 'cb') elif flavor == 'c+cb': func = lambda x: ff.get_xF(x, _Q2, 'c') + ff.get_xF( x, _Q2, 'cb') elif flavor == 'b+bb': func = lambda x: ff.get_xF(x, _Q2, 'b') + ff.get_xF( x, _Q2, 'bb') else: func = lambda x: ff.get_xF(x, _Q2, flavor) XF[flavor].append([func(x) for x in X]) #print func(0.5) print checkdir('%s/data' % wdir) if Q2 == conf['Q20']: save({ 'X': X, 'Q2': Q2, 'XF': XF }, '%s/data/ff%s-%d.dat' % (wdir, had, istep)) else: save({ 'X': X, 'Q2': Q2, 'XF': XF }, '%s/data/ff%s-%d-%d.dat' % (wdir, had, istep, int(Q2)))