def gen_jar_file(wdir, kc): print('\ngen jar file using (best cluster) %s\n' % wdir) replicas = core.get_replicas(wdir) conf = {} load_config('%s/input.py' % wdir) istep = core.get_istep() core.mod_conf(istep, replicas[0]) #--set conf as specified in istep cluster, colors, nc, cluster_order = classifier.get_clusters( wdir, istep, kc) best_cluster = cluster_order[0] conf['order'] = replicas[0]['order'][istep] conf['replicas'] = [] cnt = 0 for i in range(len(replicas)): replica = replicas[i] if cluster[i] != best_cluster: continue cnt += 1 conf['replicas'].append(replica['params'][istep]) print('number of replicas = %d' % cnt) checkdir('%s/data' % wdir) save(conf, '%s/data/jar-%d.dat' % (wdir, istep))
def generate_xf(wdir, groups, flavors, Q2=None): load_config('%s/input.py' % wdir) ## setup kinematics xs = np.logspace(-2.1, -0.04, 100) if Q2 == None: Q2 = conf['Q20'] print('\ngenerating pdf-proton from LHAPDF at Q2 = %f' % Q2) ## compute xf for all replicas xfs = {} for group in groups: xfs[group] = lhapdf_xfs(group, xs, flavors, Q2) if group == 'JAM19PDF_proton_nlo': xfs[group]['color'] = 'g' elif group == 'NNPDF31_nlo_as_0118': xfs[group]['color'] = 'm' elif group == 'CJ15nlo': xfs[group]['color'] = 'k' elif group == 'MMHT2014nlo68cl': xfs[group]['color'] = 'Yellow' elif group == 'CSKK_nnlo_EIG': xfs[group]['color'] = 'chocolate' elif group == 'ABMP16_3_nlo': xfs[group]['color'] = 'b' print save({ 'X': xs, 'Q2': Q2, 'XF': xfs, 'groups': groups }, '%s/analysis/qpdlib/lhapdf-%f.dat' % (os.environ['FITPACK'], Q2))
def __init__(self,tag): load_config('data/input-%s.py'%tag) if 'basis' not in conf: conf['basis'] = 'default'
def update_idis_tabs(wdir, tabs, tar): load_config('%s/input.py' % wdir) istep = core.get_istep() data = load('%s/data/predictions-%d-sim.dat' % (wdir, istep)) blist = [] blist.append('thy') blist.append('shift') blist.append('residuals') blist.append('prediction') blist.append('N') blist.append('Shift') blist.append('W2') blist.append('alpha') blist.append('residuals-rep') blist.append('r-residuals') for _ in data['reactions']: for idx in data['reactions'][_]: print('update %s %d%s...' % (_, idx, tar)) tab = data['reactions'][_][idx] for k in blist: try: del tab[k] except: continue tab['value'] = np.mean(tab['prediction-rep'], axis=0) for i in range(len(tab['prediction-rep'])): tab['value%s' % (i + 1)] = tab['prediction-rep'][i] del tab['prediction-rep'] df = pd.DataFrame(tab) df.to_excel('%s/sim/%s%s.xlsx' % (wdir, idx, tar), index=False)
def gen_labels(wdir, kc): print('\ngenerating labels for %s\n' % wdir) load_config('%s/input.py' % wdir) istep = core.get_istep() if istep not in kc.nc: print("istep not in kmeanconf.py") sys.exit() nc = kc.nc[istep] hook = kc.hooks[istep] istep = core.get_istep() labels = {} labels['chi2dof'] = _gen_labels_by_chi2(wdir, istep) labels['echi2dof'] = _gen_labels_by_echi2(wdir, istep) labels['cluster'] = _gen_labels_by_cluster(wdir, istep, nc, hook) clusters = sorted(list(set(labels['cluster']))) chi_2_clusters = [0.0 for _ in clusters] length_clusters = [0.0 for _ in clusters] for i in range(len(labels['cluster'])): for j in range(len(clusters)): if labels['cluster'][i] == clusters[j]: chi_2_clusters[j] += labels['chi2dof'][i] length_clusters[j] += 1.0 labels['cluster_average_chi2'] = np.array(chi_2_clusters) / np.array( length_clusters) checkdir('%s/data' % wdir) save(labels, '%s/data/labels-%d.dat' % (wdir, istep))
def line_figure(self, wdir, Q2, dpi): ## this block may contain something not useful load_config('%s/input.py' % wdir) istep = core.get_istep() # jar = load('%s/data/jar-%d.dat' % (wdir, istep)) labels = load('%s/data/labels-%d.dat' % (wdir, istep)) self.cluster = labels['cluster'] self.cluster_average_chi2 = labels['cluster_average_chi2'] self.best_cluster = np.argmin(self.cluster_average_chi2) nrows, ncols = 1, 1 fig = py.figure(figsize=(ncols * 5.0, nrows * 3.0)) ax = py.subplot(nrows, ncols, 1) ## polarized PDF if Q2 == None: Q2 = conf['Q20'] if Q2 == conf['Q20']: self.xf_data = load('%s/data/ppdf-%d.dat' % (wdir, istep)) else: self.xf_data = load('%s/data/ppdf-%d-%f.dat' % (wdir, istep, Q2)) print '\nplotting polarized PDF line figure from %s at Q2 = %.2f' % ( wdir, Q2) self.plot_lines(ax, 'g', 'r', all_cluster=2) ax.set_ylim(-0.11, 0.3) ax.set_yticks([-0.1, 0.0, 0.1, 0.2]) ax.set_yticklabels([r'$-0.1$', r'$0.0$', r'$0.1$', r'$0.2$']) # ax.text(0.7, 0.8, r'\boldmath$x\Delta g$', color = 'k', transform = axs[3].transAxes, size = 28) ax.title.set_text( r'\boldmath$x\Delta g~\mathrm{at}~Q^2 = %.2f~\mathrm{GeV}^2$' % Q2) legends = ax.legend(frameon=0, loc='best') for line in legends.get_lines(): line.set_linewidth(1.0) ax.semilogx() ax.set_xlim(8e-3, 9e-1) ax.set_xticks([1e-2, 1e-1]) ax.tick_params(axis='both', which='both', right=True, top=True, direction='in', labelsize=20) ax.set_xticklabels([r'', r'']) ax.axhline(0.0, color='k', linestyle='dashdot', linewidth=0.5) ax.set_xticks([0.01, 0.1, 0.5, 0.8]) ax.set_xticklabels([r'$0.01$', r'$0.1$', r'$0.5$', r'$0.8$']) # py.subplots_adjust(left = 0.12, bottom = 0.08, right = 0.99, top = 0.97, wspace = None, hspace = 0.2) py.tight_layout() if Q2 == conf['Q20']: py.savefig('%s/gallery/ppdf-lines-pjet-%d.png' % (wdir, istep), dpi=dpi) else: py.savefig('%s/gallery/ppdf-lines-pjet-%d-%f.png' % (wdir, istep, Q2), dpi=dpi)
def collect_exp_data(wdir): load_config('%s/input.py'%wdir) conf['bootstrap']=False istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=1,parallel=False,datasets=True) obsres={} if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares print('\ncollecting exp data using the setup as in %s\n'%wdir) data={} for reaction in obsres: tabs={} for idx in obsres[reaction].tabs: tabs[idx]={} tabs[idx]['value']=obsres[reaction].tabs[idx]['value'] tabs[idx]['alpha']=obsres[reaction].tabs[idx]['alpha'] data[reaction]=tabs save(data,'%s/expdata.dat'%(wdir)) #--close resman resman.shutdown()
def gen_conf(wdir, tar, est): print('Modifying config with new experimental data file...') load_config('%s/input.py' % wdir) istep = core.get_istep() conf['steps'][istep]['datasets'] = {} conf['steps'][istep]['datasets']['pidis'] = [] conf['datasets']['pidis']['filters'] = [] #--placeholder index idx = 80000 conf['datasets']['pidis']['xlsx'][idx] = './%s/sim/all-%s-%s.xlsx' % ( wdir, tar, est) conf['steps'][istep]['datasets']['pidis'].append(idx) fdir = os.environ['FITPACK'] fn = [fdir + '/database/EIC/expdata/1000.xlsx'] conf['pidis grid'] = {} if tar == 'p': conf['pidis grid']['overwrite'] = True if tar == 'd': conf['pidis grid']['overwrite'] = False conf['pidis grid']['xlsx'] = fn conf['idis grid'] = {} if tar == 'p': conf['idis grid']['overwrite'] = True if tar == 'd': conf['idis grid']['overwrite'] = False conf['idis grid']['xlsx'] = fn return conf
def gen_config(file="", task=0, runid=0, reaction="sidis"): args = Namespace(config=file, task=task, runid=runid, file=file, reaction=reaction) load_config(file) conf["args"] = args
def histogram_figure(self, wdir, Q2, dpi): ## plot histogram for parameters of all flavors load_config('%s/input.py' % wdir) istep = core.get_istep() # jar = load('%s/data/jar-%d.dat' % (wdir, istep)) labels = load('%s/data/labels-%d.dat' % (wdir, istep)) self.cluster = labels['cluster'] self.best_cluster = np.argmin(labels['cluster_average_chi2']) if Q2 == None: Q2 = conf['Q20'] if Q2 == conf['Q20']: self.ppdf_parameter_data = load('%s/data/ppdf-parameters-%d.dat' % (wdir, istep)) else: self.ppdf_parameter_data = load('%s/data/ppdf-parameters-%d-%f.dat' % (wdir, istep, Q2)) shapes = self.ppdf_parameter_data.keys() flavors = self.ppdf_parameter_data[shapes[0]].keys() for _ in sorted(self.ppdf_parameter_data[shapes[0]][flavors[0]]): print '\nplotting PPDF parameter %s histograms from %s' % (_, wdir) if len(flavors) == 8: n_rows, n_columns = 4, 2 figure = py.figure(figsize = (n_columns * 5.0, n_rows * 3.0)) axs = [py.subplot(n_rows, n_columns, count + 1) for count in range(8)] sys.exit('please make a subplot configuration for your flavors') elif len(flavors) == 7: n_rows, n_columns = 4, 2 figure = py.figure(figsize = (n_columns * 5.0, n_rows * 3.0)) axs = [py.subplot(n_rows, n_columns, count + 1) for count in range(8) if count != 7] else: sys.exit('please make a subplot configuration for your flavors') if sorted(flavors) == sorted(['g', 'up', 'ub', 'dp', 'db', 'sp', 'sb']): self.ppdf_histogram(axs[0], 'up', _, r'\boldmath$u_+$') self.ppdf_histogram(axs[1], 'ub', _, r'\boldmath$\overline{u}$') self.ppdf_histogram(axs[2], 'dp', _, r'\boldmath$d_+$') self.ppdf_histogram(axs[3], 'db', _, r'\boldmath$\overline{d}$') self.ppdf_histogram(axs[4], 'sp', _, r'\boldmath$s_+$') self.ppdf_histogram(axs[5], 'sb', _, r'\boldmath$\overline{s}$') self.ppdf_histogram(axs[6], 'g', _, r'\boldmath$g$') for ax in axs: # ax.semilogx() # ax.tick_params(axis = 'both', which = 'both', right = True, top = True, direction = 'in', labelsize = 20) # ax.set_xticklabels([r'', r'']) ax.legend(frameon = 0, loc = 'best', fontsize = 17) axs[0].title.set_text(r'$\mathrm{parameter}~%s~\mathrm{at}~Q^2 = %.2f~\mathrm{GeV^2}$' % (_, Q2)) axs[1].title.set_text(r'$\mathrm{parameter}~%s~\mathrm{at}~Q^2 = %.2f~\mathrm{GeV^2}$' % (_, Q2)) py.tight_layout() if Q2 == conf['Q20']: py.savefig('%s/gallery/ppdf-histogram-pjet-%s-%d.png' % (wdir, _, istep), dpi = dpi) else: py.savefig('%s/gallery/ppdf-histogram-pjet-%s-%d-%f.png' % (wdir, _, istep, Q2), dpi = dpi)
def __init__(self, tag): load_config('data/input-%s.py' % tag) resman = RESMAN(nworkers=1, parallel=False, datasets=False) self.parman = resman.parman self.jar = np.load('data/jar-%s.npy' % tag, allow_pickle=True, encoding='latin1').item() self.parman.order = self.jar['order'] self.par = self.jar['par'] self.nrep = len(self.par)
def make_figure(wdir, task, plot_with_factor=True, only_best_cluster=True, dpi=200): ## attribute 'plot_with_factor' is inherited from jet and not yet used ## it is kept there because someday it might be useful if task == 1: print('\nplotting PJET data from %s' % (wdir)) elif task == 2: print('\nplotting PJET data over theory from %s' % (wdir)) load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) #--set conf as specified in istep predictions = load('%s/data/predictions-%d.dat' % (wdir, istep)) if 'pjet' not in predictions['reactions']: print('PJET is not in data file') return labels = load('%s/data/labels-%d.dat' % (wdir, istep)) cluster = labels['cluster'] cluster_average_chi2 = labels['cluster_average_chi2'] best_cluster = np.argmin(cluster_average_chi2) data = predictions['reactions']['pjet'] for idx in data: predictions = copy.copy(data[idx]['prediction-rep']) del data[idx]['prediction-rep'] del data[idx]['residuals-rep'] if only_best_cluster: best_predictions = [ predictions[i] for i in range(len(predictions)) if cluster[i] == best_cluster ] data[idx]['thy'] = np.mean(best_predictions, axis=0) data[idx]['dthy'] = np.std(best_predictions, axis=0)**0.5 else: all_predictions = [predictions[i] for i in range(len(predictions))] data[idx]['thy'] = np.mean(all_predictions, axis=0) data[idx]['dthy'] = np.std(all_predictions, axis=0)**0.5 if task == 1: plot_data(wdir, data, istep, dpi, plot_with_factor) # for idx in data: # plot_data_separate(wdir, data[idx], istep, idx, dpi, plot_with_factor) elif task == 2: ## this is copied from jet plot and not yet ready plot_data_on_theory(wdir, data, istep, dpi) return
def plot_obs(wdir, kc): print('\nplotting ln data from %s' % (wdir)) load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) #--set conf as specified in istep predictions = load('%s/data/predictions-%d.dat' % (wdir, istep)) data = predictions['reactions']['ln'] plot_H1(wdir, istep, data[1000]) plot_ZEUS(wdir, istep, data[2000])
def plot_obs(wdir, only_best_cluster=True, dpi=200): print('\nplotting dy data from %s' % (wdir)) load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) #--set conf as specified in istep predictions = load('%s/data/predictions-%d.dat' % (wdir, istep)) if 'dy' not in predictions['reactions']: print('DY is not in data file') return labels = load('%s/data/labels-%d.dat' % (wdir, istep)) cluster = labels['cluster'] cluster_average_chi2 = labels['cluster_average_chi2'] best_cluster = np.argmin(cluster_average_chi2) data = predictions['reactions']['dy'] for idx in data: predictions = copy.copy(data[idx]['prediction-rep']) del data[idx]['prediction-rep'] del data[idx]['residuals-rep'] if only_best_cluster: best_predictions = [ predictions[i] for i in range(len(predictions)) if cluster[i] == best_cluster ] data[idx]['thy'] = np.mean(best_predictions, axis=0) data[idx]['dthy'] = np.std(best_predictions, axis=0)**0.5 else: all_predictions = [predictions[i] for i in range(len(predictions))] data[idx]['thy'] = np.mean(all_predictions, axis=0) data[idx]['dthy'] = np.std(all_predictions, axis=0)**0.5 Q2_bins = [] # Q2_bins.append([30, 40]) Q2_bins.append([35, 45]) ## label pp: 37 < Q2 < 42 pd: idem Q2_bins.append([45, 52]) ## label pp: 47 < Q2 < 49 pd: idem Q2_bins.append([52, 60]) ## label pp: 54 < Q2 < 58 pd: 55 < Q2 < 57 Q2_bins.append([60, 68]) ## label pp: 63 < Q2 < 66 pd: 62 < Q2 < 65 Q2_bins.append([68, 75]) ## label pp: 70 < Q2 < 73 pd: 70 < Q2 < 72 Q2_bins.append([100, 140]) ## label pp: 118 < Q2 < 131 pd: 124 < Q2 < 129 Q2_bins.append([140, 160]) ## label pp: 148 < Q2 < 156 pd: 145 < Q2 < 154 Q2_bins.append([160, 280]) ## label pp: 173 < Q2 < 222 pd: 73 < Q2 < 280 plot_ratio(wdir, data, istep, Q2_bins, dpi) return
def gen_conf_idis(wdir, idxs, tar, option=1, fn=None): load_config('%s/input.py' % wdir) istep = core.get_istep() conf['steps'][istep]['datasets']['idis'] = [] conf['datasets']['idis']['filters'] = [] for idx in idxs: conf['datasets']['idis']['xlsx'][idx] = './%s/sim/%d%s.xlsx' % ( wdir, idx, tar) conf['steps'][istep]['datasets']['idis'].append(idx) if option == 1: conf['idis grid'] = 'prediction' elif option == 2: conf['idis grid'] = fn return conf
def gen_samples(wdir,nsamples=100,nsize=10): load_config('%s/input.py'%wdir) conf['bootstrap']=False istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=5,parallel=True,datasets=True) parman=resman.parman order=parman.order obsres={} if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares print('\ngen ml samples using the setup as in %s\n'%wdir) checkdir('%s/mlsamples'%wdir) data=gen_dataframe(parman.order,obsres,nsize) for _ in range(nsamples): lprint('progress: %d/%d'%(_,nsamples)) par=parman.gen_flat(setup=True) res,rres,nres=resman.get_residuals(par) #--fill dataframe i=data['size'] data['params'][i]=par for reaction in obsres: for idx in data['reactions'][reaction]: prediction=copy.copy(obsres[reaction].tabs[idx]['prediction']) data['reactions'][reaction][idx][i]=prediction data['size']+=1 if data['size']==nsize: save(data,'%s/mlsamples/%s'%(wdir,id_generator(12))) data=gen_dataframe(parman.order,obsres,nsize) print #--close resman resman.shutdown()
def main(): parser = argparse.ArgumentParser( description='Add the volume change introduced by affine') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--in-affine-folder', type=str, help='Affine matrix in .txt files') parser.add_argument('--out-folder', type=str, help='Output location for preprocessed images', required=True) parser.add_argument( '--ref-img', type=str, help='Use this average image to impute the missing voxels in targets') parser.add_argument('--data-file-list', type=str, help='Data file list') args = parser.parse_args() config = load_config(args.config) preprocess_obj = JacobianAffineCorrection(config, args.in_folder, args.in_affine_folder, args.out_folder, args.ref_img, args.data_file_list) preprocess_obj.run_parallel()
def main(): parser = argparse.ArgumentParser(description='Run resample on in data folder') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder-1', type=str, help='Folder 1 of input data', required=True) parser.add_argument('--in-folder-2', type=str, help='Folder 2 of input data', required=True) parser.add_argument('--file-list-txt', type=str, help='Help to define the order of the files') parser.add_argument('--out-png-path', type=str, help='Output as png') args = parser.parse_args() config = load_config(args.config) loss = GetLossBetweenFolder(config, args.in_folder_1, args.in_folder_2, args.file_list_txt) loss.print_file_list() loss.run_non_parallel() array_nrmse = loss.get_nrmse() print(array_nrmse, flush=True) plt.figure(figsize=(30, 15)) x = range(20) plt.plot(x, array_nrmse, 'ro-', linewidth=2) plt.xlabel('PCs') plt.title('nrmse diff') print(f'Plot nrmse to {args.out_png_path}') plt.savefig(args.out_png_path, bbox_inches='tight', pad_inches=0)
def main(): parser = argparse.ArgumentParser( description='Run preprocess on in data folder') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--out-folder', type=str, help='Output location for preprocessed images', required=True) parser.add_argument( '--ref-img', type=str, help='Use this average image to impute the missing voxels in targets') parser.add_argument('--data-file-list', type=str, help='Data file list') args = parser.parse_args() config = load_config(args.config) preprocess_obj = CalcJacobian(config, args.in_folder, args.out_folder, args.ref_img, args.data_file_list) preprocess_obj.run_parallel()
def main(): parser = argparse.ArgumentParser( description='Run resample on in data folder') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--ref-img', type=str, help='Reference image to define the space.', required=True) parser.add_argument('--out-folder', type=str, help='Output location for preprocessed images', required=True) args = parser.parse_args() config = load_config(args.config) preprocess_obj = DownSampleNiftyReg(config, args.in_folder, args.out_folder, args.ref_img) preprocess_obj.run_parallel()
def main(): parser = argparse.ArgumentParser(description='ThoraxPCA') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--save-path', type=str, help='Output location for union average', required=True) parser.add_argument('--data-file-list', type=str, help='Data file list') args = parser.parse_args() config = load_config(args.config) average_tool = None if args.data_file_list: data_folder_obj = DataFolder(args.in_folder, args.data_file_list) average_tool = AverageScans(config, in_data_folder_obj=data_folder_obj) else: average_tool = AverageScans(config, args.in_folder) average_tool.get_average_image_union(args.save_path)
def plot_obs(wdir, kc): print('\nplotting dy-pion data from %s' % (wdir)) load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) #--set conf as specified in istep predictions = load('%s/data/predictions-%d.dat' % (wdir, istep)) if 'dy-pion' not in predictions['reactions']: return data = predictions['reactions']['dy-pion'] plot_E615(wdir, istep, data[10001]) plot_NA10(wdir, istep, data[10002], data[10003]) return
def plot_obs(wdir, kc): print('\nplotting DY-pion (qT) from %s' % (wdir)) load_config('%s/input.py' % wdir) istep = core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep, replicas[0]) #--set conf as specified in istep predictions = load('%s/data/predictions-%d.dat' % (wdir, istep)) #print(predictions['reactions'].keys()) if 'pion_qT' not in predictions['reactions']: return data = predictions['reactions']['pion_qT'] plot_E615_Q(wdir, istep, data[1001]) plot_E615_xF(wdir, istep, data[1002]) return
def plot_params(wdir,dist,kc): load_config('%s/input.py'%wdir) istep=core.get_istep() replicas = core.get_replicas(wdir) core.mod_conf(istep,replicas[0]) clusters,colors,nc,cluster_order = classifier.get_clusters(wdir,istep,kc) _order = replicas[0]['order'][istep] #--get correct order from dist order = [] idx = [] for i in range(len(_order)): if _order[i][1] != dist: continue order.append(_order[i][2]) idx.append(i) #--get correct params from dist params = np.zeros((len(order),len(replicas))) for i in range(len(order)): for j in range(len(replicas)): params[i][j] = replicas[j]['params'][istep][idx[i]] #--create plot with enough space for # of parameters nrows, ncols = np.ceil(len(order)/5.0), 5 fig = py.figure(figsize=(ncols*7,nrows*4)) X = np.linspace(1,len(replicas),len(replicas)) #--create plot for i in range(len(order)): ax = py.subplot(nrows,ncols, i+1) ax.set_title('%s'%(order[i]), size=20) #params = [replicas[j]['params'][istep][i] for j in range(len(replicas))] color = [colors[clusters[j]] for j in range(len(replicas))] ax.scatter(X,params[i],color=color) ax.plot(X,np.ones(len(X))*np.average(params[i]),'k--',alpha=0.5) filename='%s/gallery/%s-params.png'%(wdir,dist) checkdir('%s/gallery'%wdir) py.savefig(filename) print 'Saving figure to %s'%filename
def gen_dist_from_fitpack(wdir,dist): print('\ngenerating %s tables for benchmark using %s'%(dist,wdir)) load_config('%s/input.py'%wdir) istep=core.get_istep() core.mod_conf(istep) #--set conf as specified in istep resman=RESMAN(nworkers=1,parallel=False,datasets=False) parman=resman.parman jar=load('%s/data/jar-%d.dat'%(wdir,istep)) replicas=jar['replicas'] X,Q2=gen_grid(dist) qpd=conf[dist] nx=len(X) nQ2=len(Q2) flavs=[-5,-4,-3,-2,-1,1,2,3,4,5,21] fmap={-5:'bb',-4:'cb',-3:'sb',-2:'ub',-1:'db'} fmap.update({5:'b',4:'c',3:'s',2:'u',1:'d'}) fmap.update({21:'g'}) #--gen qpd data per replica checkdir('%s/benchmark'%wdir) cnt=0 for par in replicas: lprint('progress: %d/%d'%(cnt+1,len(replicas))) parman.set_new_params(par) data={} for _ in Q2: data[_]={} for flav in flavs: data[_][flav]=np.array([qpd.get_xF(x,_,fmap[flav]) for x in X]) save(data,'%s/benchmark/%s-%d.dat'%(wdir,dist,cnt)) cnt+=1 print
def print_individual_parameters(working_directory, distribution, norm, index=0): load_config('%s/input.py' % working_directory) istep = core.get_istep() jar = load('%s/data/jar-%d.dat' % (working_directory, istep)) if norm: for i in range(len(jar['order'])): if jar['order'][i][0] == 2: print jar['order'][i][1].ljust(8), '%s' % str( jar['order'][i][2]).ljust( 10), ':', jar['replicas'][index][i] else: for i in range(len(jar['order'])): if (jar['order'][i][0] == 1) and (jar['order'][i][1] == distribution): print jar['order'][i][2].ljust( 13), ':', jar['replicas'][index][i]
def main(): parser = argparse.ArgumentParser( description='Run preprocess on in data folder') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-ori-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--in-jac-folder', type=str, required=True) parser.add_argument('--mask-img-path', type=str, required=True) parser.add_argument('--in-file-list', type=str, help='file list to use in pca') parser.add_argument('--out-pc-folder', type=str, help='Output location of principle images') parser.add_argument('--n-components', type=int, help='Number of pc', required=True) parser.add_argument('--batch-size', type=int, help='Batch size') parser.add_argument('--save-pca-result-path', type=str, help='Save the result to the given location') parser.add_argument('--load-pca-obj-file-path', type=str) args = parser.parse_args() config = load_config(args.config) scan_folder_reader = ScanFolderConcatBatchReaderWithMask( args.mask_img_path, config, args.in_ori_folder, args.in_jac_folder, args.batch_size, file_list_txt=args.in_file_list) pca_nii_3d = PCA_NII_3D_Batch(scan_folder_reader, None, args.n_components) if args.load_pca_obj_file_path: pca_nii_3d.load_pca(args.load_pca_obj_file_path) else: pca_nii_3d.run_pca() if args.save_pca_result_path: pca_nii_3d.save_pca_obj(args.save_pca_result_path) if args.out_pc_folder: mkdir_p(args.out_pc_folder) pca_nii_3d.write_pc(args.out_pc_folder)
def load_model(config_path, weights_path): conf = load_config(config_path) model = zoo.__dict__[conf['network']](seg_classes=3, backbone_arch=conf['encoder']) model = torch.nn.DataParallel(model).cuda() print("=> loading checkpoint '{}'".format(weights_path)) checkpoint = torch.load(weights_path, map_location="cpu") print("best_dice", checkpoint['dice_best']) print("f_score_best", checkpoint['f_score_best']) print("epoch", checkpoint['epoch']) model.load_state_dict(checkpoint['state_dict']) model.eval() return model
def gen_priors(wdir, kc, nsamples): load_config('%s/input.py' % wdir) istep = core.get_istep() #cluster,colors,nc,cluster_order = core.get_clusters(wdir,istep) cluster, colors, nc, cluster_order = classifier.get_clusters( wdir, istep, kc) best_cluster = cluster_order[0] #--get data replicas = core.get_replicas(wdir) step = conf['steps'][istep] samples = [] for i in range(len(replicas)): if cluster[i] != best_cluster: continue replica = replicas[i] order = replica['order'][istep] params = replica['params'][istep] samples.append(params) samples = np.array(samples) pmin = [np.amin(p) for p in samples.T] pmax = [np.amax(p) for p in samples.T] new = [] for i in range(len(pmin)): new.append(np.random.uniform(low=pmin[i], high=pmax[i], size=nsamples)) new = np.transpose(new) checkdir('%s/msr-opt-priors' % wdir) replica = replicas[0] #--template replica for i in range(nsamples): lprint('%d/%d' % (i + 1, nsamples)) replica['params'][istep] = new[i] fname = '%s/msr-opt-priors/%s.msr' % (wdir, id_generator(12)) save(replica, fname) print
def main(): parser = argparse.ArgumentParser( description='Run preprocess on in data folder') parser.add_argument('--config', type=str, help='Path to the YAML config file', required=True) parser.add_argument('--in-folder', type=str, help='Folder of input data', required=True) parser.add_argument('--ref-img', type=str, help='Use the average as reference image', required=True) parser.add_argument('--out-pc-folder', type=str, help='Output location of principle images') parser.add_argument('--out-mean-img', type=str, help='Output the mean of pca', required=True) parser.add_argument('--n-components', type=int, help='Number of pc', required=True) parser.add_argument('--save-pca-result-path', type=str, help='Save the result to the given location') args = parser.parse_args() config = load_config(args.config) scan_folder_reader = ScanFolderFlatReader(config, args.in_folder, args.ref_img) scan_folder_reader.read_data() pca_nii_3d = PCA_NII_3D(scan_folder_reader, args.ref_img, args.n_components) pca_nii_3d.run_pca() if args.save_pca_result_path: pca_nii_3d.save_pca_obj(args.save_pca_result_path) if args.out_pc_folder: mkdir_p(args.out_pc_folder) pca_nii_3d.write_pc(args.out_pc_folder) if args.out_mean_img: pca_nii_3d.write_mean(args.out_mean_img)