def gen_jar_file(wdir, kc):
print('\ngen jar file using (best cluster) %s\n' % wdir)
replicas = core.get_replicas(wdir)
conf = {}
load_config('%s/input.py' % wdir)
istep = core.get_istep()
core.mod_conf(istep, replicas[0]) #--set conf as specified in istep
cluster, colors, nc, cluster_order = classifier.get_clusters(
wdir, istep, kc)
best_cluster = cluster_order[0]
conf['order'] = replicas[0]['order'][istep]
conf['replicas'] = []
cnt = 0
for i in range(len(replicas)):
replica = replicas[i]
if cluster[i] != best_cluster: continue
cnt += 1
conf['replicas'].append(replica['params'][istep])
print('number of replicas = %d' % cnt)
checkdir('%s/data' % wdir)
save(conf, '%s/data/jar-%d.dat' % (wdir, istep))
def generate_xf(wdir, groups, flavors, Q2=None):
load_config('%s/input.py' % wdir)
## setup kinematics
xs = np.logspace(-2.1, -0.04, 100)
if Q2 == None: Q2 = conf['Q20']
print('\ngenerating pdf-proton from LHAPDF at Q2 = %f' % Q2)
## compute xf for all replicas
xfs = {}
for group in groups:
xfs[group] = lhapdf_xfs(group, xs, flavors, Q2)
if group == 'JAM19PDF_proton_nlo':
xfs[group]['color'] = 'g'
elif group == 'NNPDF31_nlo_as_0118':
xfs[group]['color'] = 'm'
elif group == 'CJ15nlo':
xfs[group]['color'] = 'k'
elif group == 'MMHT2014nlo68cl':
xfs[group]['color'] = 'Yellow'
elif group == 'CSKK_nnlo_EIG':
xfs[group]['color'] = 'chocolate'
elif group == 'ABMP16_3_nlo':
xfs[group]['color'] = 'b'
print
save({
'X': xs,
'Q2': Q2,
'XF': xfs,
'groups': groups
}, '%s/analysis/qpdlib/lhapdf-%f.dat' % (os.environ['FITPACK'], Q2))
def __init__(self,tag):
load_config('data/input-%s.py'%tag)
if 'basis' not in conf:
conf['basis'] = 'default'
def update_idis_tabs(wdir, tabs, tar):
load_config('%s/input.py' % wdir)
istep = core.get_istep()
data = load('%s/data/predictions-%d-sim.dat' % (wdir, istep))
blist = []
blist.append('thy')
blist.append('shift')
blist.append('residuals')
blist.append('prediction')
blist.append('N')
blist.append('Shift')
blist.append('W2')
blist.append('alpha')
blist.append('residuals-rep')
blist.append('r-residuals')
for _ in data['reactions']:
for idx in data['reactions'][_]:
print('update %s %d%s...' % (_, idx, tar))
tab = data['reactions'][_][idx]
for k in blist:
try:
del tab[k]
except:
continue
tab['value'] = np.mean(tab['prediction-rep'], axis=0)
for i in range(len(tab['prediction-rep'])):
tab['value%s' % (i + 1)] = tab['prediction-rep'][i]
del tab['prediction-rep']
df = pd.DataFrame(tab)
df.to_excel('%s/sim/%s%s.xlsx' % (wdir, idx, tar), index=False)
def gen_labels(wdir, kc):
print('\ngenerating labels for %s\n' % wdir)
load_config('%s/input.py' % wdir)
istep = core.get_istep()
if istep not in kc.nc:
print("istep not in kmeanconf.py")
sys.exit()
nc = kc.nc[istep]
hook = kc.hooks[istep]
istep = core.get_istep()
labels = {}
labels['chi2dof'] = _gen_labels_by_chi2(wdir, istep)
labels['echi2dof'] = _gen_labels_by_echi2(wdir, istep)
labels['cluster'] = _gen_labels_by_cluster(wdir, istep, nc, hook)
clusters = sorted(list(set(labels['cluster'])))
chi_2_clusters = [0.0 for _ in clusters]
length_clusters = [0.0 for _ in clusters]
for i in range(len(labels['cluster'])):
for j in range(len(clusters)):
if labels['cluster'][i] == clusters[j]:
chi_2_clusters[j] += labels['chi2dof'][i]
length_clusters[j] += 1.0
labels['cluster_average_chi2'] = np.array(chi_2_clusters) / np.array(
length_clusters)
checkdir('%s/data' % wdir)
save(labels, '%s/data/labels-%d.dat' % (wdir, istep))
def line_figure(self, wdir, Q2, dpi):
## this block may contain something not useful
load_config('%s/input.py' % wdir)
istep = core.get_istep()
# jar = load('%s/data/jar-%d.dat' % (wdir, istep))
labels = load('%s/data/labels-%d.dat' % (wdir, istep))
self.cluster = labels['cluster']
self.cluster_average_chi2 = labels['cluster_average_chi2']
self.best_cluster = np.argmin(self.cluster_average_chi2)
nrows, ncols = 1, 1
fig = py.figure(figsize=(ncols * 5.0, nrows * 3.0))
ax = py.subplot(nrows, ncols, 1)
## polarized PDF
if Q2 == None: Q2 = conf['Q20']
if Q2 == conf['Q20']:
self.xf_data = load('%s/data/ppdf-%d.dat' % (wdir, istep))
else:
self.xf_data = load('%s/data/ppdf-%d-%f.dat' % (wdir, istep, Q2))
print '\nplotting polarized PDF line figure from %s at Q2 = %.2f' % (
wdir, Q2)
self.plot_lines(ax, 'g', 'r', all_cluster=2)
ax.set_ylim(-0.11, 0.3)
ax.set_yticks([-0.1, 0.0, 0.1, 0.2])
ax.set_yticklabels([r'$-0.1$', r'$0.0$', r'$0.1$', r'$0.2$'])
# ax.text(0.7, 0.8, r'\boldmath$x\Delta g$', color = 'k', transform = axs[3].transAxes, size = 28)
ax.title.set_text(
r'\boldmath$x\Delta g~\mathrm{at}~Q^2 = %.2f~\mathrm{GeV}^2$' % Q2)
legends = ax.legend(frameon=0, loc='best')
for line in legends.get_lines():
line.set_linewidth(1.0)
ax.semilogx()
ax.set_xlim(8e-3, 9e-1)
ax.set_xticks([1e-2, 1e-1])
ax.tick_params(axis='both',
which='both',
right=True,
top=True,
direction='in',
labelsize=20)
ax.set_xticklabels([r'', r''])
ax.axhline(0.0, color='k', linestyle='dashdot', linewidth=0.5)
ax.set_xticks([0.01, 0.1, 0.5, 0.8])
ax.set_xticklabels([r'$0.01$', r'$0.1$', r'$0.5$', r'$0.8$'])
# py.subplots_adjust(left = 0.12, bottom = 0.08, right = 0.99, top = 0.97, wspace = None, hspace = 0.2)
py.tight_layout()
if Q2 == conf['Q20']:
py.savefig('%s/gallery/ppdf-lines-pjet-%d.png' % (wdir, istep),
dpi=dpi)
else:
py.savefig('%s/gallery/ppdf-lines-pjet-%d-%f.png' %
(wdir, istep, Q2),
dpi=dpi)
def collect_exp_data(wdir):
load_config('%s/input.py'%wdir)
conf['bootstrap']=False
istep=core.get_istep()
core.mod_conf(istep) #--set conf as specified in istep
resman=RESMAN(nworkers=1,parallel=False,datasets=True)
obsres={}
if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres
if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres
if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres
if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres
if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres
if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares
print('\ncollecting exp data using the setup as in %s\n'%wdir)
data={}
for reaction in obsres:
tabs={}
for idx in obsres[reaction].tabs:
tabs[idx]={}
tabs[idx]['value']=obsres[reaction].tabs[idx]['value']
tabs[idx]['alpha']=obsres[reaction].tabs[idx]['alpha']
data[reaction]=tabs
save(data,'%s/expdata.dat'%(wdir))
#--close resman
resman.shutdown()
def gen_conf(wdir, tar, est):
print('Modifying config with new experimental data file...')
load_config('%s/input.py' % wdir)
istep = core.get_istep()
conf['steps'][istep]['datasets'] = {}
conf['steps'][istep]['datasets']['pidis'] = []
conf['datasets']['pidis']['filters'] = []
#--placeholder index
idx = 80000
conf['datasets']['pidis']['xlsx'][idx] = './%s/sim/all-%s-%s.xlsx' % (
wdir, tar, est)
conf['steps'][istep]['datasets']['pidis'].append(idx)
fdir = os.environ['FITPACK']
fn = [fdir + '/database/EIC/expdata/1000.xlsx']
conf['pidis grid'] = {}
if tar == 'p': conf['pidis grid']['overwrite'] = True
if tar == 'd': conf['pidis grid']['overwrite'] = False
conf['pidis grid']['xlsx'] = fn
conf['idis grid'] = {}
if tar == 'p': conf['idis grid']['overwrite'] = True
if tar == 'd': conf['idis grid']['overwrite'] = False
conf['idis grid']['xlsx'] = fn
return conf
def gen_config(file="", task=0, runid=0, reaction="sidis"):
args = Namespace(config=file,
task=task,
runid=runid,
file=file,
reaction=reaction)
load_config(file)
conf["args"] = args
def histogram_figure(self, wdir, Q2, dpi):
## plot histogram for parameters of all flavors
load_config('%s/input.py' % wdir)
istep = core.get_istep()
# jar = load('%s/data/jar-%d.dat' % (wdir, istep))
labels = load('%s/data/labels-%d.dat' % (wdir, istep))
self.cluster = labels['cluster']
self.best_cluster = np.argmin(labels['cluster_average_chi2'])
if Q2 == None: Q2 = conf['Q20']
if Q2 == conf['Q20']:
self.ppdf_parameter_data = load('%s/data/ppdf-parameters-%d.dat' % (wdir, istep))
else:
self.ppdf_parameter_data = load('%s/data/ppdf-parameters-%d-%f.dat' % (wdir, istep, Q2))
shapes = self.ppdf_parameter_data.keys()
flavors = self.ppdf_parameter_data[shapes[0]].keys()
for _ in sorted(self.ppdf_parameter_data[shapes[0]][flavors[0]]):
print '\nplotting PPDF parameter %s histograms from %s' % (_, wdir)
if len(flavors) == 8:
n_rows, n_columns = 4, 2
figure = py.figure(figsize = (n_columns * 5.0, n_rows * 3.0))
axs = [py.subplot(n_rows, n_columns, count + 1) for count in range(8)]
sys.exit('please make a subplot configuration for your flavors')
elif len(flavors) == 7:
n_rows, n_columns = 4, 2
figure = py.figure(figsize = (n_columns * 5.0, n_rows * 3.0))
axs = [py.subplot(n_rows, n_columns, count + 1) for count in range(8) if count != 7]
else:
sys.exit('please make a subplot configuration for your flavors')
if sorted(flavors) == sorted(['g', 'up', 'ub', 'dp', 'db', 'sp', 'sb']):
self.ppdf_histogram(axs[0], 'up', _, r'\boldmath$u_+$')
self.ppdf_histogram(axs[1], 'ub', _, r'\boldmath$\overline{u}$')
self.ppdf_histogram(axs[2], 'dp', _, r'\boldmath$d_+$')
self.ppdf_histogram(axs[3], 'db', _, r'\boldmath$\overline{d}$')
self.ppdf_histogram(axs[4], 'sp', _, r'\boldmath$s_+$')
self.ppdf_histogram(axs[5], 'sb', _, r'\boldmath$\overline{s}$')
self.ppdf_histogram(axs[6], 'g', _, r'\boldmath$g$')
for ax in axs:
# ax.semilogx()
# ax.tick_params(axis = 'both', which = 'both', right = True, top = True, direction = 'in', labelsize = 20)
# ax.set_xticklabels([r'', r''])
ax.legend(frameon = 0, loc = 'best', fontsize = 17)
axs[0].title.set_text(r'$\mathrm{parameter}~%s~\mathrm{at}~Q^2 = %.2f~\mathrm{GeV^2}$' % (_, Q2))
axs[1].title.set_text(r'$\mathrm{parameter}~%s~\mathrm{at}~Q^2 = %.2f~\mathrm{GeV^2}$' % (_, Q2))
py.tight_layout()
if Q2 == conf['Q20']:
py.savefig('%s/gallery/ppdf-histogram-pjet-%s-%d.png' % (wdir, _, istep), dpi = dpi)
else:
py.savefig('%s/gallery/ppdf-histogram-pjet-%s-%d-%f.png' % (wdir, _, istep, Q2), dpi = dpi)
def __init__(self, tag):
load_config('data/input-%s.py' % tag)
resman = RESMAN(nworkers=1, parallel=False, datasets=False)
self.parman = resman.parman
self.jar = np.load('data/jar-%s.npy' % tag,
allow_pickle=True,
encoding='latin1').item()
self.parman.order = self.jar['order']
self.par = self.jar['par']
self.nrep = len(self.par)
def make_figure(wdir,
task,
plot_with_factor=True,
only_best_cluster=True,
dpi=200):
## attribute 'plot_with_factor' is inherited from jet and not yet used
## it is kept there because someday it might be useful
if task == 1:
print('\nplotting PJET data from %s' % (wdir))
elif task == 2:
print('\nplotting PJET data over theory from %s' % (wdir))
load_config('%s/input.py' % wdir)
istep = core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep, replicas[0]) #--set conf as specified in istep
predictions = load('%s/data/predictions-%d.dat' % (wdir, istep))
if 'pjet' not in predictions['reactions']:
print('PJET is not in data file')
return
labels = load('%s/data/labels-%d.dat' % (wdir, istep))
cluster = labels['cluster']
cluster_average_chi2 = labels['cluster_average_chi2']
best_cluster = np.argmin(cluster_average_chi2)
data = predictions['reactions']['pjet']
for idx in data:
predictions = copy.copy(data[idx]['prediction-rep'])
del data[idx]['prediction-rep']
del data[idx]['residuals-rep']
if only_best_cluster:
best_predictions = [
predictions[i] for i in range(len(predictions))
if cluster[i] == best_cluster
]
data[idx]['thy'] = np.mean(best_predictions, axis=0)
data[idx]['dthy'] = np.std(best_predictions, axis=0)**0.5
else:
all_predictions = [predictions[i] for i in range(len(predictions))]
data[idx]['thy'] = np.mean(all_predictions, axis=0)
data[idx]['dthy'] = np.std(all_predictions, axis=0)**0.5
if task == 1:
plot_data(wdir, data, istep, dpi, plot_with_factor)
# for idx in data:
# plot_data_separate(wdir, data[idx], istep, idx, dpi, plot_with_factor)
elif task == 2:
## this is copied from jet plot and not yet ready
plot_data_on_theory(wdir, data, istep, dpi)
return
def plot_obs(wdir, kc):
print('\nplotting ln data from %s' % (wdir))
load_config('%s/input.py' % wdir)
istep = core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep, replicas[0]) #--set conf as specified in istep
predictions = load('%s/data/predictions-%d.dat' % (wdir, istep))
data = predictions['reactions']['ln']
plot_H1(wdir, istep, data[1000])
plot_ZEUS(wdir, istep, data[2000])
def plot_obs(wdir, only_best_cluster=True, dpi=200):
print('\nplotting dy data from %s' % (wdir))
load_config('%s/input.py' % wdir)
istep = core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep, replicas[0]) #--set conf as specified in istep
predictions = load('%s/data/predictions-%d.dat' % (wdir, istep))
if 'dy' not in predictions['reactions']:
print('DY is not in data file')
return
labels = load('%s/data/labels-%d.dat' % (wdir, istep))
cluster = labels['cluster']
cluster_average_chi2 = labels['cluster_average_chi2']
best_cluster = np.argmin(cluster_average_chi2)
data = predictions['reactions']['dy']
for idx in data:
predictions = copy.copy(data[idx]['prediction-rep'])
del data[idx]['prediction-rep']
del data[idx]['residuals-rep']
if only_best_cluster:
best_predictions = [
predictions[i] for i in range(len(predictions))
if cluster[i] == best_cluster
]
data[idx]['thy'] = np.mean(best_predictions, axis=0)
data[idx]['dthy'] = np.std(best_predictions, axis=0)**0.5
else:
all_predictions = [predictions[i] for i in range(len(predictions))]
data[idx]['thy'] = np.mean(all_predictions, axis=0)
data[idx]['dthy'] = np.std(all_predictions, axis=0)**0.5
Q2_bins = []
# Q2_bins.append([30, 40])
Q2_bins.append([35, 45]) ## label pp: 37 < Q2 < 42 pd: idem
Q2_bins.append([45, 52]) ## label pp: 47 < Q2 < 49 pd: idem
Q2_bins.append([52, 60]) ## label pp: 54 < Q2 < 58 pd: 55 < Q2 < 57
Q2_bins.append([60, 68]) ## label pp: 63 < Q2 < 66 pd: 62 < Q2 < 65
Q2_bins.append([68, 75]) ## label pp: 70 < Q2 < 73 pd: 70 < Q2 < 72
Q2_bins.append([100, 140]) ## label pp: 118 < Q2 < 131 pd: 124 < Q2 < 129
Q2_bins.append([140, 160]) ## label pp: 148 < Q2 < 156 pd: 145 < Q2 < 154
Q2_bins.append([160, 280]) ## label pp: 173 < Q2 < 222 pd: 73 < Q2 < 280
plot_ratio(wdir, data, istep, Q2_bins, dpi)
return
def gen_conf_idis(wdir, idxs, tar, option=1, fn=None):
load_config('%s/input.py' % wdir)
istep = core.get_istep()
conf['steps'][istep]['datasets']['idis'] = []
conf['datasets']['idis']['filters'] = []
for idx in idxs:
conf['datasets']['idis']['xlsx'][idx] = './%s/sim/%d%s.xlsx' % (
wdir, idx, tar)
conf['steps'][istep]['datasets']['idis'].append(idx)
if option == 1: conf['idis grid'] = 'prediction'
elif option == 2: conf['idis grid'] = fn
return conf
def gen_samples(wdir,nsamples=100,nsize=10):
load_config('%s/input.py'%wdir)
conf['bootstrap']=False
istep=core.get_istep()
core.mod_conf(istep) #--set conf as specified in istep
resman=RESMAN(nworkers=5,parallel=True,datasets=True)
parman=resman.parman
order=parman.order
obsres={}
if 'idis' in conf['datasets'] : obsres['idis'] = resman.idisres
if 'pidis' in conf['datasets'] : obsres['pidis'] = resman.pidisres
if 'sidis' in conf['datasets'] : obsres['sidis'] = resman.sidisres
if 'psidis' in conf['datasets'] : obsres['psidis'] = resman.psidisres
if 'dy' in conf['datasets'] : obsres['dy'] = resman.dyres
if 'sia' in conf['datasets'] : obsres['sia'] = resman.siares
print('\ngen ml samples using the setup as in %s\n'%wdir)
checkdir('%s/mlsamples'%wdir)
data=gen_dataframe(parman.order,obsres,nsize)
for _ in range(nsamples):
lprint('progress: %d/%d'%(_,nsamples))
par=parman.gen_flat(setup=True)
res,rres,nres=resman.get_residuals(par)
#--fill dataframe
i=data['size']
data['params'][i]=par
for reaction in obsres:
for idx in data['reactions'][reaction]:
prediction=copy.copy(obsres[reaction].tabs[idx]['prediction'])
data['reactions'][reaction][idx][i]=prediction
data['size']+=1
if data['size']==nsize:
save(data,'%s/mlsamples/%s'%(wdir,id_generator(12)))
data=gen_dataframe(parman.order,obsres,nsize)
print
#--close resman
resman.shutdown()
def main():
parser = argparse.ArgumentParser(
description='Add the volume change introduced by affine')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--in-affine-folder',
type=str,
help='Affine matrix in .txt files')
parser.add_argument('--out-folder',
type=str,
help='Output location for preprocessed images',
required=True)
parser.add_argument(
'--ref-img',
type=str,
help='Use this average image to impute the missing voxels in targets')
parser.add_argument('--data-file-list', type=str, help='Data file list')
args = parser.parse_args()
config = load_config(args.config)
preprocess_obj = JacobianAffineCorrection(config, args.in_folder,
args.in_affine_folder,
args.out_folder, args.ref_img,
args.data_file_list)
preprocess_obj.run_parallel()
def main():
parser = argparse.ArgumentParser(description='Run resample on in data folder')
parser.add_argument('--config', type=str,
help='Path to the YAML config file', required=True)
parser.add_argument('--in-folder-1', type=str,
help='Folder 1 of input data', required=True)
parser.add_argument('--in-folder-2', type=str,
help='Folder 2 of input data', required=True)
parser.add_argument('--file-list-txt', type=str,
help='Help to define the order of the files')
parser.add_argument('--out-png-path', type=str,
help='Output as png')
args = parser.parse_args()
config = load_config(args.config)
loss = GetLossBetweenFolder(config, args.in_folder_1, args.in_folder_2, args.file_list_txt)
loss.print_file_list()
loss.run_non_parallel()
array_nrmse = loss.get_nrmse()
print(array_nrmse, flush=True)
plt.figure(figsize=(30, 15))
x = range(20)
plt.plot(x, array_nrmse, 'ro-', linewidth=2)
plt.xlabel('PCs')
plt.title('nrmse diff')
print(f'Plot nrmse to {args.out_png_path}')
plt.savefig(args.out_png_path, bbox_inches='tight', pad_inches=0)
def main():
parser = argparse.ArgumentParser(
description='Run preprocess on in data folder')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--out-folder',
type=str,
help='Output location for preprocessed images',
required=True)
parser.add_argument(
'--ref-img',
type=str,
help='Use this average image to impute the missing voxels in targets')
parser.add_argument('--data-file-list', type=str, help='Data file list')
args = parser.parse_args()
config = load_config(args.config)
preprocess_obj = CalcJacobian(config, args.in_folder, args.out_folder,
args.ref_img, args.data_file_list)
preprocess_obj.run_parallel()
def main():
parser = argparse.ArgumentParser(
description='Run resample on in data folder')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--ref-img',
type=str,
help='Reference image to define the space.',
required=True)
parser.add_argument('--out-folder',
type=str,
help='Output location for preprocessed images',
required=True)
args = parser.parse_args()
config = load_config(args.config)
preprocess_obj = DownSampleNiftyReg(config, args.in_folder,
args.out_folder, args.ref_img)
preprocess_obj.run_parallel()
def main():
parser = argparse.ArgumentParser(description='ThoraxPCA')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--save-path',
type=str,
help='Output location for union average',
required=True)
parser.add_argument('--data-file-list', type=str, help='Data file list')
args = parser.parse_args()
config = load_config(args.config)
average_tool = None
if args.data_file_list:
data_folder_obj = DataFolder(args.in_folder, args.data_file_list)
average_tool = AverageScans(config, in_data_folder_obj=data_folder_obj)
else:
average_tool = AverageScans(config, args.in_folder)
average_tool.get_average_image_union(args.save_path)
def plot_obs(wdir, kc):
print('\nplotting dy-pion data from %s' % (wdir))
load_config('%s/input.py' % wdir)
istep = core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep, replicas[0]) #--set conf as specified in istep
predictions = load('%s/data/predictions-%d.dat' % (wdir, istep))
if 'dy-pion' not in predictions['reactions']: return
data = predictions['reactions']['dy-pion']
plot_E615(wdir, istep, data[10001])
plot_NA10(wdir, istep, data[10002], data[10003])
return
def plot_obs(wdir, kc):
print('\nplotting DY-pion (qT) from %s' % (wdir))
load_config('%s/input.py' % wdir)
istep = core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep, replicas[0]) #--set conf as specified in istep
predictions = load('%s/data/predictions-%d.dat' % (wdir, istep))
#print(predictions['reactions'].keys())
if 'pion_qT' not in predictions['reactions']: return
data = predictions['reactions']['pion_qT']
plot_E615_Q(wdir, istep, data[1001])
plot_E615_xF(wdir, istep, data[1002])
return
def plot_params(wdir,dist,kc):
load_config('%s/input.py'%wdir)
istep=core.get_istep()
replicas = core.get_replicas(wdir)
core.mod_conf(istep,replicas[0])
clusters,colors,nc,cluster_order = classifier.get_clusters(wdir,istep,kc)
_order = replicas[0]['order'][istep]
#--get correct order from dist
order = []
idx = []
for i in range(len(_order)):
if _order[i][1] != dist: continue
order.append(_order[i][2])
idx.append(i)
#--get correct params from dist
params = np.zeros((len(order),len(replicas)))
for i in range(len(order)):
for j in range(len(replicas)):
params[i][j] = replicas[j]['params'][istep][idx[i]]
#--create plot with enough space for # of parameters
nrows, ncols = np.ceil(len(order)/5.0), 5
fig = py.figure(figsize=(ncols*7,nrows*4))
X = np.linspace(1,len(replicas),len(replicas))
#--create plot
for i in range(len(order)):
ax = py.subplot(nrows,ncols, i+1)
ax.set_title('%s'%(order[i]), size=20)
#params = [replicas[j]['params'][istep][i] for j in range(len(replicas))]
color = [colors[clusters[j]] for j in range(len(replicas))]
ax.scatter(X,params[i],color=color)
ax.plot(X,np.ones(len(X))*np.average(params[i]),'k--',alpha=0.5)
filename='%s/gallery/%s-params.png'%(wdir,dist)
checkdir('%s/gallery'%wdir)
py.savefig(filename)
print 'Saving figure to %s'%filename
def gen_dist_from_fitpack(wdir,dist):
print('\ngenerating %s tables for benchmark using %s'%(dist,wdir))
load_config('%s/input.py'%wdir)
istep=core.get_istep()
core.mod_conf(istep) #--set conf as specified in istep
resman=RESMAN(nworkers=1,parallel=False,datasets=False)
parman=resman.parman
jar=load('%s/data/jar-%d.dat'%(wdir,istep))
replicas=jar['replicas']
X,Q2=gen_grid(dist)
qpd=conf[dist]
nx=len(X)
nQ2=len(Q2)
flavs=[-5,-4,-3,-2,-1,1,2,3,4,5,21]
fmap={-5:'bb',-4:'cb',-3:'sb',-2:'ub',-1:'db'}
fmap.update({5:'b',4:'c',3:'s',2:'u',1:'d'})
fmap.update({21:'g'})
#--gen qpd data per replica
checkdir('%s/benchmark'%wdir)
cnt=0
for par in replicas:
lprint('progress: %d/%d'%(cnt+1,len(replicas)))
parman.set_new_params(par)
data={}
for _ in Q2:
data[_]={}
for flav in flavs:
data[_][flav]=np.array([qpd.get_xF(x,_,fmap[flav]) for x in X])
save(data,'%s/benchmark/%s-%d.dat'%(wdir,dist,cnt))
cnt+=1
print
def print_individual_parameters(working_directory,
distribution,
norm,
index=0):
load_config('%s/input.py' % working_directory)
istep = core.get_istep()
jar = load('%s/data/jar-%d.dat' % (working_directory, istep))
if norm:
for i in range(len(jar['order'])):
if jar['order'][i][0] == 2:
print jar['order'][i][1].ljust(8), '%s' % str(
jar['order'][i][2]).ljust(
10), ':', jar['replicas'][index][i]
else:
for i in range(len(jar['order'])):
if (jar['order'][i][0] == 1) and (jar['order'][i][1]
== distribution):
print jar['order'][i][2].ljust(
13), ':', jar['replicas'][index][i]
def main():
parser = argparse.ArgumentParser(
description='Run preprocess on in data folder')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-ori-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--in-jac-folder', type=str, required=True)
parser.add_argument('--mask-img-path', type=str, required=True)
parser.add_argument('--in-file-list',
type=str,
help='file list to use in pca')
parser.add_argument('--out-pc-folder',
type=str,
help='Output location of principle images')
parser.add_argument('--n-components',
type=int,
help='Number of pc',
required=True)
parser.add_argument('--batch-size', type=int, help='Batch size')
parser.add_argument('--save-pca-result-path',
type=str,
help='Save the result to the given location')
parser.add_argument('--load-pca-obj-file-path', type=str)
args = parser.parse_args()
config = load_config(args.config)
scan_folder_reader = ScanFolderConcatBatchReaderWithMask(
args.mask_img_path,
config,
args.in_ori_folder,
args.in_jac_folder,
args.batch_size,
file_list_txt=args.in_file_list)
pca_nii_3d = PCA_NII_3D_Batch(scan_folder_reader, None, args.n_components)
if args.load_pca_obj_file_path:
pca_nii_3d.load_pca(args.load_pca_obj_file_path)
else:
pca_nii_3d.run_pca()
if args.save_pca_result_path:
pca_nii_3d.save_pca_obj(args.save_pca_result_path)
if args.out_pc_folder:
mkdir_p(args.out_pc_folder)
pca_nii_3d.write_pc(args.out_pc_folder)
def load_model(config_path, weights_path):
conf = load_config(config_path)
model = zoo.__dict__[conf['network']](seg_classes=3, backbone_arch=conf['encoder'])
model = torch.nn.DataParallel(model).cuda()
print("=> loading checkpoint '{}'".format(weights_path))
checkpoint = torch.load(weights_path, map_location="cpu")
print("best_dice", checkpoint['dice_best'])
print("f_score_best", checkpoint['f_score_best'])
print("epoch", checkpoint['epoch'])
model.load_state_dict(checkpoint['state_dict'])
model.eval()
return model
def gen_priors(wdir, kc, nsamples):
load_config('%s/input.py' % wdir)
istep = core.get_istep()
#cluster,colors,nc,cluster_order = core.get_clusters(wdir,istep)
cluster, colors, nc, cluster_order = classifier.get_clusters(
wdir, istep, kc)
best_cluster = cluster_order[0]
#--get data
replicas = core.get_replicas(wdir)
step = conf['steps'][istep]
samples = []
for i in range(len(replicas)):
if cluster[i] != best_cluster: continue
replica = replicas[i]
order = replica['order'][istep]
params = replica['params'][istep]
samples.append(params)
samples = np.array(samples)
pmin = [np.amin(p) for p in samples.T]
pmax = [np.amax(p) for p in samples.T]
new = []
for i in range(len(pmin)):
new.append(np.random.uniform(low=pmin[i], high=pmax[i], size=nsamples))
new = np.transpose(new)
checkdir('%s/msr-opt-priors' % wdir)
replica = replicas[0] #--template replica
for i in range(nsamples):
lprint('%d/%d' % (i + 1, nsamples))
replica['params'][istep] = new[i]
fname = '%s/msr-opt-priors/%s.msr' % (wdir, id_generator(12))
save(replica, fname)
print
def main():
parser = argparse.ArgumentParser(
description='Run preprocess on in data folder')
parser.add_argument('--config',
type=str,
help='Path to the YAML config file',
required=True)
parser.add_argument('--in-folder',
type=str,
help='Folder of input data',
required=True)
parser.add_argument('--ref-img',
type=str,
help='Use the average as reference image',
required=True)
parser.add_argument('--out-pc-folder',
type=str,
help='Output location of principle images')
parser.add_argument('--out-mean-img',
type=str,
help='Output the mean of pca',
required=True)
parser.add_argument('--n-components',
type=int,
help='Number of pc',
required=True)
parser.add_argument('--save-pca-result-path',
type=str,
help='Save the result to the given location')
args = parser.parse_args()
config = load_config(args.config)
scan_folder_reader = ScanFolderFlatReader(config, args.in_folder,
args.ref_img)
scan_folder_reader.read_data()
pca_nii_3d = PCA_NII_3D(scan_folder_reader, args.ref_img,
args.n_components)
pca_nii_3d.run_pca()
if args.save_pca_result_path:
pca_nii_3d.save_pca_obj(args.save_pca_result_path)
if args.out_pc_folder:
mkdir_p(args.out_pc_folder)
pca_nii_3d.write_pc(args.out_pc_folder)
if args.out_mean_img:
pca_nii_3d.write_mean(args.out_mean_img)
