# bytes for 10 double elements
nbytes = element_num * itemsize
else:
nbytes = 0
# on rank 0 of comm, create the contiguous shared block
win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm)
buf1, itemsize = win1.Shared_query(0)
result_buff = numpy.ndarray(buffer=buf1,
dtype='d',
shape=(grid_num,
grid_num * 4)) # array filled with zero
total_grid_num = grid_num * grid_num
my_grids = tool_box.alloc([i for i in range(total_grid_num)], numprocs)[rank]
idxs = numpy.linspace(0.31, 1.2, grid_num)
radii = numpy.linspace(0.2, 0.8, grid_num)
gal_flux = 6000
noise_sig = 60
shear_num = 40
pixel_scale = 0.187
stamp_size = 64
g1_result = numpy.zeros((2, shear_num))
g2_result = numpy.zeros((2, shear_num))
temp = numpy.zeros((4, 3))
sigma8_list = numpy.linspace(0.3, 2.5, sigma8_num)
omega_c_list = numpy.linspace(0.01, 0.8,omega_c_num)
omega_b_list = numpy.linspace(0.01,0.2,omega_b_num)
theta_deg = numpy.zeros((15, theta_num))
for i in range(15):
theta_deg[i] = tool_box.set_bin_log(0.8, 60, theta_num)/60
theta_deg = theta_deg.flatten()
task_list = []
for i in range(sigma8_num):
for j in range(omega_c_num):
for k in range(omega_b_num):
task_list.append([i,j,k])
my_task = tool_box.alloc(task_list, numprocs, "mean")[rank]
my_xip = numpy.zeros((sigma8_num, omega_c_num, omega_b_num, int(theta_num*15)), dtype=numpy.float32)
logger = tool_box.get_logger("./logs/log_%d.dat"%rank)
for tag, ijk in enumerate(my_task):
t1 = time.time()
i,j,k = ijk
sigma8 = sigma8_list[i]
omega_c = omega_c_list[j]
omega_b = omega_b_list[k]
logger.info("%d. %.4f %.4f %.4f." % (tag, sigma8, omega_c, omega_b))
try:
ccl_xip = cf_tool.get_tomo_xi_ccl_2(sigma8, omega_c, omega_b, h, zebin_cent, zehist, theta_deg, theta_num, ell, used_zbins)
with open(total_path + "/cat_inform/nname_field_chips.dat", "w") as f:
f.writelines(all_files)
with open(total_path + "/cat_inform/nname_field.dat", "w") as f:
f.writelines(fields)
print(len(fields))
elif mode == "hdf5_cata":
# convert the .dat to .hdf5
fields, field_name = tool_box.field_dict(
total_path + "/cat_inform/nname_field_chips.dat")
# if rank == 0:
# print(len(field_name))
field_name_sub = tool_box.alloc(field_name, cpus, "seq")[rank]
fields_expo_sub_avail = []
fields_expo_sub_raw_avail = []
fields_sub_avail = []
fields_raw_sub_avail = []
exception_sub = []
for fns in field_name_sub:
# read the the field data
field_src_path = total_path + "/%s/result" % fns
try:
expos = list(fields[fns].keys())
expos_num = len(expos)
# read the exposure files
fq.get_radius_new(psf_pow, 2)
# show PSF
# img = Image_Plot()
# img.subplots(1,2)
# img.axs[0][0].set_title("PSF")
# fig = img.axs[0][0].imshow(psf_img)
# img.figure.colorbar(fig, ax=img.axs[0][0])
# img.axs[0][1].set_title("e-PSF")
# fig = img.axs[0][1].imshow(epsf_img)
# img.figure.colorbar(fig, ax=img.axs[0][1])
# img.show_img()
task_list = [[i,j] for i in range(si_num) for j in range(sr_num)]
my_task_list = tool_box.alloc(task_list, numprocs)[rank]
# simulate galaxy images
for ij in my_task_list:
i,j = ij
# measured shear
mgs = numpy.zeros((2, shear_num))
for k in range(shear_num):
gal = galsim.Sersic(scale_radius=scale_radius[j], n=sersic_index[i], trunc=4.5 * scale_radius[j], flux=1.0)
GN = numpy.zeros((3,))
for ik in range(gal_num):
source_list_nm = "/cat_inform/nname_field_raw_avail.dat"
fields = []
with open(src_path + source_list_nm, "r") as f:
conts = f.readlines()
for nm in conts:
field_nm = nm.split("\n")[0]
if field_nm not in fields:
fields.append(field_nm)
if rank == 0:
print("Totally ",len(fields)," fields")
sub_fields = tool_box.alloc(fields, cpus)[rank]
for fnm in sub_fields:
expos = []
expos_name = []
files = os.listdir(src_path + "/%s/result"%fnm)
for nm in files:
if "p_all_raw" in nm:
expos.append(src_path + "/%s/result/%s"%(fnm, nm))
expos_name.append(nm.split("_")[0])
if len(expos) > 0:
# for tag, expo_path in enumerate(expos):
# h5f = h5py.File(expo_path, "r")
# temp = h5f["/data"][()]
if rank == 0:
print(tag_1, tag_2, data_path)
itemsize = MPI.DOUBLE.Get_size()
if rank == 0:
# bytes for 10 double elements
nbytes = 2 * chi_guess_num * itemsize
else:
nbytes = 0
win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm)
buf1, itemsize = win1.Shared_query(0)
chisq_arr = numpy.ndarray(buffer=buf1, dtype='d', shape=(2, chi_guess_num))
task_list = [i for i in range(chi_guess_num)]
my_task_list = tool_box.alloc(task_list, cpus)[rank]
if rank == 0:
if not os.path.exists(data_path + "/mg_bin_%s.hdf5" % data_type):
fq = Fourier_Quad(12, 12412)
h5f = h5py.File(data_path + "/data_%d_%s.hdf5" % (tag_1, data_type),
"r")
shear_data = h5f["/data_0"][()]
h5f.close()
mg_bin = fq.set_bin(shear_data[:, 0], mg_bin_num)
h5f = h5py.File(data_path + "/mg_bin_%s.hdf5" % data_type, "w")
h5f["/data"] = mg_bin
h5f.close()
print(mg_bin)
comm.Barrier()
for fnm in files:
if ori_cat_chara in fnm:
band_files.append("%s\n" % fnm.split(".")[0])
with open(total_path + "/code/exposures_%s_band.dat" % ib, "w") as f:
f.writelines(band_files)
comm.Barrier()
need_idx = [0, 1, 5, 6, 14, 21, 24, 25, 26, 27, 28, 29, 30]
for ib in data_band:
with open(total_path + "/code/exposures_%s_band.dat" % ib, "r") as f:
cc = f.readlines()
files = tool_box.alloc(cc, numprocs, "seq")[rank]
exposures_candidates_avail_sub = []
for fnm in files:
nm = fnm.split("\n")[0]
src_path = ori_cat_path + "/%s/%s.cat" % (ib, nm)
dst_path = total_path + "/%s/%s.hdf5" % (ib, nm)
try:
src_data = numpy.loadtxt(src_path, dtype=numpy.float32)
row, col = src_data.shape
except:
if os.path.exists(src_path):
log_inform = "%d Failed in reading %s %d Bytes !\n" % (
rank, src_path, os.path.getsize(src_path))
# [[0,-1],
# [1,0]],
# [[0,-1],
# [-1,0]]][theta_tag]
rotation = [[0, -1], [1, 0]]
if rank == 0:
print("rotation: %d %d\n"
" %d %d" %
(rotation[0][0], rotation[0][1], rotation[1][0], rotation[1][1]))
if not os.path.exists(data_path + "/mix"):
os.makedirs(data_path + "/mix")
comm.Barrier()
my_task = tool_box.alloc([i for i in range(40)], numprocs)[rank]
for i in my_task:
data_nm_1 = data_path + "/data_noisy_cpp_%d.hdf5" % i
data_nm_2 = data_path + "/data_noise_free_%d.hdf5" % i
# data_nm_2 = data_path + "/data_gal_noise_cross_term_%d.hdf5"%i
data_nm_3 = data_path + "/data_pure_gal_noise_cross_term_est_r_%d.hdf5" % i
# data_nm_2 = data_path + "/data_noise_free_%d.hdf5"%i
h5f = h5py.File(data_nm_1, "r")
# mg1 = h5f["/mg1"][()]
# mg2 = h5f["/mg2"][()]
mn = h5f["/mn"][()]
mu = h5f["/mu"][()]
mv = h5f["/mv"][()]
h5f.close()
src_dec = ori_src_dec[idx] src_ra = ori_src_ra[idx] src_z = ori_src_z[idx] src_g1 = ori_src_g1[idx] src_g2 = ori_src_g2[idx] # src_halo_id = ori_src_halo_id[idx] print(idx.sum(), idx.shape[0]) com_dist_halo = cosmos.comoving_distance(halo_z).value*h0 com_dist_src = cosmos.comoving_distance(src_z).value*h0 src_position = SkyCoord(ra = src_ra*units.deg, dec = src_dec*units.deg,frame="fk5") # halo_position = SkyCoord(ra=halo_ra*units.deg, dec=halo_dec*units.deg) halo_num = halo_dec.shape[0] my_halo = tool_box.alloc([i for i in range(halo_num)], numprocs)[rank] my_cache = numpy.zeros((4, radius_bin_num)) # # itemsize = MPI.DOUBLE.Get_size() # # if rank == 0: # # bytes for 10 double elements # nbytes = 4*radius_bin_num*itemsize # else: # nbytes = 0 # # win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm) # buf1, itemsize = win1.Shared_query(0) # total_cache = numpy.ndarray(buffer=buf1, dtype='d', shape=(4, radius_bin_num))
if rank == 0:
psf_img = galsim.ImageD(stamp_size, stamp_size)
psf.drawImage(image=psf_img, scale=pixel_scale)
psf_arr = numpy.float32(psf_img.array)
hdu = fits.PrimaryHDU(psf_arr)
psf_path = parent_path + '/imgs/psf.fits'
hdu.writeto(psf_path, overwrite=True)
gal = galsim.Sersic(scale_radius=scale_radius,
n=sersic_idx,
trunc=4.5 * scale_radius,
flux=1.0)
# task allocation
chip_tags = [i for i in range(total_chips)]
chip_tags_rank = tool_box.alloc(chip_tags, cpus)[rank]
shear_path = parent_path + "/parameters/shear.hdf5"
h5f = h5py.File(shear_path, "r")
g1_input = h5f["/g1"][()]
g2_input = h5f["/g2"][()]
h5f.close()
img_buffer = numpy.zeros((ny, nx))
counts = 0
for shear_id in range(shear_num):
if rank == 0:
if not os.path.exists(parent_path + "/imgs/%d" % shear_id):
os.makedirs(parent_path + "/imgs/%d" % shear_id)
comm.Barrier()
if cmd == "prepare":
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
total_expos = []
with open(fourier_cata_path + "/cat_inform/exposure_avail.dat", "r") as f:
f_lines = f.readlines()
for ff in f_lines:
total_expos.append(ff.split("\n")[0])
my_expos = tool_box.alloc(total_expos,cpus, method="order")[rank]
expo_avail_sub = []
exception_sub = []
for cat_path in my_expos:
expo_name = cat_path.split("/")[-1]
h5f_src = h5py.File(cat_path,"r")
data = h5f_src["/data"][()]
h5f_src.close()
# selection
idx1 = data[:, nstar_idx] >= nstar_thresh
idx2 = data[:, flux2_alt_idx] >= flux2_alt_thresh
idx3 = numpy.abs(data[:, gf1_idx]) <= gf1_thresh
idx4 = numpy.abs(data[:, gf2_idx]) <= gf2_thresh
expos.append(target_path + "/" + fn)
else:
expos = None
expos = comm.bcast(expos, root=0)
expos_num = len(expos)
if rank == 0:
print(target_path)
print(name_need)
print(dst_path)
print(expos_num, " exposures")
if expos_num < 1:
exit()
my_sub_area_list = tool_box.alloc(expos, cpus)[rank]
# print(rank, i, len(my_sub_area_list))
stack_tag = 0
if len(my_sub_area_list) > 0:
for tag, expo_path in enumerate(my_sub_area_list):
h5f = h5py.File(expo_path, "r")
temp = h5f["/data"][()]
h5f.close()
if stack_tag == 0:
stack_data = temp
stack_tag = 1
else:
stack_data = numpy.row_stack((stack_data, temp))
pos = []
with open(parent_path + "/cat_inform/exposure_avail_inform.dat", "r") as f:
cc = f.readlines()
for tag, row in enumerate(cc):
if tag > 0:
iterms = row.split("\n")[0].split("\t")
file_name.append(iterms[0])
temp = []
for i in range(1, len(iterms) - 1):
temp.append(float(iterms[i]))
pos_arr = numpy.array(pos)
file_num = pos_arr.shape[0]
task_label = [i for i in range(file_num)]
task_label_sub = tool_box.alloc(task_label, numprocs)[rank]
for i in task_label_sub:
t1 = time.time()
h5f = h5py.File(parent_path + "/cat_hdf5/%s.hdf5" % file_name[i], "r")
data = h5f["/data"][()]
h5f.close()
ra1 = numpy.ascontiguousarray(data[:, ra_col], dtype=numpy.float32)
dec1 = numpy.ascontiguousarray(data[:, dec_col], dtype=numpy.float32)
z1 = numpy.ascontiguousarray(data[:, z_col], dtype=numpy.float32)
ra1_cent = ra1.mean()
dec1_cent = dec1.mean()
src_path = total_path + "/cat_hdf5"
pic_path = total_path + "/cat_inform/field_distortion"
source_list_nm = "exposure_avail.dat"
with open(total_path + "/cat_inform/"+source_list_nm, "r") as f:
content = f.readlines()
all_exposures_path = []
all_exposures = []
for cc in content:
all_exposures.append(cc.split("/")[-1].split("\n")[0])
all_exposures_path.append(cc.split("\n")[0])
my_exposures_path = tool_box.alloc(all_exposures_path, cpus)[rank]
my_exposures = tool_box.alloc(all_exposures, cpus)[rank]
for tag, fnm in enumerate(my_exposures):
# print(my_exposures_path[tag], fnm)
h5f = h5py.File(my_exposures_path[tag], "r")
data = h5f["/data"][()]
h5f.close()
col_shift = 16
ichip = data[:, col_shift] - 1
xc = data[:,col_shift+2]
yc = data[:,col_shift+3]
ra = data[:,col_shift+13]
source_list_nm = "/cat_inform/nname_field_raw_avail.dat"
fields = []
with open(src_path + source_list_nm, "r") as f:
conts = f.readlines()
for nm in conts:
field_nm = nm.split("\n")[0]
if field_nm not in fields:
fields.append(field_nm)
if rank == 0:
print("Totally ",len(fields)," fields")
sub_fields = tool_box.alloc(fields, cpus)[rank]
fq = Fourier_Quad(12,123)
bin_num = 20
gf_bin = numpy.linspace(-0.005, 0.005, bin_num+1)
gf_bin_c = (gf_bin[1:] + gf_bin[:-1])/2
result = numpy.zeros((2, bin_num))
# for fnm in sub_fields:
# if os.path.exists(src_path + "/%s/result/%s_raw.hdf5" % (fnm, fnm)):
# h5f = h5py.File(src_path + "/%s/result/%s_raw.hdf5" % (fnm, fnm), "r")
# data = h5f["/data"][()]
# h5f.close()
# col_shift = 0
h5f.close()
h5f = h5py.File(data_path3, "r")
mg1_corr = h5f["/data"][()][:, 0]
mnu1_corr = h5f["/data"][()][:, 2] + h5f["/data"][()][:, 3]
h5f.close()
G1_bin = tool_box.set_bin(mg1[:num_s], mg_bin_num, 1000)
G1_hist_bin = gglensing_tool.set_bin(mg1[:num_s], 2500, 1.001, "log")
NU1_hist_bin = gglensing_tool.set_bin(mnu1[:num_s], 2500, 1.001, "log")
dilute_case = [0, 0.1, 0.2, 0.3, 0.4, 0.5]
dilute_num = len(dilute_case)
task_list = [i for i in range(dilute_num)]
task_list_sub = tool_box.alloc(task_list, numprocs)[rank]
itemsize = MPI.DOUBLE.Get_size()
if rank == 0:
# bytes for 10 double elements
nbytes = 4 * dilute_num * itemsize
else:
nbytes = 0
# on rank 0 of comm, create the contiguous shared block
win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm)
buf1, itemsize = win1.Shared_query(0)
result_min = numpy.ndarray(buffer=buf1, dtype='d', shape=(4, dilute_num))
comm.Barrier()
omega_bm_num = 10
omega_bm_bin = numpy.linspace(0.01,0.1,omega_bm_num)
omega_cm_num = 91
omega_cm_bin = numpy.linspace(0.1,1,omega_cm_num)
sigma8_num = 71
sigma8_bin = numpy.linspace(0.3, 1,sigma8_num)
chisqs = numpy.zeros((omega_bm_num, omega_cm_num, sigma8_num))
ijk = []
for i in range(omega_bm_num):
for j in range(omega_cm_num):
for k in range(sigma8_num):
ijk.append([i,j,k])
ijk_sub = tool_box.alloc(ijk, numprocs)[rank]
h5f = h5py.File("./%s/zhist.hdf5"%data_path,"r")
zehist = h5f["/zhist"][()]
zebin = h5f["/zbin"][()]
zebin_cent = h5f["/zbin_cent"][()]
h5f.close()
redshift_bin_num = zehist.shape[0]
tomo_panel_num = int((redshift_bin_num * redshift_bin_num + redshift_bin_num) / 2)
resample_num = 200
h5f = h5py.File("./%s/result_cache_%d_%s.hdf5"%(data_path,resample_num, expo_type), "r")
pz_data = h5f["/data"][()]
h5f.close()
# print("%d Read Pz cata"%rank)
comm.Barrier()
comm.Barrier()
# zbin = numpy.array([0.025 + i * 0.05 for i in range(70)])
exposures_candidates = []
with open(total_path + "/cat_inform/exposure_name.dat", "r") as f:
contents = f.readlines()
for expo_name in contents:
exposures_candidates.append(expo_name.split("\n")[0])
exposures_candidates_sub = tool_box.alloc(exposures_candidates, numprocs, "seq")[rank]
exposures_candidates_avail_sub = []
exception_sub = []
for fns in exposures_candidates_sub:
# read the the field data
expo_src_path = total_path + "/cat_ori/%s.cat" % fns
expo_h5_path = total_path + "/cat_hdf5/%s.hdf5" % fns
try:
src_data = numpy.loadtxt(expo_src_path, dtype=numpy.float32)
row, col = src_data.shape
except:
if os.path.exists(expo_src_path):
log_inform = "%d Failed in reading %s %d Bytes !\n" % (
rank, expo_src_path, os.path.getsize(expo_src_path))
# f.writelines(field_nm)
total_path = argv[1]
dst_path = total_path + "/cat_hdf5/stack_expo"
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
numprocs = comm.Get_size()
with open("/home/hklee/work/CFHT/field_expo.dat", "r") as f:
field_expo = f.readlines()
if rank == 0:
print(len(field_expo))
field_expo_sub = tool_box.alloc(field_expo, numprocs)[rank]
if rank == 0:
if os.path.exists(dst_path):
shutil.rmtree(dst_path, ignore_errors=True)
os.mkdir(dst_path)
field_avail_sub = []
for fnm in field_expo_sub:
temp_str = fnm.split()
field_name = temp_str[0]
max_len = 0
data_list = []
if field in f_fields:
pre_fields.append(field)
# loop the areas,
# the catalog of each area will be stored in "w_i"
num = []
for area_id in range(1, area_num + 1):
# distribution the files
field_tar = []
for field in pre_fields:
if "w%d" % area_id in field:
field_tar.append(field)
# if fields are more than the threads,
# some thread will get an empty "field_pool"
field_pool = tool_box.alloc(field_tar, cpus)[rank]
# check
anothers = ["w%d" % i for i in range(1, area_num + 1) if i != area_id]
for field_name in field_pool:
for ext_field in anothers:
if ext_field in field_name:
print("WRONG", rank, area_id, field_name)
exit()
field_count = 0
cata_data = None
for field_name in field_pool:
# read the catalogs from Fourier_Quad and CFHT
# for the old version catalog
f_data_path = fourier_cata_path + "%s/result_ext/%s_shear.dat" % (
for i in range(4):
h5f = h5py.File(data_path + "/group_predict_%d.hdf5"%i,"r")
label = h5f["/data"][()]
h5f.close()
h5f = h5py.File(data_path + "/stack_data_%d.hdf5"%i, "r")
data = h5f["/data"][()][:,:4]
h5f.close()
label_min, label_max = label.min(), label.max()
label_list = [i for i in range(label_min, label_max+1)]
label_list_sub = tool_box.alloc(label_list, cpus)[rank]
results = numpy.zeros((len(label_list_sub), 5))
sp = (len(label_list_sub), 5)
for tag, ib in enumerate(label_list_sub):
idx = label == ib
data_sub = data[idx]
mg1, mg2, mn, mu = data_sub[:,0],data_sub[:,1],data_sub[:,2],data_sub[:,3]
gh1, gh1_sig = FQlib.find_shear_cpp(data_sub[:,0], data_sub[:,2] + data_sub[:,3],20)[:2]
gh2, gh2_sig = FQlib.find_shear_cpp(data_sub[:,1], data_sub[:,2] - data_sub[:,3],20)[:2]
results[tag] = ib, gh1, gh1_sig, gh2, gh2_sig
