rank = comm.Get_rank()
cpus = comm.Get_size()
if rank == 0:
print(cmd_avail)
print(time_start, "prepare")
print(source_list_path)
total_expos = []
with open(source_list_path, "r") as f:
f_lines = f.readlines()
for ff in f_lines:
total_expos.append(ff.split("\n")[0])
my_expos = hk_tool_box.alloc(total_expos, cpus, method="order")[rank]
expo_avail_sub = []
exception_sub = []
for cat_path in my_expos:
expo_name = cat_path.split("/")[-1]
h5f_src = h5py.File(cat_path, "r")
data = h5f_src["/data"][()]
h5f_src.close()
# labels = c4py.deblend_i(data[:,xc_idx], data[:,yc_idx], data[:,redshift_idx], data[:,ichip_idx], sep_pix, sep_z)
# selection
idx1 = data[:, nstar_idx] >= nstar_thresh
idx2 = data[:, flux2_alt_idx] >= flux2_alt_thresh
if mode == "hdf5_cata":
# convert the .dat to .hdf5
total_path = argv[2]
ori_cat_path = argv[3]
exposures_candidates = []
exposures_candidates_band = []
with open(total_path + "/cat_inform/exposure_name_all_band.dat", "r") as f:
contents = f.readlines()
for expo_name in contents:
expo_nm, band = expo_name.split("\n")[0].split()
exposures_candidates.append(expo_nm)
exposures_candidates_band.append(band)
exposures_candidates_sub = hk_tool_box.alloc(exposures_candidates,
numprocs, "seq")[rank]
exposures_candidates_band_sub = hk_tool_box.alloc(
exposures_candidates_band, numprocs, "seq")[rank]
exposures_candidates_avail_sub = []
exception_sub = []
for tag, fns in enumerate(exposures_candidates_sub):
# read the the field data
iband = exposures_candidates_band_sub[tag]
expo_src_path = ori_cat_path + "/%s/%s.cat" % (iband, fns)
expo_h5_path = total_path + "/cat_hdf5/%s/%s.hdf5" % (iband, fns)
try:
src_data = numpy.loadtxt(expo_src_path, dtype=numpy.float32)
row, col = src_data.shape
except:
expos.append(target_path + "/" + fn)
else:
expos = None
expos = comm.bcast(expos, root=0)
expos_num = len(expos)
if rank == 0:
print(target_path)
print(name_need)
print(dst_path)
print(expos_num, " exposures")
if expos_num < 1:
exit()
my_sub_area_list = hk_tool_box.alloc(expos, cpus)[rank]
# print(rank, i, len(my_sub_area_list))
if len(my_sub_area_list) > 0:
for tag, expo_path in enumerate(my_sub_area_list):
h5f = h5py.File(expo_path, "r")
temp = h5f["/data"][()][:, 2:8]
h5f.close()
# # Nan check
# idx = numpy.isnan(temp)
# if idx.sum() > 0:
# num = temp.shape[0]
# label = numpy.arange(0,num)
# idx = numpy.isnan(temp[:,-2])
mgt_ny = (shear_est_ny[:, 0] * cos_2theta -
shear_est_ny[:, 1] * sin_2theta) * coeff_1
mgx_ny = (shear_est_ny[:, 0] * sin_2theta +
shear_est_ny[:, 1] * cos_2theta) * coeff_1
# mu_ny = shear_est_ny[:,3]*cos_4theta - shear_est_ny[:,4]*sin_4theta
# mn_ny = shear_est_ny[:,2]
mnu1_ny = (shear_est_ny[:, 2] + shear_est_ny[:, 3] * cos_4theta -
shear_est_ny[:, 4] * sin_4theta) * coeff_2
mnu2_ny = (shear_est_ny[:, 2] - shear_est_ny[:, 3] * cos_4theta +
shear_est_ny[:, 4] * sin_4theta) * coeff_2
radius_bin_num = numprocs
radius_bin = hk_tool_box.set_bin_log(0.2, 18, radius_bin_num + 1)
radius_bin_tag = [i for i in range(radius_bin_num)]
my_radius_bin_tag = hk_tool_box.alloc(radius_bin_tag, numprocs)[rank]
itemsize = MPI.DOUBLE.Get_size()
if rank == 0:
# bytes for 10 double elements
nbytes1 = len(pdf_bin_num) * 4 * radius_bin_num * itemsize
nbytes2 = len(pdf_bin_num) * 4 * radius_bin_num * itemsize
nbytes3 = 3 * radius_bin_num * itemsize
else:
nbytes1 = 0
nbytes2 = 0
nbytes3 = 0
# on rank 0 of comm, create the contiguous shared block
win1 = MPI.Win.Allocate_shared(nbytes1, itemsize, comm=comm)
if rank > 0:
h5f = h5py.File(stack_file_path, "r")
total_data = h5f["/data"][()]
group_label = h5f["/group_label"][()]
h5f.close()
cent_num = group_label.max() + 1
# assign the source into the artificial exposures
min_src_num = 100
expos_avail_sub = []
expos_count = 0
group_list = [i for i in range(cent_num)]
sub_group_list = hk_tool_box.alloc(group_list, cpus)[rank]
# divide the group into many exposures
for group_tag in sub_group_list:
# select individual group
idx_group = group_label == group_tag
sub_data = total_data[idx_group]
ground_src_num = idx_group.sum()
if ground_src_num <= min_src_num:
expos_name = "%d-0" % group_tag
expos_path = result_cata_path + "/foreground/%s.hdf5" % expos_name
h5f_expos = h5py.File(expos_path, "w")
h5f_expos["/data"] = sub_data
Mass = 3*10 ** 13.5 # M_sun/h
conc = 6 # concentration
len_z = 0.3 # redshift
halo_position = galsim.PositionD(0, 0) # arcsec
com_dist_len = cosmos.comoving_distance(len_z).value * h # Mpc/h
print("Lens plane at z = %.2f, %.5f Mpc/h" % (len_z, com_dist_len))
# lens profile
CF = hk_gglensing_tool.Cosmos_flat(omega_m0, 100*h)
CF.NFW((0,0), Mass, conc, len_z)
data_path = "/home/hklee/work/Galaxy_Galaxy_lensing_test/cata/background/continue_source_z_6"
test_num = numprocs
task_list = hk_tool_box.alloc([i for i in range(test_num)], numprocs)[rank]
ds_shape = (9, test_num)
sigma_z = float(argv[1])
for dz in [0, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40]:
ds_result = numpy.zeros(ds_shape)
rng = numpy.random.RandomState(12312 + rank*100)
for i in task_list:
h5f = h5py.File(data_path + "/data/segment_sheared_noisy_cpp.hdf5", "r")
src_z_ori = h5f["/%d/z" % i][()]
ra_ori = h5f["/%d/ra" % i][()]
dec_ori = h5f["/%d/dec" % i][()]
sep_radius_ori = h5f["/%d/sep_radius" % i][()]
rand_total_data = h5f["/data"][()]
rand_group_label = h5f["/group_label"][()]
h5f.close()
comm.Barrier()
# assign the source into the artificial exposures
min_src_num = 50
expos_avail_sub = []
expos_count = 0
group_list = [i for i in range(cent_num)]
sub_group_list = hk_tool_box.alloc(group_list, cpus)[rank]
# divide the group into many exposures
for group_tag in sub_group_list:
# select individual group
idx_group = group_label == group_tag
sub_data = total_data[idx_group]
ground_src_num = idx_group.sum()
if ground_src_num <= min_src_num:
expos_name = "%d-0" %group_tag
expos_path = result_cata_path + "/foreground/%s.hdf5" % expos_name
h5f_expos = h5py.File(expos_path, "w")
h5f_expos["/data"] = sub_data
labels, idx = hk_c4py.find_overlap_mask(ra, dec, decals_mask, ra_bin,
dec_bin)
return labels, idx
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
deg2rad = numpy.pi / 180
cata_path = argv[1]
with open(cata_path + "/file_list", "r") as f:
lines = f.readlines()
lines_sub = hk_tool_box.alloc(lines, cpus)[rank]
NSIDE = 256
# cent_num = 200
for fnm in lines_sub:
fnm = fnm.split("\n")[0]
for tag, ff in enumerate(["gal_jkf"]):
data = fits.open(cata_path + "/%s/%s.fits" % (fnm, ff))[1].data
col_nm = data.dtype
src_num = data.shape[0]
if tag == 0:
src_num_0 = src_num