z_max_lb_c = 17
odds_lb_c = 18
# cut off
flux_alt_thresh = 4.64
nstar_thresh = 12
total_area_thresh = 1
field_g1_bound = 0.005
field_g2_bound = 0.0075
z_min, z_max = 0.0, 15
c1_correction = -0.000475
c2_correction = 0.000504
odds_thresh = 0.3
############################# Fourier_Quad Option ################################################
logger = tool_box.get_logger("%s/work/test/log/%d.dat" % (my_home, rank))
############################# Fourier_quad data collection #######################################
# combine the data of each field into one big catalog
if cmd == "collect":
t1 = time.time()
h5f_path = fourier_cata_path + "fourier_cata.hdf5"
if rank == 0:
h5f = h5py.File(h5f_path, "w")
h5f.close()
c_dicts, c_fields = tool_box.field_dict(cfht_cata_path + "nname.dat")
f_dicts, f_fields = tool_box.field_dict(fourier_cata_path + "nname.dat")
from astropy.io import fits
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
source = argv[1]
ini_path = "%s/work/envs/envs.dat" % my_home
total_path, para_path = tool_box.config(
ini_path, ['get', 'get'],
[['selection_bias', "%s_path" % source, '1'],
['selection_bias', "%s_path_para" % source, '1']])
log_name = total_path + "logs/R_%02d.dat" % rank
logger = tool_box.get_logger(log_name)
# the parameters
para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1],
["para", "stamp_col", 1], ["para", "shear_num", 1],
["para", "noise_sig", 1], ["para", "pixel_scale", 1]]
para_items = tool_box.config(para_path + "para.ini",
['get', 'get', 'get', 'get', 'get', 'get'],
para_contents)
total_chip_num = int(para_items[0])
stamp_size = int(para_items[1])
columns = int(para_items[2])
shear_num = int(para_items[3])
stamp_num = 10000
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
numprocs = comm.Get_size()
total_path = argv[1]
log_path = total_path + "/logs/py_%02d.dat" % rank
stamp_size = 64
stamp_num = 10000
stamp_nx = 100
noise_sigma = 60
total_chips = 1000
shear_num = 10
logger = tool_box.get_logger(log_path)
m, n = divmod(total_chips, numprocs)
seed_step = 2
seed_gap = 2 * seed_step * shear_num * (m + 1)
seed = seed_gap * rank + 1
chips_label = numpy.zeros((numprocs, ), dtype=numpy.intc) + m
for i in range(n):
chips_label[i] += 1
chip_st = chips_label[:rank].sum()
chip_ed = chip_st + chips_label[rank]
for i in range(shear_num):
for j in range(chip_st, chip_ed):
theta_deg = numpy.zeros((15, theta_num))
for i in range(15):
theta_deg[i] = tool_box.set_bin_log(0.8, 60, theta_num)/60
theta_deg = theta_deg.flatten()
task_list = []
for i in range(sigma8_num):
for j in range(omega_c_num):
for k in range(omega_b_num):
task_list.append([i,j,k])
my_task = tool_box.alloc(task_list, numprocs, "mean")[rank]
my_xip = numpy.zeros((sigma8_num, omega_c_num, omega_b_num, int(theta_num*15)), dtype=numpy.float32)
logger = tool_box.get_logger("./logs/log_%d.dat"%rank)
for tag, ijk in enumerate(my_task):
t1 = time.time()
i,j,k = ijk
sigma8 = sigma8_list[i]
omega_c = omega_c_list[j]
omega_b = omega_b_list[k]
logger.info("%d. %.4f %.4f %.4f." % (tag, sigma8, omega_c, omega_b))
try:
ccl_xip = cf_tool.get_tomo_xi_ccl_2(sigma8, omega_c, omega_b, h, zebin_cent, zehist, theta_deg, theta_num, ell, used_zbins)
my_xip[i, j, k] = ccl_xip
t2 = time.time()
logger.info("%d. %.2f sec." % (tag, t2 - t1))
num = int(paras[0])
size = int(paras[1])
num_i = int(num / loops)
mag_s, mag_e = float(paras[2]), float(paras[3])
radius_s, radius_e = float(paras[4]), float(paras[5])
stamp_num = 10000
if rank == 0:
print(num * stamp_num, size, mag_s, mag_e, radius_s, radius_e)
comm.Barrier()
pic = para_path + "/pic/ellip_%d.png" % rank
plt.figure(figsize=(16, 16))
h5_path = para_path + '/para_%d.hdf5' % rank
para_logs_path = para_path + "/logs/logs_%d.dat" % rank
logger = tool_box.get_logger(para_logs_path)
log_inform = "RANK: %d, LOOP: %d, TOTAL NUM: %d, NUM in LOOP: %d, " \
"SIZEL %d, MAG: %f ~ %f, RADIUS: %.2f ~ %.2f\n"%(rank, loops, num, num_i, size, mag_s, mag_e, radius_s, radius_e)
logger.info(log_inform)
h5f = h5py.File(h5_path, "w")
itemsize = MPI.DOUBLE.Get_size()
element_num = cpus
if rank == 0:
# bytes for 10 double elements
nbytes = element_num * itemsize
else:
nbytes = 0
# on rank 0 of comm, create the contiguous shared block
win1 = MPI.Win.Allocate_shared(nbytes * 2, itemsize, comm=comm)
fg2 = shear["arr_1"][:, 0]
else:
fg2 = shear["arr_1"]
sex_path = total_path + "result/data/%s/sex_%d.npz" % (filter_name, rank)
para_h5_path = para_path + "para_%d.hdf5" % rank
para_h5 = h5py.File(para_h5_path, "r")
mag_true = para_h5["/mag"].value
para_h5.close()
# rfactor_path = result_path + "data/resolution_factor_%d.npz"%rank
if rank == 0:
log = "START -- %s, %7s, %9s, , %5.2f, %12s" % (
total_path.split("/")[-2], filter_name, cut, r_thresh, argv[0])
logger = tool_box.get_logger(
"%s/work/selection_bias/sym_mc_plot/cutoff.dat" % my_home)
logger.info(log)
shear_esti_h5path = result_path + "data/data_%d.hdf5" % rank
shear_esti_file = h5py.File(shear_esti_h5path, "r")
shear_esti_data = shear_esti_file["/data"].value
shear_esti_file.close()
fq = Fourier_Quad(60, 152356)
noise_sig = 60
MG1 = shear_esti_data[:, 2]
MG2 = shear_esti_data[:, 3]
MN = shear_esti_data[:, 4]
MU = shear_esti_data[:, 5]
MV = shear_esti_data[:, 6]
source = argv[1]
envs_path = "%s/work/envs/envs.dat" % my_home
get_contents = [['selection_bias', "%s_path" % source, '1'],
['selection_bias',
"%s_path_result" % source, '1'],
['selection_bias',
"%s_path_para" % source, '1'],
['selection_bias',
"%s_path_log" % source, '1']]
path_items = tool_box.config(envs_path, ['get', 'get', 'get', 'get'],
get_contents)
total_path, result_path, para_path, log_path = path_items
logger = tool_box.get_logger(log_path + "%d_logs.dat" % rank)
# the parameters
para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1],
["para", "stamp_col", 1], ["para", "shear_num", 1],
["para", "noise_sig", 1], ["para", "pixel_scale", 1]]
para_items = tool_box.config(para_path + "para.ini",
['get', 'get', 'get', 'get', 'get', 'get'],
para_contents)
total_chips_num = int(para_items[0])
stamp_size = int(para_items[1])
stamp_col = int(para_items[2])
shear_num = int(para_items[3])
noise_sig = int(para_items[4])
pixel_scale = float(para_items[5])
scale_radius = 0.35
gal_flux = gal_fluxs[tag]
noise_sig = 60
shear_num = 20
total_chips = 2000
pixel_scale = 0.187
stamp_size = 48
stamp_nx = 100
stamp_ny = 100
stamp_num = stamp_nx * stamp_ny
nx, ny = stamp_nx * stamp_size, stamp_ny * stamp_size
fq = Fourier_Quad(stamp_size, 123)
logger = tool_box.get_logger(parent_path + "/logs/%d_logs.dat" % rank)
psf = galsim.Moffat(beta=3.5, fwhm=0.7, flux=1.0, trunc=1.4).shear(e1=0.1,
e2=0.)
if rank == 0:
psf_img = galsim.ImageD(stamp_size, stamp_size)
psf.drawImage(image=psf_img, scale=pixel_scale)
psf_arr = numpy.float32(psf_img.array)
hdu = fits.PrimaryHDU(psf_arr)
psf_path = parent_path + '/imgs/psf.fits'
hdu.writeto(psf_path, overwrite=True)
gal = galsim.Sersic(scale_radius=scale_radius,
n=sersic_idx,
trunc=4.5 * scale_radius,
flux=1.0)
area_num = 4
result_source = "result_ext"
envs_path = "%s/work/envs/envs.dat" % my_home
gets_item = [["gg_lensing", "ggl_path_data", "0"]]
path_items = tool_box.config(envs_path, ["get"], gets_item)
data_path = path_items[0]
h5f_path_cut = data_path + "cata_%s_cut.hdf5" % result_source
h5f_path_grid = data_path + "cata_%s_grid.hdf5" % result_source
logger = tool_box.get_logger(
"/home/hklee/work/CFHT/gg_lensing/log/check_%d.dat" % rank)
pre_h5f = h5py.File(h5f_path_cut, "r")
grid_h5f = h5py.File(h5f_path_grid, "r")
data_name = [
"Z", "DISTANCE", "RA", "DEC", "COS_DEC", "G1", "G2", "N", "U", "V",
"num_in_block", "block_start", "block_end", "block_boundy", "block_boundx"
]
if rank < area_num:
grid_data_sets = [
"/background/w_%d/%s" % (rank + 1, data_name[i])
for i in range(len(data_name))
]
