z_max_lb_c = 17 odds_lb_c = 18 # cut off flux_alt_thresh = 4.64 nstar_thresh = 12 total_area_thresh = 1 field_g1_bound = 0.005 field_g2_bound = 0.0075 z_min, z_max = 0.0, 15 c1_correction = -0.000475 c2_correction = 0.000504 odds_thresh = 0.3 ############################# Fourier_Quad Option ################################################ logger = tool_box.get_logger("%s/work/test/log/%d.dat" % (my_home, rank)) ############################# Fourier_quad data collection ####################################### # combine the data of each field into one big catalog if cmd == "collect": t1 = time.time() h5f_path = fourier_cata_path + "fourier_cata.hdf5" if rank == 0: h5f = h5py.File(h5f_path, "w") h5f.close() c_dicts, c_fields = tool_box.field_dict(cfht_cata_path + "nname.dat") f_dicts, f_fields = tool_box.field_dict(fourier_cata_path + "nname.dat")
from astropy.io import fits comm = MPI.COMM_WORLD rank = comm.Get_rank() cpus = comm.Get_size() source = argv[1] ini_path = "%s/work/envs/envs.dat" % my_home total_path, para_path = tool_box.config( ini_path, ['get', 'get'], [['selection_bias', "%s_path" % source, '1'], ['selection_bias', "%s_path_para" % source, '1']]) log_name = total_path + "logs/R_%02d.dat" % rank logger = tool_box.get_logger(log_name) # the parameters para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1], ["para", "stamp_col", 1], ["para", "shear_num", 1], ["para", "noise_sig", 1], ["para", "pixel_scale", 1]] para_items = tool_box.config(para_path + "para.ini", ['get', 'get', 'get', 'get', 'get', 'get'], para_contents) total_chip_num = int(para_items[0]) stamp_size = int(para_items[1]) columns = int(para_items[2]) shear_num = int(para_items[3]) stamp_num = 10000
comm = MPI.COMM_WORLD rank = comm.Get_rank() numprocs = comm.Get_size() total_path = argv[1] log_path = total_path + "/logs/py_%02d.dat" % rank stamp_size = 64 stamp_num = 10000 stamp_nx = 100 noise_sigma = 60 total_chips = 1000 shear_num = 10 logger = tool_box.get_logger(log_path) m, n = divmod(total_chips, numprocs) seed_step = 2 seed_gap = 2 * seed_step * shear_num * (m + 1) seed = seed_gap * rank + 1 chips_label = numpy.zeros((numprocs, ), dtype=numpy.intc) + m for i in range(n): chips_label[i] += 1 chip_st = chips_label[:rank].sum() chip_ed = chip_st + chips_label[rank] for i in range(shear_num): for j in range(chip_st, chip_ed):
theta_deg = numpy.zeros((15, theta_num)) for i in range(15): theta_deg[i] = tool_box.set_bin_log(0.8, 60, theta_num)/60 theta_deg = theta_deg.flatten() task_list = [] for i in range(sigma8_num): for j in range(omega_c_num): for k in range(omega_b_num): task_list.append([i,j,k]) my_task = tool_box.alloc(task_list, numprocs, "mean")[rank] my_xip = numpy.zeros((sigma8_num, omega_c_num, omega_b_num, int(theta_num*15)), dtype=numpy.float32) logger = tool_box.get_logger("./logs/log_%d.dat"%rank) for tag, ijk in enumerate(my_task): t1 = time.time() i,j,k = ijk sigma8 = sigma8_list[i] omega_c = omega_c_list[j] omega_b = omega_b_list[k] logger.info("%d. %.4f %.4f %.4f." % (tag, sigma8, omega_c, omega_b)) try: ccl_xip = cf_tool.get_tomo_xi_ccl_2(sigma8, omega_c, omega_b, h, zebin_cent, zehist, theta_deg, theta_num, ell, used_zbins) my_xip[i, j, k] = ccl_xip t2 = time.time() logger.info("%d. %.2f sec." % (tag, t2 - t1))
num = int(paras[0]) size = int(paras[1]) num_i = int(num / loops) mag_s, mag_e = float(paras[2]), float(paras[3]) radius_s, radius_e = float(paras[4]), float(paras[5]) stamp_num = 10000 if rank == 0: print(num * stamp_num, size, mag_s, mag_e, radius_s, radius_e) comm.Barrier() pic = para_path + "/pic/ellip_%d.png" % rank plt.figure(figsize=(16, 16)) h5_path = para_path + '/para_%d.hdf5' % rank para_logs_path = para_path + "/logs/logs_%d.dat" % rank logger = tool_box.get_logger(para_logs_path) log_inform = "RANK: %d, LOOP: %d, TOTAL NUM: %d, NUM in LOOP: %d, " \ "SIZEL %d, MAG: %f ~ %f, RADIUS: %.2f ~ %.2f\n"%(rank, loops, num, num_i, size, mag_s, mag_e, radius_s, radius_e) logger.info(log_inform) h5f = h5py.File(h5_path, "w") itemsize = MPI.DOUBLE.Get_size() element_num = cpus if rank == 0: # bytes for 10 double elements nbytes = element_num * itemsize else: nbytes = 0 # on rank 0 of comm, create the contiguous shared block win1 = MPI.Win.Allocate_shared(nbytes * 2, itemsize, comm=comm)
fg2 = shear["arr_1"][:, 0] else: fg2 = shear["arr_1"] sex_path = total_path + "result/data/%s/sex_%d.npz" % (filter_name, rank) para_h5_path = para_path + "para_%d.hdf5" % rank para_h5 = h5py.File(para_h5_path, "r") mag_true = para_h5["/mag"].value para_h5.close() # rfactor_path = result_path + "data/resolution_factor_%d.npz"%rank if rank == 0: log = "START -- %s, %7s, %9s, , %5.2f, %12s" % ( total_path.split("/")[-2], filter_name, cut, r_thresh, argv[0]) logger = tool_box.get_logger( "%s/work/selection_bias/sym_mc_plot/cutoff.dat" % my_home) logger.info(log) shear_esti_h5path = result_path + "data/data_%d.hdf5" % rank shear_esti_file = h5py.File(shear_esti_h5path, "r") shear_esti_data = shear_esti_file["/data"].value shear_esti_file.close() fq = Fourier_Quad(60, 152356) noise_sig = 60 MG1 = shear_esti_data[:, 2] MG2 = shear_esti_data[:, 3] MN = shear_esti_data[:, 4] MU = shear_esti_data[:, 5] MV = shear_esti_data[:, 6]
source = argv[1] envs_path = "%s/work/envs/envs.dat" % my_home get_contents = [['selection_bias', "%s_path" % source, '1'], ['selection_bias', "%s_path_result" % source, '1'], ['selection_bias', "%s_path_para" % source, '1'], ['selection_bias', "%s_path_log" % source, '1']] path_items = tool_box.config(envs_path, ['get', 'get', 'get', 'get'], get_contents) total_path, result_path, para_path, log_path = path_items logger = tool_box.get_logger(log_path + "%d_logs.dat" % rank) # the parameters para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1], ["para", "stamp_col", 1], ["para", "shear_num", 1], ["para", "noise_sig", 1], ["para", "pixel_scale", 1]] para_items = tool_box.config(para_path + "para.ini", ['get', 'get', 'get', 'get', 'get', 'get'], para_contents) total_chips_num = int(para_items[0]) stamp_size = int(para_items[1]) stamp_col = int(para_items[2]) shear_num = int(para_items[3]) noise_sig = int(para_items[4]) pixel_scale = float(para_items[5])
scale_radius = 0.35 gal_flux = gal_fluxs[tag] noise_sig = 60 shear_num = 20 total_chips = 2000 pixel_scale = 0.187 stamp_size = 48 stamp_nx = 100 stamp_ny = 100 stamp_num = stamp_nx * stamp_ny nx, ny = stamp_nx * stamp_size, stamp_ny * stamp_size fq = Fourier_Quad(stamp_size, 123) logger = tool_box.get_logger(parent_path + "/logs/%d_logs.dat" % rank) psf = galsim.Moffat(beta=3.5, fwhm=0.7, flux=1.0, trunc=1.4).shear(e1=0.1, e2=0.) if rank == 0: psf_img = galsim.ImageD(stamp_size, stamp_size) psf.drawImage(image=psf_img, scale=pixel_scale) psf_arr = numpy.float32(psf_img.array) hdu = fits.PrimaryHDU(psf_arr) psf_path = parent_path + '/imgs/psf.fits' hdu.writeto(psf_path, overwrite=True) gal = galsim.Sersic(scale_radius=scale_radius, n=sersic_idx, trunc=4.5 * scale_radius, flux=1.0)
area_num = 4 result_source = "result_ext" envs_path = "%s/work/envs/envs.dat" % my_home gets_item = [["gg_lensing", "ggl_path_data", "0"]] path_items = tool_box.config(envs_path, ["get"], gets_item) data_path = path_items[0] h5f_path_cut = data_path + "cata_%s_cut.hdf5" % result_source h5f_path_grid = data_path + "cata_%s_grid.hdf5" % result_source logger = tool_box.get_logger( "/home/hklee/work/CFHT/gg_lensing/log/check_%d.dat" % rank) pre_h5f = h5py.File(h5f_path_cut, "r") grid_h5f = h5py.File(h5f_path_grid, "r") data_name = [ "Z", "DISTANCE", "RA", "DEC", "COS_DEC", "G1", "G2", "N", "U", "V", "num_in_block", "block_start", "block_end", "block_boundy", "block_boundx" ] if rank < area_num: grid_data_sets = [ "/background/w_%d/%s" % (rank + 1, data_name[i]) for i in range(len(data_name)) ]