# bytes for 10 double elements nbytes = element_num * itemsize else: nbytes = 0 # on rank 0 of comm, create the contiguous shared block win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm) buf1, itemsize = win1.Shared_query(0) result_buff = numpy.ndarray(buffer=buf1, dtype='d', shape=(grid_num, grid_num * 4)) # array filled with zero total_grid_num = grid_num * grid_num my_grids = tool_box.alloc([i for i in range(total_grid_num)], numprocs)[rank] idxs = numpy.linspace(0.31, 1.2, grid_num) radii = numpy.linspace(0.2, 0.8, grid_num) gal_flux = 6000 noise_sig = 60 shear_num = 40 pixel_scale = 0.187 stamp_size = 64 g1_result = numpy.zeros((2, shear_num)) g2_result = numpy.zeros((2, shear_num)) temp = numpy.zeros((4, 3))
sigma8_list = numpy.linspace(0.3, 2.5, sigma8_num) omega_c_list = numpy.linspace(0.01, 0.8,omega_c_num) omega_b_list = numpy.linspace(0.01,0.2,omega_b_num) theta_deg = numpy.zeros((15, theta_num)) for i in range(15): theta_deg[i] = tool_box.set_bin_log(0.8, 60, theta_num)/60 theta_deg = theta_deg.flatten() task_list = [] for i in range(sigma8_num): for j in range(omega_c_num): for k in range(omega_b_num): task_list.append([i,j,k]) my_task = tool_box.alloc(task_list, numprocs, "mean")[rank] my_xip = numpy.zeros((sigma8_num, omega_c_num, omega_b_num, int(theta_num*15)), dtype=numpy.float32) logger = tool_box.get_logger("./logs/log_%d.dat"%rank) for tag, ijk in enumerate(my_task): t1 = time.time() i,j,k = ijk sigma8 = sigma8_list[i] omega_c = omega_c_list[j] omega_b = omega_b_list[k] logger.info("%d. %.4f %.4f %.4f." % (tag, sigma8, omega_c, omega_b)) try: ccl_xip = cf_tool.get_tomo_xi_ccl_2(sigma8, omega_c, omega_b, h, zebin_cent, zehist, theta_deg, theta_num, ell, used_zbins)
with open(total_path + "/cat_inform/nname_field_chips.dat", "w") as f: f.writelines(all_files) with open(total_path + "/cat_inform/nname_field.dat", "w") as f: f.writelines(fields) print(len(fields)) elif mode == "hdf5_cata": # convert the .dat to .hdf5 fields, field_name = tool_box.field_dict( total_path + "/cat_inform/nname_field_chips.dat") # if rank == 0: # print(len(field_name)) field_name_sub = tool_box.alloc(field_name, cpus, "seq")[rank] fields_expo_sub_avail = [] fields_expo_sub_raw_avail = [] fields_sub_avail = [] fields_raw_sub_avail = [] exception_sub = [] for fns in field_name_sub: # read the the field data field_src_path = total_path + "/%s/result" % fns try: expos = list(fields[fns].keys()) expos_num = len(expos) # read the exposure files
fq.get_radius_new(psf_pow, 2) # show PSF # img = Image_Plot() # img.subplots(1,2) # img.axs[0][0].set_title("PSF") # fig = img.axs[0][0].imshow(psf_img) # img.figure.colorbar(fig, ax=img.axs[0][0]) # img.axs[0][1].set_title("e-PSF") # fig = img.axs[0][1].imshow(epsf_img) # img.figure.colorbar(fig, ax=img.axs[0][1]) # img.show_img() task_list = [[i,j] for i in range(si_num) for j in range(sr_num)] my_task_list = tool_box.alloc(task_list, numprocs)[rank] # simulate galaxy images for ij in my_task_list: i,j = ij # measured shear mgs = numpy.zeros((2, shear_num)) for k in range(shear_num): gal = galsim.Sersic(scale_radius=scale_radius[j], n=sersic_index[i], trunc=4.5 * scale_radius[j], flux=1.0) GN = numpy.zeros((3,)) for ik in range(gal_num):
source_list_nm = "/cat_inform/nname_field_raw_avail.dat" fields = [] with open(src_path + source_list_nm, "r") as f: conts = f.readlines() for nm in conts: field_nm = nm.split("\n")[0] if field_nm not in fields: fields.append(field_nm) if rank == 0: print("Totally ",len(fields)," fields") sub_fields = tool_box.alloc(fields, cpus)[rank] for fnm in sub_fields: expos = [] expos_name = [] files = os.listdir(src_path + "/%s/result"%fnm) for nm in files: if "p_all_raw" in nm: expos.append(src_path + "/%s/result/%s"%(fnm, nm)) expos_name.append(nm.split("_")[0]) if len(expos) > 0: # for tag, expo_path in enumerate(expos): # h5f = h5py.File(expo_path, "r") # temp = h5f["/data"][()]
if rank == 0: print(tag_1, tag_2, data_path) itemsize = MPI.DOUBLE.Get_size() if rank == 0: # bytes for 10 double elements nbytes = 2 * chi_guess_num * itemsize else: nbytes = 0 win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm) buf1, itemsize = win1.Shared_query(0) chisq_arr = numpy.ndarray(buffer=buf1, dtype='d', shape=(2, chi_guess_num)) task_list = [i for i in range(chi_guess_num)] my_task_list = tool_box.alloc(task_list, cpus)[rank] if rank == 0: if not os.path.exists(data_path + "/mg_bin_%s.hdf5" % data_type): fq = Fourier_Quad(12, 12412) h5f = h5py.File(data_path + "/data_%d_%s.hdf5" % (tag_1, data_type), "r") shear_data = h5f["/data_0"][()] h5f.close() mg_bin = fq.set_bin(shear_data[:, 0], mg_bin_num) h5f = h5py.File(data_path + "/mg_bin_%s.hdf5" % data_type, "w") h5f["/data"] = mg_bin h5f.close() print(mg_bin) comm.Barrier()
for fnm in files: if ori_cat_chara in fnm: band_files.append("%s\n" % fnm.split(".")[0]) with open(total_path + "/code/exposures_%s_band.dat" % ib, "w") as f: f.writelines(band_files) comm.Barrier() need_idx = [0, 1, 5, 6, 14, 21, 24, 25, 26, 27, 28, 29, 30] for ib in data_band: with open(total_path + "/code/exposures_%s_band.dat" % ib, "r") as f: cc = f.readlines() files = tool_box.alloc(cc, numprocs, "seq")[rank] exposures_candidates_avail_sub = [] for fnm in files: nm = fnm.split("\n")[0] src_path = ori_cat_path + "/%s/%s.cat" % (ib, nm) dst_path = total_path + "/%s/%s.hdf5" % (ib, nm) try: src_data = numpy.loadtxt(src_path, dtype=numpy.float32) row, col = src_data.shape except: if os.path.exists(src_path): log_inform = "%d Failed in reading %s %d Bytes !\n" % ( rank, src_path, os.path.getsize(src_path))
# [[0,-1], # [1,0]], # [[0,-1], # [-1,0]]][theta_tag] rotation = [[0, -1], [1, 0]] if rank == 0: print("rotation: %d %d\n" " %d %d" % (rotation[0][0], rotation[0][1], rotation[1][0], rotation[1][1])) if not os.path.exists(data_path + "/mix"): os.makedirs(data_path + "/mix") comm.Barrier() my_task = tool_box.alloc([i for i in range(40)], numprocs)[rank] for i in my_task: data_nm_1 = data_path + "/data_noisy_cpp_%d.hdf5" % i data_nm_2 = data_path + "/data_noise_free_%d.hdf5" % i # data_nm_2 = data_path + "/data_gal_noise_cross_term_%d.hdf5"%i data_nm_3 = data_path + "/data_pure_gal_noise_cross_term_est_r_%d.hdf5" % i # data_nm_2 = data_path + "/data_noise_free_%d.hdf5"%i h5f = h5py.File(data_nm_1, "r") # mg1 = h5f["/mg1"][()] # mg2 = h5f["/mg2"][()] mn = h5f["/mn"][()] mu = h5f["/mu"][()] mv = h5f["/mv"][()] h5f.close()
src_dec = ori_src_dec[idx] src_ra = ori_src_ra[idx] src_z = ori_src_z[idx] src_g1 = ori_src_g1[idx] src_g2 = ori_src_g2[idx] # src_halo_id = ori_src_halo_id[idx] print(idx.sum(), idx.shape[0]) com_dist_halo = cosmos.comoving_distance(halo_z).value*h0 com_dist_src = cosmos.comoving_distance(src_z).value*h0 src_position = SkyCoord(ra = src_ra*units.deg, dec = src_dec*units.deg,frame="fk5") # halo_position = SkyCoord(ra=halo_ra*units.deg, dec=halo_dec*units.deg) halo_num = halo_dec.shape[0] my_halo = tool_box.alloc([i for i in range(halo_num)], numprocs)[rank] my_cache = numpy.zeros((4, radius_bin_num)) # # itemsize = MPI.DOUBLE.Get_size() # # if rank == 0: # # bytes for 10 double elements # nbytes = 4*radius_bin_num*itemsize # else: # nbytes = 0 # # win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm) # buf1, itemsize = win1.Shared_query(0) # total_cache = numpy.ndarray(buffer=buf1, dtype='d', shape=(4, radius_bin_num))
if rank == 0: psf_img = galsim.ImageD(stamp_size, stamp_size) psf.drawImage(image=psf_img, scale=pixel_scale) psf_arr = numpy.float32(psf_img.array) hdu = fits.PrimaryHDU(psf_arr) psf_path = parent_path + '/imgs/psf.fits' hdu.writeto(psf_path, overwrite=True) gal = galsim.Sersic(scale_radius=scale_radius, n=sersic_idx, trunc=4.5 * scale_radius, flux=1.0) # task allocation chip_tags = [i for i in range(total_chips)] chip_tags_rank = tool_box.alloc(chip_tags, cpus)[rank] shear_path = parent_path + "/parameters/shear.hdf5" h5f = h5py.File(shear_path, "r") g1_input = h5f["/g1"][()] g2_input = h5f["/g2"][()] h5f.close() img_buffer = numpy.zeros((ny, nx)) counts = 0 for shear_id in range(shear_num): if rank == 0: if not os.path.exists(parent_path + "/imgs/%d" % shear_id): os.makedirs(parent_path + "/imgs/%d" % shear_id)
comm.Barrier() if cmd == "prepare": comm = MPI.COMM_WORLD rank = comm.Get_rank() cpus = comm.Get_size() total_expos = [] with open(fourier_cata_path + "/cat_inform/exposure_avail.dat", "r") as f: f_lines = f.readlines() for ff in f_lines: total_expos.append(ff.split("\n")[0]) my_expos = tool_box.alloc(total_expos,cpus, method="order")[rank] expo_avail_sub = [] exception_sub = [] for cat_path in my_expos: expo_name = cat_path.split("/")[-1] h5f_src = h5py.File(cat_path,"r") data = h5f_src["/data"][()] h5f_src.close() # selection idx1 = data[:, nstar_idx] >= nstar_thresh idx2 = data[:, flux2_alt_idx] >= flux2_alt_thresh idx3 = numpy.abs(data[:, gf1_idx]) <= gf1_thresh idx4 = numpy.abs(data[:, gf2_idx]) <= gf2_thresh
expos.append(target_path + "/" + fn) else: expos = None expos = comm.bcast(expos, root=0) expos_num = len(expos) if rank == 0: print(target_path) print(name_need) print(dst_path) print(expos_num, " exposures") if expos_num < 1: exit() my_sub_area_list = tool_box.alloc(expos, cpus)[rank] # print(rank, i, len(my_sub_area_list)) stack_tag = 0 if len(my_sub_area_list) > 0: for tag, expo_path in enumerate(my_sub_area_list): h5f = h5py.File(expo_path, "r") temp = h5f["/data"][()] h5f.close() if stack_tag == 0: stack_data = temp stack_tag = 1 else: stack_data = numpy.row_stack((stack_data, temp))
pos = [] with open(parent_path + "/cat_inform/exposure_avail_inform.dat", "r") as f: cc = f.readlines() for tag, row in enumerate(cc): if tag > 0: iterms = row.split("\n")[0].split("\t") file_name.append(iterms[0]) temp = [] for i in range(1, len(iterms) - 1): temp.append(float(iterms[i])) pos_arr = numpy.array(pos) file_num = pos_arr.shape[0] task_label = [i for i in range(file_num)] task_label_sub = tool_box.alloc(task_label, numprocs)[rank] for i in task_label_sub: t1 = time.time() h5f = h5py.File(parent_path + "/cat_hdf5/%s.hdf5" % file_name[i], "r") data = h5f["/data"][()] h5f.close() ra1 = numpy.ascontiguousarray(data[:, ra_col], dtype=numpy.float32) dec1 = numpy.ascontiguousarray(data[:, dec_col], dtype=numpy.float32) z1 = numpy.ascontiguousarray(data[:, z_col], dtype=numpy.float32) ra1_cent = ra1.mean() dec1_cent = dec1.mean()
src_path = total_path + "/cat_hdf5" pic_path = total_path + "/cat_inform/field_distortion" source_list_nm = "exposure_avail.dat" with open(total_path + "/cat_inform/"+source_list_nm, "r") as f: content = f.readlines() all_exposures_path = [] all_exposures = [] for cc in content: all_exposures.append(cc.split("/")[-1].split("\n")[0]) all_exposures_path.append(cc.split("\n")[0]) my_exposures_path = tool_box.alloc(all_exposures_path, cpus)[rank] my_exposures = tool_box.alloc(all_exposures, cpus)[rank] for tag, fnm in enumerate(my_exposures): # print(my_exposures_path[tag], fnm) h5f = h5py.File(my_exposures_path[tag], "r") data = h5f["/data"][()] h5f.close() col_shift = 16 ichip = data[:, col_shift] - 1 xc = data[:,col_shift+2] yc = data[:,col_shift+3] ra = data[:,col_shift+13]
source_list_nm = "/cat_inform/nname_field_raw_avail.dat" fields = [] with open(src_path + source_list_nm, "r") as f: conts = f.readlines() for nm in conts: field_nm = nm.split("\n")[0] if field_nm not in fields: fields.append(field_nm) if rank == 0: print("Totally ",len(fields)," fields") sub_fields = tool_box.alloc(fields, cpus)[rank] fq = Fourier_Quad(12,123) bin_num = 20 gf_bin = numpy.linspace(-0.005, 0.005, bin_num+1) gf_bin_c = (gf_bin[1:] + gf_bin[:-1])/2 result = numpy.zeros((2, bin_num)) # for fnm in sub_fields: # if os.path.exists(src_path + "/%s/result/%s_raw.hdf5" % (fnm, fnm)): # h5f = h5py.File(src_path + "/%s/result/%s_raw.hdf5" % (fnm, fnm), "r") # data = h5f["/data"][()] # h5f.close() # col_shift = 0
h5f.close() h5f = h5py.File(data_path3, "r") mg1_corr = h5f["/data"][()][:, 0] mnu1_corr = h5f["/data"][()][:, 2] + h5f["/data"][()][:, 3] h5f.close() G1_bin = tool_box.set_bin(mg1[:num_s], mg_bin_num, 1000) G1_hist_bin = gglensing_tool.set_bin(mg1[:num_s], 2500, 1.001, "log") NU1_hist_bin = gglensing_tool.set_bin(mnu1[:num_s], 2500, 1.001, "log") dilute_case = [0, 0.1, 0.2, 0.3, 0.4, 0.5] dilute_num = len(dilute_case) task_list = [i for i in range(dilute_num)] task_list_sub = tool_box.alloc(task_list, numprocs)[rank] itemsize = MPI.DOUBLE.Get_size() if rank == 0: # bytes for 10 double elements nbytes = 4 * dilute_num * itemsize else: nbytes = 0 # on rank 0 of comm, create the contiguous shared block win1 = MPI.Win.Allocate_shared(nbytes, itemsize, comm=comm) buf1, itemsize = win1.Shared_query(0) result_min = numpy.ndarray(buffer=buf1, dtype='d', shape=(4, dilute_num)) comm.Barrier()
omega_bm_num = 10 omega_bm_bin = numpy.linspace(0.01,0.1,omega_bm_num) omega_cm_num = 91 omega_cm_bin = numpy.linspace(0.1,1,omega_cm_num) sigma8_num = 71 sigma8_bin = numpy.linspace(0.3, 1,sigma8_num) chisqs = numpy.zeros((omega_bm_num, omega_cm_num, sigma8_num)) ijk = [] for i in range(omega_bm_num): for j in range(omega_cm_num): for k in range(sigma8_num): ijk.append([i,j,k]) ijk_sub = tool_box.alloc(ijk, numprocs)[rank] h5f = h5py.File("./%s/zhist.hdf5"%data_path,"r") zehist = h5f["/zhist"][()] zebin = h5f["/zbin"][()] zebin_cent = h5f["/zbin_cent"][()] h5f.close() redshift_bin_num = zehist.shape[0] tomo_panel_num = int((redshift_bin_num * redshift_bin_num + redshift_bin_num) / 2) resample_num = 200 h5f = h5py.File("./%s/result_cache_%d_%s.hdf5"%(data_path,resample_num, expo_type), "r")
pz_data = h5f["/data"][()] h5f.close() # print("%d Read Pz cata"%rank) comm.Barrier() comm.Barrier() # zbin = numpy.array([0.025 + i * 0.05 for i in range(70)]) exposures_candidates = [] with open(total_path + "/cat_inform/exposure_name.dat", "r") as f: contents = f.readlines() for expo_name in contents: exposures_candidates.append(expo_name.split("\n")[0]) exposures_candidates_sub = tool_box.alloc(exposures_candidates, numprocs, "seq")[rank] exposures_candidates_avail_sub = [] exception_sub = [] for fns in exposures_candidates_sub: # read the the field data expo_src_path = total_path + "/cat_ori/%s.cat" % fns expo_h5_path = total_path + "/cat_hdf5/%s.hdf5" % fns try: src_data = numpy.loadtxt(expo_src_path, dtype=numpy.float32) row, col = src_data.shape except: if os.path.exists(expo_src_path): log_inform = "%d Failed in reading %s %d Bytes !\n" % ( rank, expo_src_path, os.path.getsize(expo_src_path))
# f.writelines(field_nm) total_path = argv[1] dst_path = total_path + "/cat_hdf5/stack_expo" comm = MPI.COMM_WORLD rank = comm.Get_rank() numprocs = comm.Get_size() with open("/home/hklee/work/CFHT/field_expo.dat", "r") as f: field_expo = f.readlines() if rank == 0: print(len(field_expo)) field_expo_sub = tool_box.alloc(field_expo, numprocs)[rank] if rank == 0: if os.path.exists(dst_path): shutil.rmtree(dst_path, ignore_errors=True) os.mkdir(dst_path) field_avail_sub = [] for fnm in field_expo_sub: temp_str = fnm.split() field_name = temp_str[0] max_len = 0 data_list = []
if field in f_fields: pre_fields.append(field) # loop the areas, # the catalog of each area will be stored in "w_i" num = [] for area_id in range(1, area_num + 1): # distribution the files field_tar = [] for field in pre_fields: if "w%d" % area_id in field: field_tar.append(field) # if fields are more than the threads, # some thread will get an empty "field_pool" field_pool = tool_box.alloc(field_tar, cpus)[rank] # check anothers = ["w%d" % i for i in range(1, area_num + 1) if i != area_id] for field_name in field_pool: for ext_field in anothers: if ext_field in field_name: print("WRONG", rank, area_id, field_name) exit() field_count = 0 cata_data = None for field_name in field_pool: # read the catalogs from Fourier_Quad and CFHT # for the old version catalog f_data_path = fourier_cata_path + "%s/result_ext/%s_shear.dat" % (
for i in range(4): h5f = h5py.File(data_path + "/group_predict_%d.hdf5"%i,"r") label = h5f["/data"][()] h5f.close() h5f = h5py.File(data_path + "/stack_data_%d.hdf5"%i, "r") data = h5f["/data"][()][:,:4] h5f.close() label_min, label_max = label.min(), label.max() label_list = [i for i in range(label_min, label_max+1)] label_list_sub = tool_box.alloc(label_list, cpus)[rank] results = numpy.zeros((len(label_list_sub), 5)) sp = (len(label_list_sub), 5) for tag, ib in enumerate(label_list_sub): idx = label == ib data_sub = data[idx] mg1, mg2, mn, mu = data_sub[:,0],data_sub[:,1],data_sub[:,2],data_sub[:,3] gh1, gh1_sig = FQlib.find_shear_cpp(data_sub[:,0], data_sub[:,2] + data_sub[:,3],20)[:2] gh2, gh2_sig = FQlib.find_shear_cpp(data_sub[:,1], data_sub[:,2] - data_sub[:,3],20)[:2] results[tag] = ib, gh1, gh1_sig, gh2, gh2_sig