def cal_chisq(mg, mnu, bin_num, gh):
""" calculate \chi^2 of PDF_SYM """
fq = Fourier_Quad(12, 124)
bin_num2 = int(bin_num/2)
inverse = range(int(bin_num / 2 - 1), -1, -1)
mg_bin = fq.set_bin(mg, bin_num, 100)
chisq_num = gh.shape[0]
chisq = numpy.zeros((chisq_num,))
for i in range(chisq_num):
chisq[i] = fq.get_chisq(mg, mnu, gh[i], mg_bin, bin_num2, inverse, 0)
return chisq
mu_lb = int(para_items[9]) mv_lb = int(para_items[10]) z_lb = -3 mag_lb = -2 block_scale = 60 margin = 0.5 flux_alt_thresh = 8.77 nstar_thresh = 12 total_area_thresh = 7 field_g1_bound = 0.005 field_g2_bound = 0.0075 c2_correction = 0.000498 fq = Fourier_Quad(64, 123) h5f = h5py.File(data_path + "cata_result_ext.hdf5", "r") cat_data = h5f["/w_%d" % area_id].value h5f.close() # cut off flux_alt_idx = cat_data[:, flux_alt_lb] >= flux_alt_thresh nstar_idx = cat_data[:, nstar_lb] >= nstar_thresh total_area_idx = cat_data[:, total_area_lb] >= total_area_thresh fg1 = numpy.abs(cat_data[:, field_g1_lb]) fg2 = numpy.abs(cat_data[:, field_g2_lb]) fg1_idx = fg1 <= field_g1_bound fg2_idx = fg2 <= field_g2_bound
tool_box.mag_to_flux(24)
])
else:
flux = numpy.array([
tool_box.mag_to_flux(22.7),
tool_box.mag_to_flux(23.83),
tool_box.mag_to_flux(24.02),
tool_box.mag_to_flux(24.12)
])
flux_num = len(flux)
noise_sig = 60
detect_thresh = 2
fq = Fourier_Quad(size, seed)
fq_p = Fourier_Quad(size, 2814)
# all the images are added by the same noise
noise = fq.draw_noise(0, noise_sig)
shear_beta = numpy.linspace(0, numpy.pi, num)
input_g = numpy.linspace(-0.06, 0.06, num)
total_path = "./imgs/"
img_path = total_path + "fits/%d/" % file_tag
os.mkdir(img_path)
if source_label == "galsim":
psf = galsim.Moffat(beta=3.5, fwhm=psf_r, flux=1.0, trunc=psf_r * 3)
psf_img = galsim.ImageD(size, size)
psf.drawImage(image=psf_img, scale=pixel_scale)
mport galsim
import xlwt
import math
import numpy
import random
import pywt
from ellipse import *
from mytools import *
import galsim.hsm as hsm
from mydns import *
import matplotlib.pyplot as plt
from Fourier_Quad import Fourier_Quad
f = open("./20170823/20170823","w")
FQ = Fourier_Quad()
random_seed=553728
sky_level=1.e4
pixel_scale=0.28
nx=64
ny=64
gal_flux_min=1.e4
gal_flux_max=1.e5
gal_hlr_min=0.3
gal_hlr_max=1.3
gal_e_min=0.
gal_e_max=0.8
psf_fwhm=[1.5,0.7,0.6,0.5]
psf_beta=[2.4,2.5,2.3,1.5]
n = 10000
m = 20
from plot_tool import Image_Plot
import numpy
import h5py
import tool_box
from Fourier_Quad import Fourier_Quad
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
data_path = argv[1]
shear_point = rank
data_type = argv[2]
fq = Fourier_Quad(12,124)
pic_path = data_path + "/diff_%d_%s.png"%(shear_point,data_type)
chi_pic_path = data_path + "/chi_%d_%s.png"%(shear_point,data_type)
h5f = h5py.File(data_path+"/shear.hdf5","r")
g1 = h5f["/g1"][()]
g2 = h5f["/g2"][()]
h5f.close()
print("g1: %.4f g2: %.4f"%(g1[shear_point],g2[shear_point]))
if data_type == "epsf":
h5f = h5py.File(data_path+"/data_%d_noise_free_epsf.hdf5"%shear_point,"r")
print("epsf")
else:
import tool_box import GGLensing_tool import numpy import h5py import galsim from Fourier_Quad import Fourier_Quad from astropy.cosmology import FlatLambdaCDM from astropy.coordinates import SkyCoord from astropy import units from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() numprocs = comm.Get_size() fq = Fourier_Quad(12, 123) # cosmology omega_m0 = 0.31 omega_lam0 = 1 - omega_m0 h = 0.6735 C_0_hat = 2.99792458 H_0 = 100 * h coeff = 1000 * C_0_hat / h coeff_crit = C_0_hat**2 / 4 / numpy.pi / 6.674 cosmos = FlatLambdaCDM(H_0, Om0=omega_m0) # Halo parameters Mass = 10**14 # M_sun/h
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logfile = log_path + '%d_log.dat' % rank
lf = logging.FileHandler(logfile, 'w')
form = logging.Formatter('%(asctime)s - %(message)s')
lf.setFormatter(form)
logger.addHandler(lf)
stamp_size = 96
pixel_scale = 0.2
chips_num = 5
seed = rank * 3424 + 53412
chip_s_id, shear_id = divmod(rank, 14)
fq = Fourier_Quad(stamp_size, seed)
shear_cata = para_path + "shear.npz"
shear = numpy.load(shear_cata)
g1 = shear["arr_0"][shear_id]
g2 = shear["arr_1"][shear_id]
paras = para_path + "para_%d.hdf5" % shear_id
f = h5py.File(paras, 'r')
e1s = f["/e1"].value
e2s = f["/e2"].value
radius = f["/radius"].value
flux = f["/flux"].value
fbt = f['/btr'].value
f.close()
numprocs = comm.Get_size()
psf_tag = argv[1]
gal_num = int(argv[2])
ellip = float(argv[3])
shear_num = 17
g = numpy.linspace(-0.04,0.04,shear_num)
theta = numpy.random.uniform(0,numpy.pi*2,shear_num)
g1 = g*numpy.cos(2*theta)
g2 = g*numpy.sin(2*theta)
stamp_size = 128
fq = Fourier_Quad(stamp_size,432)
# parameters
si_num = 11
sr_num = 11
sersic_index = numpy.linspace(0.4, 2, si_num)
scale_radius = numpy.linspace(0.2, 1, sr_num)
pixel_scale = 0.187
psf_e = 0.1
psf_scale = 0.7
if ellip > 0.8:
gal_e = numpy.random.uniform(0.1, 0.7, gal_num)
else:
import h5py
from mpi4py import MPI
from sys import path, argv
path.append("/home/hklee/work/mylib")
import tool_box
from Fourier_Quad import Fourier_Quad
import numpy
from plot_tool import Image_Plot
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
numprocs = comm.Get_size()
fq = Fourier_Quad(12, 124)
shear_num = 20
n, m = divmod(shear_num, numprocs)
tasks = [i for i in range(shear_num)]
my_task = tool_box.allot(tasks, numprocs)[rank]
print(rank, my_task)
total_path = argv[1]
# # columns: [col_st, col_ed]
# col_st, col_ed = int(argv[2]), int(argv[3])
# # name of the sub-data file
# sep_data_nm = argv[4]
dst_nms = [
from sys import path, argv
path.append('%s/work/fourier_quad/' % my_home)
path.append("E:/Github/astrophy-research/my_lib")
from astropy.io import fits
import numpy
from Fourier_Quad import Fourier_Quad
import tool_box
import matplotlib.pyplot as plt
size1 = 32
size2 = 128
pts_n = 50
flux = tool_box.mag_to_flux(22)
fq_noise = Fourier_Quad(256, 1110)
noise = fq_noise.draw_noise(0, 60)
nsp = noise.shape
fq_pts1 = Fourier_Quad(size1, 1231)
fq_pts2 = Fourier_Quad(size2, 1231)
pts = fq_pts1.ran_pos(pts_n, 6)
gal1 = fq_pts1.convolve_psf(pts, 4, flux/pts_n)+noise[int(nsp[0]/2-size1/2):int(nsp[0]/2+size1/2),int(nsp[1]/2-size1/2):int(nsp[1]/2+size1/2)]
gal2 = fq_pts2.convolve_psf(pts, 4, flux/pts_n)+noise[int(nsp[0]/2-size2/2):int(nsp[0]/2+size2/2),int(nsp[1]/2-size2/2):int(nsp[1]/2+size2/2)]
print(gal1.shape, gal2.shape)
gal1_p = fq_pts1.pow_spec(gal1)
snr1 = fq_pts1.snr_f(gal1)
gal2_p = fq_pts2.pow_spec(gal2)
snr2 = fq_pts2.snr_f(gal2)
logger.addHandler(lf)
stamp_size = 52
pixel_scale = 0.2
psf_r = 4
psf_model = "Moffat"
p_num = 45
chip_num = 20
total_num = 200000
num_in_chip = int(total_num / chip_num)
#seeds = numpy.loadtxt("/home/hklee/seed.txt")
#(seeds.shape)
seed1 = int(rank * 10 + 41230)
seed2 = int(rank * 15 + 8986920)
fq = Fourier_Quad(stamp_size, seed1)
fqn = Fourier_Quad(stamp_size, seed2)
logger.info("%ds process: seed1: %d, seed2: %d" % (rank, seed1, seed2))
# distribute jobs
chip_paths_list = [
total_path + '%d/gal_chip_%s.fits' % (rank, str(j).zfill(3))
for j in range(chip_num)
]
# LSST
prop = lsstetc.ETC(band='r',
pixel_scale=pixel_scale,
stamp_size=stamp_size,
nvisits=180)
fgs = [(fg1+dfg1)[:g1_num], (fg2+dfg2)[:g2_num]]
gnums = [g1_num, g2_num]
cut_num = 20
if rank == 0:
print(fgs[0])
print(fgs[1])
# nstar total_area flux2 flux_alt gf1 gf2 g1(G1) g2(G2) de(N) h1(U) h2(V)
# 0 1 2 3 4 5 6 7 8 9 10
select_idx = {"flux2": 2, "flux_alt": 3}
# the column in original catalog
ori_data_labels = {"flux2": flux2_lb, "flux_alt": flux_alt_lb}
fq = Fourier_Quad(48, 123)
itemsize = MPI.DOUBLE.Get_size()
if rank == 0:
nb1 = int(itemsize*g1_num*3*cut_num)
nb2 = int(itemsize*g2_num*3*cut_num)
nb3 = int(itemsize*cut_num)
else:
nb1 = 0
nb2 = 0
nb3 = 0
win1 = MPI.Win.Allocate_shared(nb1, itemsize, comm=comm)
win2 = MPI.Win.Allocate_shared(nb2, itemsize, comm=comm)
win3 = MPI.Win.Allocate_shared(nb3, itemsize, comm=comm)
stamp_num = 10000
finish_path = "%s/work/test/job/%s/finish_%d.dat" % (my_home, source, rank)
if rank == 0:
indicator = "%s/work/test/job/%s" % (my_home, source)
if os.path.exists(indicator):
shutil.rmtree(indicator)
os.makedirs(indicator)
comm.Barrier()
total_gal_num = total_chips_num * stamp_num
seed = rank * 344 + 121
rad_n = numpy.random.randint(0, 10000, 1)[0]
ny, nx = stamp_col * stamp_size, stamp_col * stamp_size
fq = Fourier_Quad(stamp_size, seed)
# PSF
psf = galsim.Moffat(beta=3.5, fwhm=0.7, flux=1.0, trunc=2)
if rank == 0:
psf_img = galsim.ImageD(stamp_size, stamp_size)
psf.drawImage(image=psf_img, scale=pixel_scale)
hdu = fits.PrimaryHDU(psf_img.array)
psf_path = total_path + 'psf.fits'
hdu.writeto(psf_path, overwrite=True)
logger.info(
"desti: %s, size: %d, pixel_scale: %.3f, noise_sig: %.2f, total galaxy number: %d"
% (source, stamp_size, pixel_scale, noise_sig, total_chips_num))
logger.info("seed: %d" % seed)
# task allocation
parent_path = "/mnt/perc/hklee/CFHT/multi_shear/cluster_field/"
envs_path = parent_path + "param_slope.dat"
contents = [['param', "RA", "1"], ['param', "DEC", "1"], ['param', "a1", "1"],
['param', "a2", "1"], ['param', "a3", "1"]]
var_items = tool_box.config(envs_path, ['get' for i in range(len(contents))],
contents)
# the field size in unit of arcmin
delta_ra = float(var_items[0])
delta_dec = float(var_items[1])
parameters = [float(var_items[2]), float(var_items[3]), float(var_items[4])]
# read the data
fq = Fourier_Quad(10, 123)
read_expo = 1
for i in range(read_expo):
h5f = h5py.File(parent_path + "result/expo_%d_slope.hdf5" % i, "r")
temp = h5f["/data"].value
h5f.close()
if i == 0:
data = temp
else:
data = numpy.row_stack((data, temp))
mg1 = data[:, 0]
mg2 = data[:, 1]
mnu1 = data[:, 2] + data[:, 3]
mnu2 = data[:, 2] - data[:, 3]
shear_true = data[:, 5]
import time
import matplotlib.pyplot as plt
from multiprocessing import Pool
import h5py
from scipy.optimize import least_squares
import MCMC_program
expo_num = int(argv[1])
ncpus = int(argv[2])
shear_cmd = argv[3]
nwalkers = 200
ndim = 4
step = 1000
print("Walker: %d. Step: %d." % (nwalkers, step))
fq = Fourier_Quad(10, 112)
bin_num = 8
bin_num2 = int(bin_num / 2)
fit_radius = 6
# number of grid
nx = 50
ny = nx
# the field size in unit of arcmin
delta_ra = 20
delta_dec = delta_ra
half_side = nx / 2
# arcmin/pix
pixel_scale = delta_ra / nx
# parameters
sigma = 2 # arcmin
mu_all = data[:, 19][idx_]
print(src_num, "Gal")
gf1 = data[:, col_shift + 14][idx_]
gf2 = data[:, col_shift + 15][idx_]
# set up gf bins
bin_num1 = 32
bin_num2 = 32
gf1_bin = numpy.linspace(-0.005, 0.005, bin_num1 + 1, dtype=numpy.float32)
gf2_bin = numpy.linspace(-0.005, 0.005, bin_num2 + 1, dtype=numpy.float32)
gf1_pts = (gf1_bin[1:] + gf1_bin[:-1]) / 2
gf2_pts = (gf2_bin[1:] + gf2_bin[:-1]) / 2
fq = Fourier_Quad(12, 124)
for i in range(bin_num1):
idx_11 = gf1 >= gf1_bin[i]
idx_12 = gf1 < gf1_bin[i + 1]
idx = idx_11 & idx_12
num1 = idx.sum()
gh1, gh1_sig = fq.find_shear(mg2_all[idx], mn_all[idx] - mu_all[idx],
8)[:2]
print("%.5f, %.5f(%.5f) %d" % (gf1_pts[i], gh1, gh1_sig, num1))
idx_11 = gf2 >= gf2_bin[i]
idx_12 = gf2 < gf2_bin[i + 1]
idx = idx_11 & idx_12
# rfactor_path = result_path + "data/resolution_factor_%d.npz"%rank
if rank == 0:
log = "START -- %s, %7s, %9s, , %5.2f, %12s" % (
total_path.split("/")[-2], filter_name, cut, r_thresh, argv[0])
logger = tool_box.get_logger(
"%s/work/selection_bias/sym_mc_plot/cutoff.dat" % my_home)
logger.info(log)
shear_esti_h5path = result_path + "data/data_%d.hdf5" % rank
shear_esti_file = h5py.File(shear_esti_h5path, "r")
shear_esti_data = shear_esti_file["/data"].value
shear_esti_file.close()
fq = Fourier_Quad(60, 152356)
noise_sig = 60
MG1 = shear_esti_data[:, 2]
MG2 = shear_esti_data[:, 3]
MN = shear_esti_data[:, 4]
MU = shear_esti_data[:, 5]
MV = shear_esti_data[:, 6]
# be careful that the "MU" defined in FRESH is different from that in ours
# MN + (-) MU for our definition (g1(2)) of MU and MV which is the same as
# those in the paper Zhang et al. 2017 ApJ, 834:8
DE1 = MN + MU
DE2 = MN - MU
cuts_num = 10
import numpy
from sys import path
path.append("D:/Github/astrophy-research/mylib")
path.append("D:/Github/astrophy-research/multi_shear_detect")
from Fourier_Quad import Fourier_Quad
import h5py
from plot_tool import Image_Plot
import tool_box
from astropy.io import fits
import os
import matplotlib.pyplot as plt
h5f = h5py.File("E:/data_cross_term_espf_0.hdf5", "r")
data_ct = h5f["/data"][()]
h5f.close()
h5f = h5py.File("E:/data_noise_residual_espf_0.hdf5", "r")
data_nr = h5f["/data"][()]
h5f.close()
fq = Fourier_Quad(12, 124)
gh, chisq = fq.get_chisq_range(data_ct[:, 0], data_ct[:, 2])
import numpy
from sys import path
path.append("D:/Github/astrophy-research/mylib")
path.append("D:/Github/astrophy-research/multi_shear_detect")
from Fourier_Quad import Fourier_Quad
from plot_tool import Image_Plot
import matplotlib.pyplot as plt
from matplotlib import cm
psf_type = "Moffat"
psf_flux = 1
psf_scale = 4
stamp_size = 48
seed = 56525
fq = Fourier_Quad(stamp_size, seed)
img = Image_Plot()
img.subplots(1, 1)
fig = img.axs[0][0].imshow(fq.kx2 + fq.ky2)
img.figure.colorbar(fig, ax=img.axs[0][0])
img.axs[0][0].set_title("$k^2$")
img.del_ticks(0, 0, [0, 1])
img.save_img("E:/kxy.png")
img.show_img()
pst_num = 50
gal_fluxs = [4000, 16000, 32000, 100000]
steps = [0.6, 1, 2, 4]
fq = Fourier_Quad(stamp_size, seed)
ts = time.clock()
with open("%s/work/envs/envs.dat" % my_home, "r") as f:
contents = f.readlines()
for path in contents:
if "cfht_data_path" in path:
data_path = path.split("=")[1]
elif "cfht_res_path" in path:
result_path = path.split("=")[1]
elif "cfht_pic_path" in path:
pic_path = path.split("=")[1]
elif "cfht_field_path" in path:
field_path = path.split("=")[1]
size = 48
fq = Fourier_Quad(size, 123)
nname_path = data_path + "nname.dat"
field_dict, fields = tool_box.field_dict(nname_path)
r_fields = tool_box.allot(fields, cpus)[rank]
# for the stacking process
count = 0
# the location of each galaxy is labeled by the field_label and exposure_label
# counting from the left, the first, third and fifth figure denotes "w_m(p)_(m)p_"
# the second and the fourth denotes "m" or "p" (1=m,0=p)
# the last two figure denote the chip NO.
for field in r_fields:
expos = list(field_dict[field].keys())
field_label = tool_box.cfht_label(field)
idxs = numpy.linspace(0.31, 1.2, grid_num)
radii = numpy.linspace(0.2, 0.8, grid_num)
gal_flux = 6000
noise_sig = 60
shear_num = 40
pixel_scale = 0.187
stamp_size = 64
g1_result = numpy.zeros((2, shear_num))
g2_result = numpy.zeros((2, shear_num))
temp = numpy.zeros((4, 3))
fq = Fourier_Quad(stamp_size, 123)
psf = galsim.Moffat(beta=3.5, fwhm=0.7, flux=1.0, trunc=1.4).shear(e1=0.15,
e2=0.)
psf_img = galsim.ImageD(stamp_size, stamp_size)
psf.drawImage(image=psf_img, scale=pixel_scale)
psf_arr = psf_img.array
# hdu = fits.PrimaryHDU(psf_arr)
# psf_path = './psf.fits'
# hdu.writeto(psf_path, overwrite=True)
psf_pow = fq.pow_spec(psf_arr)
fq.get_radius_new(psf_pow, 2)
shear_path = "./shear.hdf5"
pixel_scale = float(para_items[5])
stamp_num = 10000
finish_path = "%s/work/test/job/%s/finish_%d.dat" % (my_home, source, rank)
if rank == 0:
indicator = "%s/work/test/job/%s" % (my_home, source)
if os.path.exists(indicator):
shutil.rmtree(indicator)
os.makedirs(indicator)
comm.Barrier()
total_gal_num = total_chips_num * stamp_num
seed = rank * 344 + 11121
ny, nx = stamp_col * stamp_size, stamp_col * stamp_size
fq = Fourier_Quad(stamp_size, seed)
# PSF
psf = galsim.Moffat(beta=3.5, fwhm=0.7, flux=1.0, trunc=2)
if rank == 0:
psf_img = galsim.ImageD(stamp_size, stamp_size)
psf.drawImage(image=psf_img, scale=pixel_scale)
hdu = fits.PrimaryHDU(psf_img.array)
psf_path = total_path + 'psf.fits'
hdu.writeto(psf_path, overwrite=True)
logger.info(
"desti: %s, size: %d, pixel_scale: %.3f, noise_sig: %.2f, total galaxy number: %d"
% (source, stamp_size, pixel_scale, noise_sig, total_chips_num))
logger.info("seed: %d" % seed)
# task allocation
from Fourier_Quad import Fourier_Quad import numpy import matplotlib.pyplot as plt from matplotlib import cm seed = 12114 total_num = 25000 uni_num = 15000 a1, a2 = 0, -0.01 fq = Fourier_Quad(6,12) rng = numpy.random.RandomState(seed) bin_num = 8 bin_num2 = int(bin_num/2) inverse = range(int(bin_num2 - 1), -1, -1) bin_num_test = 8 bin_num2_test = int(bin_num_test/2) inverse_test = range(int(bin_num2_test - 1), -1, -1) hist_bin = 20 # x = numpy.random.uniform(-8, 8, gal_num) x_asy = rng.uniform(0, 8, int(total_num - uni_num)) x_uni = rng.uniform(-8, 8, uni_num)
dg2 = g2[1] - g2[0]
t1 = time.clock()
with open("%s/work/envs/envs.dat" % my_home, "r") as f:
contents = f.readlines()
for path in contents:
if "cfht_data_path" in path:
data_path = path.split("=")[1]
elif "cfht_res_path" in path:
result_path = path.split("=")[1]
elif "cfht_pic_path" in path:
pic_path = path.split("=")[1]
elif "cfht_cut_path" in path:
cut_path = path.split("=")[1]
fq = Fourier_Quad(48, 123)
g1_data_path = result_path + "g1_%d.npz" % rank
g2_data_path = result_path + "g2_%d.npz" % rank
g1num = cpus - 6
g2num = cpus
g1 = numpy.linspace(-0.004, 0.004, g1num)
g2 = numpy.linspace(-0.0055, 0.0055, g2num)
cuts_num = 20
star_thre = 16
cut_off_scale = numpy.load(result_path + cut + '.npz')["arr_0"]
if rank < g1num:
import galsim import xlwt import math import numpy import random import pywt from ellipse import * from mytools import * import galsim.hsm as hsm from mydns import * import matplotlib.pyplot as plt from Fourier_Quad import Fourier_Quad FQ = Fourier_Quad() random_seed = 553728 sky_level = 1.e1 pixel_scale = 0.28 nx = 64 ny = 64 gal_flux_min = 1.e4 gal_flux_max = 1.e5 gal_hlr_min = 0.3 gal_hlr_max = 1.3 gal_e_min = 0. gal_e_max = 0.8 psf_fwhm = [1.5, 0.7, 0.6, 0.5] psf_beta = [2.4, 2.5, 2.3, 1.5] n = 2500 m = 4 xiuzheng = 1.82 xz = [1, -1]
if not os.path.isdir(result_path):
os.makedirs(result_path)
shear = numpy.load(
'/home/hklee/work/selection_bias/parameters/shear.npz')['arr_0']
out_shear1 = shear[0]
out_shear2 = shear[1]
chip_paths_pool = [
'/lmc/selection_bias/%d/gal_chip_%s.fits' % (i, str(j).zfill(3))
for i in range(10) for j in range(chip_num)
]
chip_paths_list = tool_box.task_distri(chip_paths_pool, CPU_num)
# psf = fits.open('/lmc/selection_bias/psf.fits')[0].data
psf = Fourier_Quad().cre_psf(4, stamp_size, 'Moffat')
prop = lsstetc.ETC(band='r',
pixel_scale=pixel_scale,
stamp_size=stamp_size,
nvisits=180)
noise_sigma = prop.sigma_sky / 15
p = Pool()
t1 = time.time()
for m in range(CPU_num):
p.apply_async(shear_est,
args=(chip_paths_list[m], psf, noise_sigma, stamp_size,
m))
p.close()
p.join()
t2 = time.time()
z_max = cata_data[:, z_max_lb_f][cut_idx]
odds = cata_data[:, odds_lb_f][cut_idx]
mag = cata_data[:, mag_lb_f][cut_idx]
names = [
"Z", "RA", "DEC", "G1", "G2", "N", "U", "V", "MAG", "COS_DEC",
"Z_MIN", "Z_MAX", "ODDS"
]
datas = [
redshift, ra, dec, mg1, mg2, mn, mu, mv, mag, cos_dec, z_min,
z_max, odds
]
fq = Fourier_Quad(12, 123)
mg1_bin = fq.set_bin(mg1, mg_bin_num, 10000)
mg2_bin = fq.set_bin(mg2, mg_bin_num, 10000)
data_num = len(redshift)
total_num = ra.shape[0]
gal_label = numpy.arange(0, data_num)
# set up RA & DEC bin
ra_min, ra_max = ra.min() - margin, ra.max() + margin
dec_min, dec_max = dec.min() - margin, dec.max() + margin
nx = int((ra_max - ra_min) / block_scale) + 2
ny = int((dec_max - dec_min) / block_scale) + 2
grid_num = nx * ny
def shear_est(chip_list, psf_in, noise_sig, size, proc_id):
print('Proc_%d: begin>>>') % proc_id
# col = ["KSB_g1", "BJ_e1", "RG_e1", "FQ_G1", "fg1", "KSB_g2", "BJ_e2", "RG_e2", "FQ_G2", "fg2",
# "FG_N", "FQ_U", "FQ_V", 'KSB_R', 'BJ_R', 'RG_R', "SNR_ORI"]
psf_pow = Fourier_Quad().pow_spec(psf_in)
# psf_g = galsim.Image(psf_in)
total_chips = len(chip_list)
for i in range(total_chips):
chip_path = chip_list[i]
shear_tag, chip_name = chip_path.split('/')[3:5]
gals = fits.open(chip_path)[0].data
gal_pool = Fourier_Quad().divide_stamps(gals, size)
# data_matrix = numpy.zeros((len(gal_pool), len(col)))
data_path = '/lmc/selection_bias/result/data/' + shear_tag + '_' + chip_name.split(
'.')[0] + '.xlsx'
ts = time.time()
data1 = numpy.zeros((len(gal_pool), 1))
data2 = numpy.zeros((len(gal_pool), 1))
data3 = numpy.zeros((len(gal_pool), 1))
for k in range(len(gal_pool)):
gal = gal_pool[k]
# gal_g = galsim.Image(gal)
#
# res_k = galsim.hsm.EstimateShear(gal_g, psf_g, shear_est='KSB', strict=False)
# ksb_g1 = res_k.corrected_g1
# ksb_g2 = res_k.corrected_g2
# ksb_r = res_k.resolution_factor
#
# res_b = galsim.hsm.EstimateShear(gal_g, psf_g, shear_est='BJ', strict=False)
# bj_e1 = res_b.corrected_e1
# bj_e2 = res_b.corrected_e2
# bj_r = res_b.resolution_factor
#
# res_r = galsim.hsm.EstimateShear(gal_g, psf_g, shear_est='REGAUSS', strict=False)
# re_e1 = res_r.corrected_e1
# re_e2 = res_r.corrected_e2
# re_r = res_r.resolution_factor
noise = numpy.random.normal(loc=0., scale=noise_sig,
size=size**2).reshape(size, size)
mg1, mg2, mn, mu, mv = Fourier_Quad().shear_est(gal,
psf_pow,
size,
noise,
F=True)
# data_matrix[k, :] = ksb_g1, bj_e1, re_e1, mg1, g1_input, ksb_g2, bj_e2, re_e2, mg2, g2_input, mn, mu, mv, ksb_r, bj_r, re_r, snr[k]
data1[k] = mg1
data2[k] = mg2
data3[k] = mn
df = pandas.read_excel(data_path)
df["FQ_G1"] = data1
df["FQ_G2"] = data2
df["FQ_N"] = data3
df.to_excel(data_path)
te = time.time()
print('Proc_%d: (%d/%d) complete within time %.2f s') % (
proc_id, i + 1, total_chips, te - ts)
para_items = tool_box.config(para_path + "para.ini",
['get', 'get', 'get', 'get', 'get', 'get'],
para_contents)
total_chip_num = int(para_items[0])
stamp_size = int(para_items[1])
columns = int(para_items[2])
shear_num = int(para_items[3])
stamp_num = 10000
psf_path = total_path + "psf.fits"
chip_labels = tool_box.allot([i for i in range(total_chip_num)], cpus)[rank]
chip_num = len(chip_labels)
fq = Fourier_Quad(stamp_size, 123)
psf_img = galsim.Image(fits.open(psf_path)[0].data)
log_informs = "RANK: %d, SOURCE: %s, TOTAL CHIPS: %d, MY CHIPS: %d (%d ~ %d)" % (
rank, source, total_chip_num, chip_num, chip_labels[0], chip_labels[-1])
logger.info(log_informs)
ts = time.time()
for i in range(shear_num):
R_factor = numpy.zeros((chip_num * stamp_num, 1))
for tag, j in enumerate(chip_labels):
log_informs = "%02d/gal_chip_%04d.fits start" % (i, j)
logger.info(log_informs)
t1 = time.time()
chip_path = total_path + "%d/gal_chip_%04d.fits" % (i, j)
chip_img = fits.open(chip_path)[0].data
gals = fq.segment(chip_img)
for k in range(len(gals)):
from sys import path, argv
path.append('%s/work/mylib/' % my_home)
import numpy
from mpi4py import MPI
import h5py
from plot_tool import Image_Plot
from Fourier_Quad import Fourier_Quad
import component_fit
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
fq = Fourier_Quad(12,1234)
data_path = argv[1]
xy_bin_num, radius_bin_num = int(argv[2]), int(argv[3])
shear_scale = float(argv[4])
pic_path = data_path + "/multipole_pic_%.1f"%shear_scale
if rank == 0:
if not os.path.exists(pic_path):
os.makedirs(pic_path)
scale = 1000