crit_z = float(argv[2])
file_name = argv[3]
data_path = "/mnt/ddnfs/data_users/hkli/CFHT/gg_lensing/result/mass_map/CFHT_cluster/w_1/filter_%s/source_%s/"%(file_name,source_no)
h5f = h5py.File("/mnt/ddnfs/data_users/hkli/CFHT/gg_lensing/data/redshift.hdf5","r")
# data_path = "G:\Backup\works\Mass_Mapping/source_2/".replace("\\","/")
# h5f = h5py.File("G:/Backup/works/Mass_Mapping/redshift.hdf5","r")
dist_refer = h5f["distance"].value
red_refer = h5f["redshift"].value
h5f.close()
gets_item = [["para", "nx", "0"], ["para", "ny", "0"],
["para", "delta RA (arcmin)", "0"],["para", "delta Dec (arcmin)", "0"]]
para_items = tool_box.config(data_path+"result.dat", ["get" for i in range(len(gets_item))], gets_item)
nx, ny, d_dx, d_dy = int(para_items[0]), int(para_items[1]), float(para_items[2]),float(para_items[3])
np_data = numpy.load(data_path + "result.npz")
block_num = 10
data_set = np_data["arr_0"]
sp = data_set.shape
redshift_bin_num = int(sp[0]/block_num/ny)
ra_bin = np_data["arr_1"]
dec_bin = np_data["arr_2"]
redshift_bin = np_data["arr_3"]
import tool_box
import time
sources = ["/mnt/ddnfs/data_users/hkli/selection_bias/paper_data/pts_dimmer_new",
"/mnt/ddnfs/data_users/hkli/selection_bias/paper_data/pts_bright_new"]
max_radius = 6
cpus = 60
num = 40
jobs = numpy.zeros((num, 1))
t1 = time.time()
for source in sources:
# the parameters
para_contents = [["para", "noise_sig", 1], ["para","shear_num",1]]
para_items = tool_box.config(source + "/parameters/para.ini", ['get',"get"], para_contents)
noise_sig = float(para_items[0])
shear_num = int(para_items[1])
# while True:
# if os.path.exists("%s/work/test/job/%s/finish.dat"%(my_home, source)):
# break
# while True:
# if jobs.sum() == num and os.path.exists("%s/work/test/job/%s/finish.dat"%(my_home, source)):
# break
# for i in range(num):
# if os.path.exists("%s/work/test/job/%s/finish_%d.dat"%(my_home, source, i)):
# jobs[i, 0] = 1
import matplotlib.pyplot as plt
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
cut, filter_name, source, sig, r_thresh = argv[1], argv[2], argv[3], argv[
4], float(argv[5])
t1 = time.clock()
env_path = "%s/work/envs/envs.dat" % my_home
path_items = tool_box.config(
env_path, ['get', 'get', 'get'],
[['selection_bias', "%s_path" % source, '1'],
['selection_bias', "%s_path_result" % source, '1'],
['selection_bias', "%s_path_para" % source, '1']])
total_path, result_path, para_path = path_items
shear_path = para_path + "shear.npz"
shear = numpy.load(shear_path)
fg1 = shear["arr_0"]
if len(fg1.shape) == 2:
fg1 = fg1[:, 0]
fg2 = shear["arr_1"][:, 0]
else:
fg2 = shear["arr_1"]
sex_path = total_path + "result/data/%s/sex_%d.npz" % (filter_name, rank)
para_h5_path = para_path + "para_%d.hdf5" % rank
import matplotlib.pyplot as plt
import tool_box
from mpi4py import MPI
import h5py
import time
para_path, loops = argv[1], int(argv[2])
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
para_ini_path = para_path + "/para.ini"
paras = tool_box.config(
para_ini_path, ["get", 'get', "get", 'get', "get", 'get'],
[["para", "total_num", "0"], ["para", "stamp_size", "0"],
["para", "mag_s", "0"], ["para", "mag_e", "0"], ["para", "radius_s", "0"],
["para", "radius_e", "0"]])
num = int(paras[0])
size = int(paras[1])
num_i = int(num / loops)
mag_s, mag_e = float(paras[2]), float(paras[3])
radius_s, radius_e = float(paras[4]), float(paras[5])
stamp_num = 10000
if rank == 0:
print(num * stamp_num, size, mag_s, mag_e, radius_s, radius_e)
comm.Barrier()
pic = para_path + "/pic/ellip_%d.png" % rank
plt.figure(figsize=(16, 16))
from subprocess import Popen
import os
from sys import path
my_home = os.popen("echo $MYWORK_DIR").readlines()[0][:-1]
path.append('%s/work/fourier_quad/' % my_home)
import tool_box
import shutil
envs_path = "%s/work/envs/envs.dat" % my_home
get_contents = [['cfht', "cfht_path_cut", '1']]
gets = ["get" for i in range(len(get_contents))]
cut_path = tool_box.config(envs_path, gets, get_contents)[0]
print(cut_path)
cut = "flux_alt"
stars = [10, 12, 14, 16, 18]
areas = [1, 3, 5, 7, 9]
for i in stars:
for j in areas:
old_nm = cut_path + cut
new_nm = cut_path + cut + "_s%d_a%d" % (i, j)
if os.path.exists(old_nm):
# remove the old directory
shutil.rmtree(old_nm)
# build new one
os.mkdir(old_nm)
else:
os.mkdir(old_nm)
cmd = "mpirun -np 50 python sym_mc_plot_cfht.py %s %d %d result_int" % (
cut, i, j)
rank = comm.Get_rank()
cpus = comm.Get_size()
ts = time.clock()
source = argv[1]
envs_path = "%s/work/envs/envs.dat" % my_home
get_contents = [['selection_bias', "%s_path" % source, '1'],
['selection_bias',
"%s_path_result" % source, '1'],
['selection_bias',
"%s_path_para" % source, '1'],
['selection_bias',
"%s_path_log" % source, '1']]
path_items = tool_box.config(envs_path, ['get', 'get', 'get', 'get'],
get_contents)
total_path, result_path, para_path, log_path = path_items
logger = tool_box.get_logger(log_path + "%d_logs.dat" % rank)
# the parameters
para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1],
["para", "stamp_col", 1], ["para", "shear_num", 1],
["para", "noise_sig", 1], ["para", "pixel_scale", 1]]
para_items = tool_box.config(para_path + "para.ini",
['get', 'get', 'get', 'get', 'get', 'get'],
para_contents)
total_chips_num = int(para_items[0])
stamp_size = int(para_items[1])
stamp_col = int(para_items[2])
# shear field
g = MCMC_program.shear_field(profile_params, x, y)
contents = [['param', "grid_nx", '%d' % nx], ['param', "grid_ny",
'%d' % ny],
['param', "RA", '%.2f' % delta_ra],
['param', "DEC", '%.2f' % delta_dec],
['param', "pixel_scale",
'%.6f' % pixel_scale], ['param', "sigma",
'%.4f' % sigma],
['param', "amplitude", '%.2f' % amplitude],
['param', "dx", '%.4f' % dx], ['param', "dy",
'%.2f' % dy],
['param', "density/arcmin^2",
'%d' % dens_num]]
path_items = tool_box.config(envs_path, ['add' for i in range(len(contents))],
contents, True)
img = Image_Plot()
img.subplots(2, 2)
inverse = range(nx - 1, -1, -1)
for i in range(2):
for j in range(2):
tag = i * 2 + j
if tag < 3:
fig = img.axs[i][j].imshow(g[tag][inverse], cmap="jet")
img.figure.colorbar(fig, ax=img.axs[i][j])
else:
num_dens = numpy.histogram2d(gal_coord[0], gal_coord[1],
[ra_bin, dec_bin])[0]
fig = img.axs[i][j].imshow(num_dens, cmap="jet")
img.figure.colorbar(fig, ax=img.axs[i][j])
['cfht', "cfht_path_result", '1'],
['cfht', "cfht_path_data", '1'],
["%s" % fresh, "nstar", "1"],
["%s" % fresh, "total_area", "1"],
["%s" % fresh, "flux2", '1'],
["%s" % fresh, "flux_alt", '1'],
["%s" % fresh, "gf1", '1'],
["%s" % fresh, "gf2", '1'],
["%s" % fresh, "g1", '1'],
["%s" % fresh, "g2", '1'],
["%s" % fresh, "de", '1'],
["%s" % fresh, "h1", '1'],
["%s" % fresh, "h2", '1'],
]
path_items = tool_box.config(envs_path,
["get" for i in range(len(get_contents))],
get_contents)
result_path, data_path = path_items[0], path_items[1]
# column labels
nstar_lb = int(path_items[2])
area_lb = int(path_items[3])
flux2_lb = int(path_items[4])
flux_alt_lb = int(path_items[5])
gf1_lb = int(path_items[6])
gf2_lb = int(path_items[7])
g1_lb = int(path_items[8])
g2_lb = int(path_items[9])
de_lb = int(path_items[10])
h1_lb = int(path_items[11])
rank = comm.Get_rank()
cpus = comm.Get_size()
cmd = argv[1]
# mpirun -np 1 ....py grid
# mpirun -np 12 ....py filter cal
# mpirun -np 1 ....py filter plot
#
parent_path = "/mnt/perc/hklee/CFHT/multi_shear/cluster_field/"
envs_path = parent_path + "param_slope.dat"
contents = [['param', "RA", "1"], ['param', "DEC", "1"], ['param', "a1", "1"],
['param', "a2", "1"], ['param', "a3", "1"]]
var_items = tool_box.config(envs_path, ['get' for i in range(len(contents))],
contents)
# the field size in unit of arcmin
delta_ra = float(var_items[0])
delta_dec = float(var_items[1])
parameters = [float(var_items[2]), float(var_items[3]), float(var_items[4])]
# read the data
fq = Fourier_Quad(10, 123)
read_expo = 1
for i in range(read_expo):
h5f = h5py.File(parent_path + "result/expo_%d_slope.hdf5" % i, "r")
temp = h5f["/data"].value
h5f.close()
if i == 0:
data = temp
import h5py
warnings.filterwarnings("error")
# stack the CFHT shear catalog files into one big .hdf5 file.
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
ts = time.clock()
envs_path = "%s/work/envs/envs.dat" % my_home
get_contents = [['cfht', "cfht_path_catalog", '1'],
['cfht', "cfht_path_result", '1']]
path_items = tool_box.config(envs_path, ['get', 'get'], get_contents)
total_cata_path, result_path = path_items
field_dict, fields = tool_box.field_dict(total_cata_path +
"cfht_cata/nname.dat")
area_num = 4
if rank == 0:
h5f = h5py.File(total_cata_path + "cfht_cata/cata.hdf5", "w")
h5f.close()
gal_count = 0
num_in_field = []
for area_id in range(1, 1 + area_num):
field_paths = []
import matplotlib.pyplot as plt
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
area_num = 4
area_id = 1
result_source = "result_ext"
envs_path = "%s/work/envs/envs.dat" % my_home
gets_item = [["cfht", "cfht_path_catalog", "0"],
["gg_lensing", "ggl_path_data", "0"]]
path_items = tool_box.config(envs_path, ["get", "get"], gets_item)
cata_path, data_path = path_items
cfht_cata_path = cata_path + "cfht_cata/"
pic_path = data_path + "pic/"
gets_item = [["fresh_para_idx", "nstar", "0"],
["fresh_para_idx", "flux_alt", "0"],
["fresh_para_idx", "ra", "0"], ["fresh_para_idx", "dec", "0"],
["fresh_para_idx", "gf1", "0"], ["fresh_para_idx", "gf2", "0"],
["fresh_para_idx", "g1", "0"], ["fresh_para_idx", "g2", "0"],
["fresh_para_idx", "de", "0"], ["fresh_para_idx", "h1", "0"],
["fresh_para_idx", "h2", "0"],
["fresh_para_idx", "total_area", "0"]]
ts = time.clock()
total_path = argv[1]
source = total_path.split("/")[-1]
result_path = total_path + "/result"
para_path = total_path + "/parameters"
log_path = total_path + "/logs"
logger = tool_box.get_logger(log_path + "/%d_logs.dat" % rank)
# the parameters
para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1],
["para", "stamp_col", 1], ["para", "shear_num", 1],
["para", "noise_sig", 1], ["para", "pixel_scale", 1]]
para_items = tool_box.config(para_path + "/para.ini",
['get', 'get', 'get', 'get', 'get', 'get'],
para_contents)
total_chips_num = int(para_items[0])
stamp_size = int(para_items[1])
stamp_col = int(para_items[2])
shear_num = int(para_items[3])
noise_sig = int(para_items[4])
pixel_scale = float(para_items[5])
stamp_num = 10000
# finish_path = "%s/work/test/job/%s/finish_%d.dat"%(my_home, source, rank)
# if rank == 0:
# indicator = "%s/work/test/job/%s"%(my_home, source)
# if os.path.exists(indicator):
# shutil.rmtree(indicator)
str(2 * delta_ra)],
["para", "Dec width (arcmin)",
str(2 * delta_dec)],
["para", "delta RA (arcmin)",
str(ra_bin[1] - ra_bin[0])],
["para", "delta Dec (arcmin)",
str(dec_bin[1] - dec_bin[0])],
["para", "max radius (arcsec)",
str(radius * 60)],
["para", "smooth radius (arcsec)",
str(smooth_len * 60)], ["para", "ny", str(ny)],
["para", "nx", str(nx)], ["para", "delta Z",
str(delta_z)],
["para", "block num", str(block_num)]]
cmd = ["add" for i in range(len(paras))]
tool_box.config(log_path, cmd, paras, write=True)
it2 = time.time()
# print(rank, "%d galaxy: %.2f sec" %(igal, it2 - it1))
comm.Barrier()
comm.Barrier()
t3 = time.time()
print(rank, "total: %.2f sec,%.2f sec" % (t2 - t1, t3 - t2))
# # rotation
# center = SkyCoord(ra=foreground_ra*astro_unit.degree,
# dec=foreground_dec*astro_unit.degree, frame='icrs')
#
# background_pos = SkyCoord(ra=data[:, names.index("RA")]*astro_unit.degree,
# dec=data[:, names.index("DEC")]*astro_unit.degree, frame='icrs')
#
data_path = "/mnt/perc/hklee/CFHT/gg_lensing/data/"
cfht_cata_path = cata_path + "cfht_cata/"
fourier_cata_path = cata_path + "%s/" % origin_data
envs_path = "%s/work/envs/envs.dat" % my_home
gets_item = [["fresh_para_idx", "nstar", "0"],
["fresh_para_idx", "flux_alt", "0"],
["fresh_para_idx", "ra", "0"], ["fresh_para_idx", "dec", "0"],
["fresh_para_idx", "gf1", "0"], ["fresh_para_idx", "gf2", "0"],
["fresh_para_idx", "g1", "0"], ["fresh_para_idx", "g2", "0"],
["fresh_para_idx", "de", "0"], ["fresh_para_idx", "h1", "0"],
["fresh_para_idx", "h2", "0"],
["fresh_para_idx", "total_area", "0"]]
gets = ["get" for i in range(len(gets_item))]
para_items = tool_box.config(envs_path, gets, gets_item)
############################# Fourier_Quad Option ################################################
# the column label in Fourier_Quad catalog,
# used in Fourier_quad related option
nstar_lb = 3
total_area_lb = 7
flux2_lb = 10
flux_alt_lb = 11
ra_lb = 12
dec_lb = 13
field_g1_lb = 14
field_g2_lb = 15
select_cri, nstar_thres, area_thres, result_source = argv[1], int(argv[2]), int(argv[3]), argv[4]
ts = time.time()
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
envs_path = "%s/work/envs/envs.dat" % my_home
fresh = "fresh_para_idx"
get_contents = [['cfht', "cfht_path_result", '1'],['cfht', "cfht_path_data", '1'],['cfht', "cfht_path_cut", '1'],
["%s" % fresh, "nstar", "1"], ["%s" % fresh, "total_area", "1"],
["%s" % fresh, "flux2", '1'], ["%s" % fresh, "flux_alt", '1']]
gets = ["get" for i in range(len(get_contents))]
path_items = tool_box.config(envs_path, gets, get_contents)
result_path, data_path, cut_path = path_items[0:3]
nstar_lb, total_area_lb = int(path_items[3]), int(path_items[4])
flux2_lb, flux_alt_lb = int(path_items[5]), int(path_items[6])
get_contents = [["%s"%fresh, "g1_num", '1'], ["%s"%fresh, "g2_num", '1'],
["%s"%fresh, "g1_e", '1'], ["%s"%fresh, "g2_e", '1']]
gets = ["get" for i in range(len(get_contents))]
path_items = tool_box.config(envs_path, gets, get_contents)
# parameters for segment
g1_num = int(path_items[0])-1
g2_num = int(path_items[1])-1
g1_e = float(path_items[2])
g2_e = float(path_items[3])
import tool_box
import numpy
from sys import argv
import galsim
from Fourier_Quad import Fourier_Quad
from astropy.io import fits
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
source = argv[1]
ini_path = "%s/work/envs/envs.dat" % my_home
total_path, para_path = tool_box.config(
ini_path, ['get', 'get'],
[['selection_bias', "%s_path" % source, '1'],
['selection_bias', "%s_path_para" % source, '1']])
log_name = total_path + "logs/R_%02d.dat" % rank
logger = tool_box.get_logger(log_name)
# the parameters
para_contents = [["para", "total_num", 1], ["para", "stamp_size", 1],
["para", "stamp_col", 1], ["para", "shear_num", 1],
["para", "noise_sig", 1], ["para", "pixel_scale", 1]]
para_items = tool_box.config(para_path + "para.ini",
['get', 'get', 'get', 'get', 'get', 'get'],
para_contents)
total_chip_num = int(para_items[0])
stamp_size = int(para_items[1])
import matplotlib.pyplot as plt
import numpy
import time
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
area_num = 4
result_source = "result_ext"
envs_path = "%s/work/envs/envs.dat" % my_home
gets_item = [["gg_lensing", "ggl_path_data", "0"]]
path_items = tool_box.config(envs_path, ["get"], gets_item)
data_path = path_items[0]
h5f_path_cut = data_path + "cata_%s_cut.hdf5" % result_source
h5f_path_grid = data_path + "cata_%s_grid.hdf5" % result_source
logger = tool_box.get_logger(
"/home/hklee/work/CFHT/gg_lensing/log/check_%d.dat" % rank)
pre_h5f = h5py.File(h5f_path_cut, "r")
grid_h5f = h5py.File(h5f_path_grid, "r")
data_name = [
"Z", "DISTANCE", "RA", "DEC", "COS_DEC", "G1", "G2", "N", "U", "V",
"num_in_block", "block_start", "block_end", "block_boundy", "block_boundx"
