############################# Fourier_Quad Option ################################################
logger = tool_box.get_logger("%s/work/test/log/%d.dat" % (my_home, rank))
############################# Fourier_quad data collection #######################################
# combine the data of each field into one big catalog
if cmd == "collect":
t1 = time.time()
h5f_path = fourier_cata_path + "fourier_cata.hdf5"
if rank == 0:
h5f = h5py.File(h5f_path, "w")
h5f.close()
c_dicts, c_fields = tool_box.field_dict(cfht_cata_path + "nname.dat")
f_dicts, f_fields = tool_box.field_dict(fourier_cata_path + "nname.dat")
# choose the fields both exist in the two catalog
pre_fields = []
for field in c_fields:
if field in f_fields:
pre_fields.append(field)
# loop the areas,
# the catalog of each area will be stored in "w_i"
num = []
for area_id in range(1, area_num + 1):
# distribution the files
field_tar = []
for field in pre_fields:
from sys import path
path.append("%s/work/mylib/" % my_home)
import tool_box
from subprocess import Popen
from mpi4py import MPI
import time
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
ts = time.time()
data_path = "/mw/w1234/original/"
nm_path = data_path + "nname.dat"
fields = tool_box.field_dict(nm_path)[1]
missions = tool_box.allot(fields, cpus)[rank]
store_path = "/mnt/ddnfs/data_users/hkli/CFHT/catalog/"
for dirs in ["result", "result_int", "result_ext"]:
for field in missions:
try:
cmd = "scp -r /mw/w1234/original/%s/%s/ " \
"[email protected]:/mnt/ddnfs/data_users/hkli/CFHT/catalog/%s/"\
%(field, dirs, field)
# print(cmd)
a = Popen(cmd, shell=True)
a.wait()
except:
# The new Fourier_Quad catalog differs from the old version!!!
# collect: collect the data from the files of each field. It creates the "fourier_cata.hdf5" in
# the parent directory of the one contain the field catalog.
# If the catalog file doesn't exist, run it firstly !!!.
# It will add the redshift parameters from CFHT catalog into the finial catalog.
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
data_path = "/mnt/perc/hklee/CFHT/catalog/fourier_cata_new/"
raw_cata_path = data_path + "raw_cata_new/"
dicts, fields = tool_box.field_dict(data_path + "nname.dat")
my_field = tool_box.allot(fields, cpus)[rank]
chip_num = 36
for field_nm in my_field:
field_path = raw_cata_path + "%s/"%field_nm
files = os.listdir(field_path)
chip_exps = []
for nm in files:
if ".dat" in nm:
exp_nm = nm.split("p")[0]
if exp_nm not in chip_exps:
chip_exps.append(exp_nm)
contents = f.readlines()
for path in contents:
if "cfht_data_path" in path:
data_path = path.split("=")[1]
elif "cfht_res_path" in path:
result_path = path.split("=")[1]
elif "cfht_pic_path" in path:
pic_path = path.split("=")[1]
elif "cfht_field_path" in path:
field_path = path.split("=")[1]
size = 48
fq = Fourier_Quad(size, 123)
nname_path = data_path + "nname.dat"
field_dict, fields = tool_box.field_dict(nname_path)
r_fields = tool_box.allot(fields, cpus)[rank]
# for the stacking process
count = 0
# the location of each galaxy is labeled by the field_label and exposure_label
# counting from the left, the first, third and fifth figure denotes "w_m(p)_(m)p_"
# the second and the fourth denotes "m" or "p" (1=m,0=p)
# the last two figure denote the chip NO.
for field in r_fields:
expos = list(field_dict[field].keys())
field_label = tool_box.cfht_label(field)
for expo in expos:
expo_label = int(expo.split("p")[0])
chips = field_dict[field][expo]
# if os.path.exists(chip_path):
# all_files.append(chip_nm + "\n")
# # print(chip_nm)
# else:
# print("Can't find %s"%chip_path)
with open(total_path + "/cat_inform/nname_field_chips.dat", "w") as f:
f.writelines(all_files)
with open(total_path + "/cat_inform/nname_field.dat", "w") as f:
f.writelines(fields)
print(len(fields))
elif mode == "hdf5_cata":
# convert the .dat to .hdf5
fields, field_name = tool_box.field_dict(
total_path + "/cat_inform/nname_field_chips.dat")
# if rank == 0:
# print(len(field_name))
field_name_sub = tool_box.alloc(field_name, cpus, "seq")[rank]
fields_expo_sub_avail = []
fields_expo_sub_raw_avail = []
fields_sub_avail = []
fields_raw_sub_avail = []
exception_sub = []
for fns in field_name_sub:
# read the the field data
field_src_path = total_path + "/%s/result" % fns
try:
# stack the CFHT shear catalog files into one big .hdf5 file.
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
ts = time.clock()
envs_path = "%s/work/envs/envs.dat" % my_home
get_contents = [['cfht', "cfht_path_catalog", '1'],
['cfht', "cfht_path_result", '1']]
path_items = tool_box.config(envs_path, ['get', 'get'], get_contents)
total_cata_path, result_path = path_items
field_dict, fields = tool_box.field_dict(total_cata_path +
"cfht_cata/nname.dat")
area_num = 4
if rank == 0:
h5f = h5py.File(total_cata_path + "cfht_cata/cata.hdf5", "w")
h5f.close()
gal_count = 0
num_in_field = []
for area_id in range(1, 1 + area_num):
field_paths = []
field_count = 0
for field in fields:
if "w%d" % area_id in field:
field_path = total_cata_path + "cfht_cata/%s.dat" % field
import h5py
from astropy.io import fits
import matplotlib.pyplot as plt
cmd = argv[1]
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
# cpus = 1
# rank = 0
log_path = "./log_%d.dat" % rank
logger = tool_box.get_logger(log_path)
nm_path = "/mw/w1234/original/nname.dat"
all_expos, all_fields = tool_box.field_dict(nm_path)
fields = tool_box.allot(all_fields, cpus)[rank]
chip_data_path = "/mw/w1234/original/"
result_path = "/lmc/cfht/para_fit/"
my, mx = numpy.mgrid[0:4644, 0:2112]
myf = my.flatten()
mxf = mx.flatten()
tags = numpy.arange(0, len(myf))
if cmd == "files":
if rank == 0:
for field_name in all_fields:
pic_field_path = result_path + "pic/" + '%s/' % field_name
if not os.path.exists(pic_field_path):
my_home = os.popen("echo $MYWORK_DIR").readlines()[0][:-1]
from sys import path
path.append('%s/work/mylib/' % my_home)
import tool_box
from mpi4py import MPI
import numpy
import time
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
cpus = comm.Get_size()
dfy_path = "/home/dfy/data/CFHTLens/split-point/"
my_path = "/mnt/perc/hklee/CFHT/catalog/cfht_cata/field_dat/"
pre_fields = tool_box.field_dict(
"/mnt/perc/hklee/CFHT/catalog/cfht_cata/nname.dat")[1]
my_field = tool_box.allot(pre_fields, cpus)[rank]
nms = ["w1p4p0", "w1p3p0"]
for field_nm in my_field:
my_cata = numpy.loadtxt(my_path + field_nm + ".dat")
if field_nm == "w1p4p0":
dfy_field_nm = "w1p4m0"
elif field_nm == "w1p3p0":
dfy_field_nm = "w1p3m0"
else:
dfy_field_nm = field_nm
data_name, g1num, g2num, bin_num, thresh = argv[1], argv[2], argv[3], argv[
4], float(argv[5])
g1num, g2num, bin_num = int(g1num), int(g2num), int(bin_num)
with open("/home/hkli/work/envs/envs.dat", "r") as f:
contents = f.readlines()
for path in contents:
if "cfht_data_path" in path:
total_path = path.split("=")[1]
elif "cfht_res_path" in path:
result_path = path.split("=")[1]
field_path = result_path + "field/"
nname_path = total_path + "nname.dat"
all_fields = tool_box.field_dict(nname_path)[1]
filter_path = result_path + "field/filtered.dat"
filter_exist = os.path.exists(filter_path)
fq = Fourier_Quad(48, 123)
cache_path = result_path + data_name
print(cache_path)
arr = numpy.load(cache_path)["arr_1"]
est_g1 = arr[:g1num, 0]
fd_g1 = arr[:g1num, 6]
est_g2 = arr[:g2num, 3]
fd_g2 = arr[:g2num, 7]
dg1 = fd_g1[1] - fd_g1[0]
dg2 = fd_g2[1] - fd_g2[0]
logger = tool_box.get_logger("%s/work/test/log/%d.dat" % (my_home, rank))
############################# CFHTLenS catalog collection ########################################
# combine the data of each field into one big catalog
if cmd == "collect":
t1 = time.time()
logger.info("RANK: %03d. Begin..." % rank)
h5f_path = cfht_cata_path + "cfht_cata.hdf5"
if rank == 0:
h5f = h5py.File(h5f_path, "w")
h5f.close()
pre_fields = tool_box.field_dict(cfht_cata_path + "nname.dat")[1]
# loop the areas,
# the catalog of each area will be stored in "w_i"
num = []
for area_id in range(1, area_num + 1):
# distribution the files
field_tar = []
for field in pre_fields:
if "w%d" % area_id in field:
field_tar.append(field)
# if fields are more than the threads,
# some thread will get an empty "field_pool"
field_pool = tool_box.allot(field_tar, cpus)[rank]
