make_randoms = False
make_nz = True
mask_dir = '/Users/jackelvinpoole/DES/cats/y3//masks/lens_sample_joint_mask/'
mask_file = mask_dir + 'y3_gold_2.2.1_RING_joint_redmagic_v0.5.1_wide_maglim_v2.2_mask.fits.gz'
randfiles = {
'hidens': None,
'hilum': None,
}
########### make catalog ###########
print('highdens')
highdens = lsssys.Redmagic(highdens_file, )
highdens_zmin = binedges[np.where(np.array(catlist) == 'hidens')[0].min()]
highdens_zmax = binedges[np.where(np.array(catlist) == 'hidens')[0].max() + 1]
highdens.cutz(highdens_zmin, highdens_zmax)
highdens.cut(highdens.zlum < zlum_cut)
print('highlum')
highlum = lsssys.Redmagic(highlum_file, )
highlum_zmin = binedges[np.where(np.array(catlist) == 'hilum')[0].min()]
highlum_zmax = binedges[np.where(np.array(catlist) == 'hilum')[0].max() + 1]
highlum.cutz(highlum_zmin, highlum_zmax)
highlum.cut(highlum.zlum < zlum_cut)
if 'higherlum' in catlist:
print('higherlum')
higherlum = lsssys.Redmagic(higherlum_file, )
def main(cat_path,
mask_path,
theory_dir,
nside,
z_bin_edges,
out_dir,
chain_version=None,
data_result_dir=None,
nside_fit=None,
zbins=None,
k0=None,
nmocks=1,
min_mock=0,
lmax=3500,
n_contam=0,
gen_catalogs=False,
test=False,
force_truth=None,
force=None):
"""
Create mock density field(s) and/or catalog(s)
Note that the theory directory should be the upper-most directory for
CosmoSIS output, and should contain a directory 'galaxy_cl'
:param cat_path: The path to the catalog file
:type cat_path: ``str`` or :class:`os.PathLike`
:param mask_path: The path to the mask file
:type mask_path: ``str`` or :class:`os.PathLike`
:param theory_dir: The top level directory containing the CosmoSIS theory
output. Must contain a directory 'galaxy_cl' with files 'ell.txt' and
'bin_{i}_{j}.txt' where 'i' and 'j' are redshift bin indices
:type theory_dir: ``str`` or :class:`os.PathLike`
:param nside: The resolution at which to create the mock density field(s)
:type nside: ``int``
:param z_bin_edges: The edges of the redshift bins for all redshift bins.
Only the edges for the selected bins will be used, but all must be given
:type z_bin_edges: (``Nbins+1``,) array-like of ``float``
:param out_dir: The parent directory in which to save the mock density
map(s) and catalog(s). Make sure write permissions are available. The
actual results will be stored under here in a directory
'gaussian_mock_output' if ``k0`` is ``None`` or 'lognormal_mock_output'
otherwise: counts maps will be under that in 'ngal_maps' and
catalogs in 'catalogs'
:type out_dir: ``str`` or :class:`os.PathLike`
:param chain_version: The run version of the data chains that are used as
the "truth" for the contamination. Ignored if no contamination is added,
otherwise it **must** be specified. Default ``None``
:type chain_version: ``int`` or ``NoneType``, optional
:param data_result_dir: The directory within which the results from the data
systematics fits can be found. This is needed to ensure that the proper
coefficients are used to contaminate the mocks. The mean fit parameters
as well as the map importance order for each redshift bin should be
stored in subdirectories 'zbin#' under this directory, where '#' is
replaced by the bin numbers included from `zbins`. The names of these
files are assumed to be 'mean_parameters_nside{res}.pkl' and
'map_importance_order_const_cov_fit{res}_v{v}.npy', where 'res' is the
fitting resolution for the redshift bin and 'v' is the version of the
data fit specified by `chain_version`. Ignored if no contamination is
added, otherwise it **must** be specified. Default ``None``
:type data_result_dir: ``str`` or :class:`os.PathLike` or ``NoneType``,
optional
:param nside_fit: The resolutions used when fitting for the redshift bins.
Ignored if no contamination is being applied, otherwise it **must** be
specified. Default ``None``
:type nside_fit: ``int`` or (``Nbins``,) array-like of ``int`` or
``NoneType``, optional
:param zbins: The redshift bin(s) for which to create mocks. Assumes there
are ``Nbins`` total redshift bins, and the indexing starts from 1
(not 0!). If given, this must have length of at least 1 and no larger
than ``Nbins``. If ``None`` (default), makes a mock for all redshift
bins
:type zbins: ``NoneType`` or 1D array-like of ``int``, optional
:param k0: Parameter for the skewness of the lognormal field, which can be
different for each redshift bin, in which case it should have length
equal to ``zbins``. If ``None`` (default), creates a Gaussian density
field rather than a lognormal one
:type k0: ``NoneType`` or ``int`` or 1D array-like of ``int``, optional
:param nmocks: The number of mocks to create. Default 1
:type nmocks: ``int``, optional
:param min_mock: The number of the first mock, used to edit the random seed
and file names in the case where some mocks have already been created.
If some mocks have already been created and this is not used, an error
will be raised when trying to write the catalog. Default 0
:type min_mock: ``int``, optional
:param lmax: The maximum scale beyond which to truncate C(l). Default 3500
:type lmax: ``float``, optional
:param n_contam: The number of systematics maps by which to contaminate.
Use an array-like for multiple levels of contamination. Default 0.
:type n_contam: ``int`` or array-like of ``int``, optional
:param gen_catalogs: If ``True``, also generate mock galaxy catalog(s) from
the density field(s). Default ``False``
:type gen_catalogs: ``bool``, optional
:param test: If ``True``, set a random seed for reproducibility. Default
``False``
:type test: ``bool``, optional
:param force_truth: If ``True`` and true mock overdensity fields with the
specified numbers already exist, create new ones and overwrite the
files. This should be done with caution if adding contamination levels
to existing mocks, as it will mean the true overdensity field for
existing catalogs is overwritten. But random seed setting when `test` is
``True`` will be off when adding contamination levels unless everything
that is already stored is redone. If ``None`` (default), this is set to
``False`` if `test` is ``False`` or ``True`` if `test` is ``True``
:type force_truth: ``bool`` or ``NoneType``, optional
:param force: If ``True`` and files exist for mocks with the specified
numbers (other than the true overdensities, see `force_truth`), they
will be overwritten with the newly generated mocks rather than being
skipped. If ``None`` (default), set to the same as `force_truth`
:type force: ``bool`` or ``NoneType``, optional
"""
mask = lsssys.Mask(
pathlib.Path(mask_path).expanduser().resolve().as_posix(),
ZMAXcol=None)
cat = lsssys.Redmagic(
pathlib.Path(cat_path).expanduser().resolve().as_posix())
sorted_z = np.sort(z_bin_edges)
if zbins is not None:
if (len(zbins) >= len(sorted_z) or np.min(zbins) < 0
or np.max(zbins) > len(sorted_z)):
raise ValueError("Invalid size or value(s) for zbins")
sorted_zbins = np.sort(zbins)
zedges = np.dstack(
(sorted_z[sorted_zbins - 1], sorted_z[sorted_zbins])).squeeze()
number_density = calculate_number_density(cat, mask, zedges)[1]
else:
ngal_tot, number_density = calculate_number_density(
cat, mask, sorted_z)
sorted_zbins = np.arange(len(z_bin_edges) - 1) + 1
zedges = np.dstack((sorted_z[:-1], sorted_z[1:])).squeeze()
ngal_mean = np.atleast_1d(number_density) * hp.nside2pixarea(
nside, degrees=True) * 60.**2
output_dir = pathlib.Path(out_dir).expanduser().resolve()
del cat
if k0 is not None:
if hasattr(k0, "__len__"):
if len(k0) > 1:
if zbins is not None and len(k0) != len(zbins):
raise ValueError("Mismatch between zbins and k0")
elif zbins is None and len(k0) != (len(z_bin_edges) - 1):
raise ValueError("Mismatch between z_bin_edges and k0")
else:
k0 = k0[0]
output_dir = output_dir.joinpath("lognormal_mock_output")
ngal_fill = None
else:
output_dir = output_dir.joinpath("gaussian_mock_output")
ngal_fill = 0
output_dir.mkdir(parents=True, exist_ok=True)
np.save(output_dir.joinpath("mean_ngals.npy"), ngal_mean)
theory = initialize_theory(theory_dir, lmax, zbins, k0)
delta_output_dir = output_dir.joinpath("delta_maps")
delta_output_dir.mkdir(exist_ok=True)
ngals_output_dir = output_dir.joinpath("ngal_maps")
ngals_output_dir.mkdir(exist_ok=True)
for n in np.unique(np.append(np.atleast_1d(n_contam), 0)):
delta_output_dir.joinpath(f"n_contaminate_{n}").mkdir(exist_ok=True)
ngals_output_dir.joinpath(f"n_contaminate_{n}").mkdir(exist_ok=True)
if gen_catalogs:
noisy_ngals_output_dir = output_dir.joinpath("poisson_ngal_maps")
noisy_ngals_output_dir.mkdir(exist_ok=True)
cats_output_dir = output_dir.joinpath("catalogs")
cats_output_dir.mkdir(exist_ok=True)
for n in np.atleast_1d(n_contam):
noisy_ngals_output_dir.joinpath(f"n_contaminate_{n}").mkdir(
exist_ok=True)
cats_output_dir.joinpath(f"n_contaminate_{n}").mkdir(exist_ok=True)
if hasattr(n_contam, "__len__") or n_contam != 0:
sys_dir = pathlib.Path("/spiff/wagoner47/des/y3/systematics")
if chain_version is None:
raise ValueError(
"Must specify chain_version if contamination is required")
if data_result_dir is None:
raise ValueError(
"Must specify data_result_dir if contamination is required")
coeff_dir = pathlib.Path(data_result_dir)
if nside_fit is None:
raise ValueError(
"Must specify nside_fit if contamination is required")
if not hasattr(nside_fit, "__len__"):
nside_fit = np.full(sorted_zbins.size, nside_fit)
elif len(nside_fit) != sorted_zbins.size:
if len(nside_fit) == 1:
nside_fit = np.full(sorted_zbins.size, nside_fit[0])
else:
raise ValueError(
"Different number of specified nside_fit and redshift bins"
)
mock = lsssys.Mock(theory, nside)
mock.mask = mask.mask
if force_truth is None:
force_truth = test
if force is None:
force = force_truth
if test:
rand_states = np.arange(min_mock, nmocks + min_mock)
else:
rand_states = [None] * nmocks
for i, r_state in enumerate(
tqdm(rand_states, desc="Mock", dynamic_ncols=True), min_mock):
try:
assert not force_truth
mock = _load_deltas(mock, i, delta_output_dir / "n_contaminate_0",
sorted_zbins.size)
except (AssertionError, IOError):
mock.gen_maps(rs=np.random.RandomState(r_state))
if k0 is not None:
mock.lognormalise(
k0,
[deltai.data[~mask.mask].std() for deltai in mock.delta])
for z, deltai in enumerate(mock.delta):
deltai.save(delta_output_dir.joinpath(
"n_contaminate_0", f"mock_{i}_bin{z}.fits").as_posix(),
clobber=True)
try:
assert not force
mock = _load_ngals(mock, i, ngals_output_dir / "n_contaminate_0")
except (AssertionError, IOError):
mock.gen_ngal(ngal_mean, ngal_fill)
for z in range(len(mock.ngal)):
mock.ngal[z].data[~mask.mask] *= mask.fracdet[~mask.mask]
mock.save(ngals_output_dir.joinpath("n_contaminate_0",
f"mock_{i}.fits").as_posix(),
clobber=True)
if gen_catalogs:
for n in np.sort(np.atleast_1d(n_contam)):
if n == 0:
mock = _load_deltas(mock, i,
delta_output_dir / "n_contaminate_0",
sorted_zbins.size)
mock = _load_ngals(mock, i,
ngals_output_dir / "n_contaminate_0")
else:
try:
assert not force
mock = _load_deltas(
mock, i, delta_output_dir / f"n_contaminate_{n}",
sorted_zbins.size)
except (AssertionError, IOError):
mock = _load_deltas(
mock, i, delta_output_dir / "n_contaminate_0",
sorted_zbins.size)
mock = contaminate_mock(mock, n, sorted_zbins,
nside_fit, coeff_dir, sys_dir,
chain_version)
for z, deltai in enumerate(mock.delta):
deltai.save(delta_output_dir.joinpath(
f"n_contaminate_{n}",
f"mock_{i}_bin{z}.fits").as_posix(),
clobber=True)
try:
assert not force
mock = _load_ngals(
mock, i, ngals_output_dir / f"n_contaminate_{n}")
except (AssertionError, IOError):
mock.gen_ngal(ngal_mean, ngal_fill)
for z in range(len(mock.ngal)):
mock.ngal[z].data[~mask.
mask] *= mask.fracdet[~mask.mask]
mock.save(ngals_output_dir.joinpath(
f"n_contaminate_{n}", f"mock_{i}.fits").as_posix(),
clobber=True)
try:
assert not force
mock = _load_ngals(
mock, i, noisy_ngals_output_dir / f"n_contaminate_{n}")
except (AssertionError, IOError):
mock.poisson_sample()
mock.save(noisy_ngals_output_dir.joinpath(
f"n_contaminate_{n}", f"mock_{i}.fits").as_posix(),
clobber=True)
ngal_tot = np.array([
ng.data[~mask.mask].astype(int).sum() for ng in mock.ngal
])
gen_mock_catalogs(mock, cats_output_dir / f"n_contaminate_{n}",
i, zedges, force)
if sys.argv[1] == 'quick':
print('quick mode')
num_threads = 10
bin_slop = 0.05
np.random.seed(666)
rand.thin(0.3)
binedges = [0.15, 0.35, 0.50, 0.65, 0.80, 0.90]
nthetabins = 20
thetamax = 250. / 60.
thetamin = 2.5 / 60.
for imethod, label in enumerate(labels):
cat_file = cat_file_temp.format(label)
cat = lsssys.Redmagic(cat_file)
w_dict, corr_dict = lsssys.random2ptcalc(cat,
rand,
binedges,
nthetabins,
thetamax,
thetamin,
autoonly=True,
bin_slop=bin_slop,
bin_type='Log',
num_threads=num_threads,
use_weights=True,
comm=None,
saveinfo=True,
returncorr=True,