def test_sim_psf_type(psf_type):
seed = 431
rng = np.random.RandomState(seed)
dither = True
rotate = True
coadd_dim = 101
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=5,
layout="grid",
)
if psf_type == "ps":
se_dim = get_se_dim(coadd_dim=coadd_dim, dither=dither, rotate=rotate)
psf = make_ps_psf(rng=rng, dim=se_dim)
else:
psf = make_fixed_psf(psf_type=psf_type)
_ = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
dither=dither,
rotate=rotate,
)
def test_sim_se_dim():
"""
test sim can run
"""
seed = 74321
rng = np.random.RandomState(seed)
coadd_dim = 351
se_dim = 351
psf_dim = 51
bands = ["i"]
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=30,
layout="grid",
)
psf = make_fixed_psf(psf_type="gauss")
data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
se_dim=se_dim,
psf_dim=psf_dim,
bands=bands,
g1=0.02,
g2=0.00,
psf=psf,
)
dims = (se_dim, ) * 2
assert data['band_data']['i'][0].image.array.shape == dims
def _make_one_star_sim(
rng,
psf_type='gauss',
epochs_per_band=3,
dither=True,
rotate=True,
bands=['i'],
coadd_dim=51,
psf_dim=51,
):
cat = OneStarCatalog(coadd_dim)
psf = make_fixed_psf(psf_type=psf_type)
data = make_sim(
rng=rng,
galaxy_catalog=cat,
coadd_dim=coadd_dim,
psf_dim=psf_dim,
epochs_per_band=epochs_per_band,
g1=0.00,
g2=0.00,
bands=bands,
psf=psf,
dither=dither,
rotate=rotate,
)
return data, cat
def test_sim_epochs(epochs_per_band):
seed = 7421
bands = ["r", "i", "z"]
coadd_dim = 301
psf_dim = 47
rng = np.random.RandomState(seed)
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=10,
layout="grid",
)
psf = make_fixed_psf(psf_type="gauss")
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
psf_dim=psf_dim,
g1=0.02,
g2=0.00,
psf=psf,
bands=bands,
epochs_per_band=epochs_per_band,
)
band_data = sim_data['band_data']
for band in bands:
assert band in band_data
assert len(band_data[band]) == epochs_per_band
def test_sim_star_bleeds():
seed = 7421
coadd_dim = 201
buff = 30
rng = np.random.RandomState(seed)
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="wldeblend",
coadd_dim=coadd_dim,
buff=buff,
)
star_catalog = StarCatalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
density=100,
)
psf = make_fixed_psf(psf_type="moffat")
# tests that we actually get saturation are in test_star_masks_and_bleeds
_ = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
star_catalog=star_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
star_bleeds=True,
)
def test_sim_draw_method_smoke(draw_method):
seed = 881
coadd_dim = 201
rng = np.random.RandomState(seed)
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=30,
layout='grid',
)
kw = {}
if draw_method is not None:
kw['draw_method'] = draw_method
psf = make_fixed_psf(psf_type="gauss")
_ = make_sim(rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
**kw)
def test_galaxies_smoke(layout, gal_type, morph):
"""
test sim can run and is repeatable. This is relevant as we now support
varying galaxies
"""
seed = 74321
for trial in (1, 2):
rng = np.random.RandomState(seed)
sep = 4.0 # arcseconds
coadd_dim = 100
buff = 10
bands = ['i']
gal_config = {
'hlr': 1.0,
'mag': 22,
'morph': morph,
}
galaxy_catalog = make_galaxy_catalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
gal_type=gal_type,
gal_config=gal_config,
layout=layout,
sep=sep,
)
if layout == 'pair':
if gal_type == 'fixed':
assert isinstance(galaxy_catalog, FixedPairGalaxyCatalog)
elif gal_type == 'varying':
assert isinstance(galaxy_catalog, PairGalaxyCatalog)
else:
if gal_type == 'fixed':
assert isinstance(galaxy_catalog, FixedGalaxyCatalog)
elif gal_type == 'varying':
assert isinstance(galaxy_catalog, GalaxyCatalog)
psf = make_fixed_psf(psf_type='gauss')
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
bands=bands,
g1=0.02,
g2=0.00,
psf=psf,
)
if trial == 1:
image = sim_data['band_data']['i'][0].image.array
else:
new_image = sim_data['band_data']['i'][0].image.array
assert np.all(image == new_image)
def _make_sim(
rng,
psf_type,
epochs_per_band=3,
stars=False,
dither=True,
rotate=True,
bands=['i', 'z'],
coadd_dim=101,
se_dim=None,
psf_dim=51,
bad_columns=False,
):
buff = 5
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="exp",
coadd_dim=coadd_dim,
buff=buff,
layout="grid",
gal_config={'mag': 22},
)
if se_dim is None:
se_dim = get_se_dim(coadd_dim=coadd_dim, dither=dither, rotate=rotate)
if psf_type == "ps":
psf = make_ps_psf(rng=rng, dim=se_dim)
else:
psf = make_fixed_psf(psf_type=psf_type)
if stars:
star_catalog = StarCatalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
density=100,
)
else:
star_catalog = None
return make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
star_catalog=star_catalog,
coadd_dim=coadd_dim,
se_dim=se_dim,
psf_dim=psf_dim,
epochs_per_band=epochs_per_band,
g1=0.02,
g2=0.00,
bands=bands,
psf=psf,
dither=dither,
rotate=rotate,
bad_columns=bad_columns,
)
def test_star_mask_in_sim_repeatable():
"""
test star masking using the keyword to the sim
"""
seed = 234
bands = ['r']
coadd_dim = 100
buff = 0
star_density = 100
psf = make_fixed_psf(psf_type='moffat')
for step in (0, 1):
rng = np.random.RandomState(seed)
for i in range(1000):
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type='fixed',
coadd_dim=coadd_dim,
buff=buff,
layout='random',
)
star_catalog = StarCatalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
density=star_density,
)
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
star_catalog=star_catalog,
coadd_dim=coadd_dim,
bands=bands,
psf=psf,
g1=0, g2=0,
star_bleeds=True,
)
exp = sim_data['band_data'][bands[0]][0]
mask = exp.mask.array
wsat = np.where((mask & get_flagval('SAT')) != 0)
if wsat[0].size > 0:
if step == 0:
nmarked = wsat[0].size
break
else:
assert wsat[0].size == nmarked
break
def test_cosmics():
rng = np.random.RandomState(32)
coadd_dim = 101
buff = 5
psf_dim = 51
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="exp",
coadd_dim=coadd_dim,
buff=buff,
layout="grid",
)
psf = make_fixed_psf(psf_type='gauss')
data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
psf_dim=psf_dim,
g1=0.02,
g2=0.00,
psf=psf,
epochs_per_band=10, # so we get a CR
cosmic_rays=True,
)
coadd, exp_info = make_coadd_obs(
exps=data['band_data']['i'],
coadd_wcs=data['coadd_wcs'],
coadd_bbox=data['coadd_bbox'],
psf_dims=data['psf_dims'],
rng=rng,
remove_poisson=False, # no object poisson noise in sims
)
assert any(exp_info['maskfrac'] > 0)
cosmic_flag = coadd.coadd_exp.mask.getPlaneBitMask('CR')
bmask = coadd.coadd_exp.mask.array
noise_bmask = coadd.coadd_noise_exp.mask.array
w = np.where((bmask & cosmic_flag) != 0)
assert w[0].size != 0
w = np.where((noise_bmask & cosmic_flag) != 0)
assert w[0].size != 0
def test_sim_smoke(dither, rotate):
"""
test sim can run
"""
seed = 74321
rng = np.random.RandomState(seed)
coadd_dim = 351
psf_dim = 51
bands = ["i"]
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=30,
layout="grid",
)
psf = make_fixed_psf(psf_type="gauss")
data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
psf_dim=psf_dim,
bands=bands,
g1=0.02,
g2=0.00,
psf=psf,
dither=dither,
rotate=rotate,
)
for key in ['band_data', 'coadd_wcs', 'psf_dims', 'coadd_bbox']:
assert key in data
assert isinstance(data['coadd_wcs'], afw_geom.SkyWcs)
assert data['psf_dims'] == (psf_dim, ) * 2
extent = data['coadd_bbox'].getDimensions()
edims = (extent.getX(), extent.getY())
assert edims == (coadd_dim, ) * 2
for band in bands:
assert band in data['band_data']
for band, bdata in data['band_data'].items():
assert len(bdata) == 1
assert isinstance(bdata[0], afw_image.ExposureF)
def test_sim_wldeblend():
seed = 7421
coadd_dim = 201
rng = np.random.RandomState(seed)
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="wldeblend",
coadd_dim=coadd_dim,
buff=30,
)
psf = make_fixed_psf(psf_type="moffat")
_ = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
)
def test_sim_defects(cosmic_rays, bad_columns):
ntrial = 10
seed = 7421
rng = np.random.RandomState(seed)
coadd_dim = 201
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
layout="grid",
buff=30,
)
psf = make_fixed_psf(psf_type="gauss")
for itrial in range(ntrial):
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
cosmic_rays=cosmic_rays,
bad_columns=bad_columns,
)
for band, band_exps in sim_data['band_data'].items():
for exp in band_exps:
image = exp.image.array
mask = exp.mask.array
flags = exp.mask.getPlaneBitMask(('CR', 'BAD'))
if bad_columns or cosmic_rays:
wnan = np.where(np.isnan(image))
wflagged = np.where((mask & flags) != 0)
assert wnan[0].size == wflagged[0].size
def test_sim_layout(layout):
seed = 7421
coadd_dim = 201
rng = np.random.RandomState(seed)
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=30,
layout=layout,
)
psf = make_fixed_psf(psf_type="gauss")
_ = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
)
def go(
*,
seed,
gal_imag,
gal_type,
gal_hlr,
ngmix_model,
ntrial,
output,
show=False,
layout='grid',
loglevel='info',
):
"""
seed: int
Seed for a random number generator
ntrial: int
Number of trials to run, paired by simulation plus and minus shear
output: string
Output file path. If output is None, this is a dry
run and no output is written.
show: bool
If True, show some images. Default False
loglevel: string
Log level, default 'info'
"""
rng = np.random.RandomState(seed)
g1, g2 = 0.0, 0.0
# only over ride bands
sim_config = {
'bands': ['r', 'i'],
'psf_type': 'moffat',
'layout': layout,
}
sim_config = get_sim_config(config=sim_config)
logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger('mdet_lsst_sim')
logger.setLevel(getattr(logging, loglevel.upper()))
logger.info(str(sim_config))
dlist = []
for trial in range(ntrial):
logger.info('-' * 70)
logger.info('trial: %d/%d' % (trial + 1, ntrial))
galaxy_catalog = ColorGalaxyCatalog(
rng=rng,
coadd_dim=sim_config['coadd_dim'],
buff=sim_config['buff'],
layout=sim_config['layout'],
imag=gal_imag,
hlr=gal_hlr,
gal_type=gal_type,
)
if sim_config['psf_type'] == 'ps':
se_dim = get_se_dim(coadd_dim=sim_config['coadd_dim'])
psf = make_ps_psf(rng=rng, dim=se_dim)
else:
psf = make_psf(psf_type=sim_config["psf_type"])
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=sim_config['coadd_dim'],
g1=g1,
g2=g2,
psf=psf,
psf_dim=sim_config['psf_dim'],
dither=sim_config['dither'],
rotate=sim_config['rotate'],
bands=sim_config['bands'],
epochs_per_band=sim_config['epochs_per_band'],
noise_factor=sim_config['noise_factor'],
cosmic_rays=sim_config['cosmic_rays'],
bad_columns=sim_config['bad_columns'],
star_bleeds=sim_config['star_bleeds'],
)
if show:
show_sim(sim_data['band_data'])
mbobs = make_mbobs(band_data=sim_data['band_data'], rng=rng)
toutput = run_ngmix(
mbobs=mbobs,
rng=rng,
model=ngmix_model,
)
dlist.append(toutput)
truth = make_truth(cat=galaxy_catalog, bands=sim_config['bands'])
data = eu.numpy_util.combine_arrlist(dlist)
logger.info('writing: %s' % output)
with fitsio.FITS(output, 'rw', clobber=True) as fits:
fits.write(data, extname='model_fits')
fits.write(truth, extname='truth')
def test_sim_stars(density, min_density, max_density):
seed = 7421
coadd_dim = 201
buff = 30
config = {
'density': density,
'min_density': min_density,
'max_density': max_density,
}
for use_maker in (False, True):
rng = np.random.RandomState(seed)
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="wldeblend",
coadd_dim=coadd_dim,
buff=buff,
)
assert len(galaxy_catalog) == galaxy_catalog.shifts_array.size
if use_maker:
star_catalog = make_star_catalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
star_config=config,
)
else:
star_catalog = StarCatalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
density=config['density'],
min_density=config['min_density'],
max_density=config['max_density'],
)
assert len(star_catalog) == star_catalog.shifts_array.size
psf = make_fixed_psf(psf_type="moffat")
# tests that we actually get bright objects set are in
# test_star_masks_and_bleeds
data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
star_catalog=star_catalog,
coadd_dim=coadd_dim,
g1=0.02,
g2=0.00,
psf=psf,
)
if not use_maker:
data_nomaker = data
else:
assert np.all(data['band_data']['i'][0].image.array ==
data_nomaker['band_data']['i'][0].image.array)
def go():
if "CATSIM_DIR" not in os.environ:
# this contains the galaxy and star catalogs for generatig
# WeakLensingDeblending galaxies and stars
print('you need CATSIM_DIR defined to run this example')
seed = 761
rng = np.random.RandomState(seed)
dither = True
rotate = True
coadd_dim = 351
psf_dim = 51
bands = ['r', 'i']
# this makes WeakLensingDeblending galaxies
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type='wldeblend',
coadd_dim=coadd_dim,
buff=30,
)
# power spectrum psf
se_dim = get_se_dim(coadd_dim=coadd_dim, rotate=rotate, dither=dither)
psf = make_ps_psf(rng=rng, dim=se_dim)
# generate simulated data, see below for whats in this dict
data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
psf_dim=psf_dim,
bands=bands,
g1=0.02,
g2=0.00,
psf=psf,
dither=dither,
rotate=rotate,
)
# data is a dict with the following keys.
# band_data: a dict, keyed by band name, with values that are a list of
# exps
# coadd_wcs: is a DM wcs for use in coadding
# psf_dims: is the psf dim we sent in (psf_dim, psf_dim)
# coadd_bbox: is an lsst Box2I, for use in coadding
# bright_info: is a structured array with position and mask info for bright
# objects
for key in [
'band_data', 'coadd_wcs', 'psf_dims', 'coadd_bbox', 'bright_info'
]:
assert key in data
for band in bands:
assert band in data['band_data']
assert isinstance(data['band_data'][band][0], afw_image.ExposureF)
assert isinstance(data['coadd_wcs'], afw_geom.SkyWcs)
assert data['psf_dims'] == (psf_dim, ) * 2
extent = data['coadd_bbox'].getDimensions()
edims = (extent.getX(), extent.getY())
assert edims == (coadd_dim, ) * 2
# we should have no bright objects
assert data['bright_info'].size == 0
def go():
seed = 74321
rng = np.random.RandomState(seed)
coadd_dim = 351
psf_dim = 51
bands = ['r', 'i']
# this makes a grid of fixed exponential galaxies
# with default properties. One exposure per band
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=30,
layout="grid",
)
# gaussian psf
psf = make_fixed_psf(psf_type="gauss")
# generate simulated data, see below for whats in this dict
data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
psf_dim=psf_dim,
bands=bands,
g1=0.02,
g2=0.00,
psf=psf,
dither=True,
rotate=True,
)
# data is a dict with the following keys.
# band_data: a dict, keyed by band name, with values that are a list of
# exps
# coadd_wcs: is a DM wcs for use in coadding
# psf_dims: is the psf dim we sent in (psf_dim, psf_dim)
# coadd_bbox: is an lsst Box2I, for use in coadding
# bright_info: is a structured array with position and mask info for bright
# objects
for key in [
'band_data', 'coadd_wcs', 'psf_dims', 'coadd_bbox', 'bright_info'
]:
assert key in data
for band in bands:
assert band in data['band_data']
assert isinstance(data['band_data'][band][0], afw_image.ExposureF)
assert isinstance(data['coadd_wcs'], afw_geom.SkyWcs)
assert data['psf_dims'] == (psf_dim, ) * 2
extent = data['coadd_bbox'].getDimensions()
edims = (extent.getX(), extent.getY())
assert edims == (coadd_dim, ) * 2
# we should have no bright objects
assert data['bright_info'].size == 0
def test_star_mask_in_sim(draw_stars):
"""
test star masking using the keyword to the sim
"""
rng = np.random.RandomState(234)
bands = ['r']
coadd_dim = 100
buff = 0
star_density = 100
psf = make_fixed_psf(psf_type='moffat')
some_were_bright = False
some_were_saturated = False
for i in range(1000):
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type='fixed',
coadd_dim=coadd_dim,
buff=buff,
layout='random',
)
star_catalog = StarCatalog(
rng=rng,
coadd_dim=coadd_dim,
buff=buff,
density=star_density,
)
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
star_catalog=star_catalog,
draw_stars=draw_stars,
coadd_dim=coadd_dim,
bands=bands,
psf=psf,
g1=0, g2=0,
star_bleeds=True,
)
nbright = sim_data['bright_info'].size
if nbright > 0:
some_were_bright = True
for bi in sim_data['bright_info']:
assert 'ra' in bi.dtype.names
assert 'dec' in bi.dtype.names
assert 'radius_pixels' in bi.dtype.names
assert bi['radius_pixels'] >= 0
assert 'has_bleed' in bi.dtype.names
exp = sim_data['band_data'][bands[0]][0]
mask = exp.mask.array
image = exp.image.array
wsat = np.where((mask & get_flagval('SAT')) != 0)
if (
wsat[0].size > 0 and
np.all(image[wsat] == BAND_SAT_VALS['r'])
):
some_were_saturated = True
break
assert some_were_bright and some_were_saturated
def test_sim_exp_mag(rotate, show=False):
"""
test we get the right mag. Also test we get small flux when we rotate and
there is nothing at the sub image location we choose
This requires getting lucky with the rotation, so try a few
"""
ntrial = 10
bands = ["i"]
seed = 55
coadd_dim = 301
rng = np.random.RandomState(seed)
# use fixed single epoch dim so we can look in the same spot for the object
se_dim = get_se_dim(coadd_dim=coadd_dim, dither=False, rotate=True)
ok = False
for i in range(ntrial):
galaxy_catalog = make_galaxy_catalog(
rng=rng,
gal_type="fixed",
coadd_dim=coadd_dim,
buff=30,
layout="grid",
)
psf = make_fixed_psf(psf_type="gauss")
sim_data = make_sim(
rng=rng,
galaxy_catalog=galaxy_catalog,
coadd_dim=coadd_dim,
se_dim=se_dim,
g1=0.02,
g2=0.00,
psf=psf,
bands=bands,
rotate=rotate,
)
image = sim_data["band_data"]["i"][0].image.array
sub_image = image[93:93 + 25, 88:88 + 25]
subim_sum = sub_image.sum()
if show:
import matplotlib.pyplot as mplt
fig, ax = mplt.subplots(nrows=1, ncols=2)
ax[0].imshow(image)
ax[1].imshow(sub_image)
mplt.show()
if rotate:
# we expect nothing there
if abs(subim_sum) < 30:
ok = True
break
else:
# we expect something there with about the right magnitude
mag = ZERO_POINT - 2.5 * np.log10(subim_sum)
assert abs(mag - DEFAULT_FIXED_GAL_CONFIG['mag']) < 0.005
break
if rotate:
assert ok, 'expected at least one to be empty upon rotation'
