def runTest(self):
"""
Run tests of redmapper.VolumeLimitMask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
vlim = VolumeLimitMask(config, config.vlim_lstar)
# And test the values...
ras = np.array([140.0, 141.0, 150.0])
decs = np.array([65.25, 65.6, 30.0])
zmax = vlim.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax, [0.338, 0.341, 0.0])
# And compute a geometry mask
vlim_geom = VolumeLimitMask(config, config.vlim_lstar + 1.0, use_geometry=True)
zmax_geom = vlim_geom.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax_geom, [config.zrange[1], config.zrange[1], 0.0])
def runTest(self):
"""
Run tests of redmapper.VolumeLimitMask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
vlim = VolumeLimitMask(config, config.vlim_lstar)
# And test the values...
ras = np.array([140.0, 141.0, 150.0])
decs = np.array([65.25, 65.6, 30.0])
zmax = vlim.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax, [0.338, 0.341, 0.0])
# And compute a geometry mask
vlim_geom = VolumeLimitMask(config,
config.vlim_lstar + 1.0,
use_geometry=True)
zmax_geom = vlim_geom.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax_geom,
[config.zrange[1], config.zrange[1], 0.0])
def test_generatebkg(self):
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = test_dir
config.bkgfile = os.path.join(config.outpath, '%s_testbkg.fit' % (config.outbase))
config.zrange = [0.1, 0.2]
gen = BackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
bkg = fitsio.read(config.bkgfile, ext='CHISQBKG')
# Some spot-testing...
testing.assert_equal(bkg[0]['sigma_g'].shape, (48, 40, 5))
testing.assert_equal(bkg[0]['sigma_lng'].shape, (48, 40, 5))
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 20, 2], 2.8444533)
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 10, 3], 7.4324579)
testing.assert_almost_equal(bkg[0]['sigma_lng'][30, 10, 3], 3.7618985)
testing.assert_almost_equal(bkg[0]['sigma_lng'][45, 10, 3], 0.0)
if os.path.exists(test_dir):
shutil.rmtree(test_dir, True)
def test_generatebkg(self):
"""
Test generation of a background file.
"""
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
config.calib_nproc = 1
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.bkgfile = os.path.join(config.outpath, '%s_testbkg.fit' % (config.d.outbase))
config.zrange = [0.1, 0.2]
gen = BackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
bkg = fitsio.read(config.bkgfile, ext='CHISQBKG')
# Some spot-testing...
testing.assert_equal(bkg[0]['sigma_g'].shape, (48, 40, 5))
testing.assert_equal(bkg[0]['sigma_lng'].shape, (48, 40, 5))
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 20, 2], 2.8444533)
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 10, 3], 7.4324579)
testing.assert_almost_equal(bkg[0]['sigma_lng'][30, 10, 3], 3.7618985)
testing.assert_almost_equal(bkg[0]['sigma_lng'][45, 10, 3], 0.0)
def runTest(self):
"""
Run the redmapper.RunCatalog tests.
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
conffile = 'testconfig.yaml'
catfile = 'test_cluster_pos.fit'
config = Configuration(file_path + '/' + conffile)
config.catfile = file_path + '/' + catfile
runcat = RunCatalog(config)
runcat.run(do_percolation_masking=False)
testing.assert_equal(runcat.cat.mem_match_id, [1, 2, 3])
testing.assert_almost_equal(runcat.cat.Lambda, [24.16809273, 26.85296822, 13.36757088])
testing.assert_almost_equal(runcat.cat.lambda_e, [2.50003219, 4.83695221, 2.4651196])
testing.assert_almost_equal(runcat.cat.z_lambda, [0.22785459, 0.32256541, 0.2176394])
testing.assert_almost_equal(runcat.cat.z_lambda_e, [0.00631017, 0.01353213, 0.00984608])
runcat.run(do_percolation_masking=True)
testing.assert_equal(runcat.cat.mem_match_id, [1, 2, 3])
testing.assert_almost_equal(runcat.cat.Lambda, [24.22911263, 26.85296822, -1.])
testing.assert_almost_equal(runcat.cat.lambda_e, [2.50442076, 4.83695221, -1.])
testing.assert_almost_equal(runcat.cat.z_lambda, [0.22785437, 0.32256407, -1.])
testing.assert_almost_equal(runcat.cat.z_lambda_e, [0.00630675, 0.01353031, -1.])
def test_maskgals(self):
"""
Test generation of maskgals file.
"""
# Note that due to historical reasons, this is testing the
# new generation of maskgals with some spot checks. Independently,
# it has been checked that the distributions are the same as for
# the old IDL code which was used to generate the reference used
# in the cluster tests.
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(os.path.join(file_path, conf_filename))
# For testing, and backwards compatibility, only make one
config.maskgal_nsamples = 1
config.mask_mode = 0
mask = get_mask(config)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
maskgalfile = os.path.join(self.test_dir, 'testmaskgal.fit')
random.seed(seed=12345)
# This will generate the file if it isn't there
mask.read_maskgals(maskgalfile)
maskgals, hdr = fitsio.read(maskgalfile, ext=1, header=True)
self.assertEqual(maskgals.size, config.maskgal_ngals * config.maskgal_nsamples)
self.assertEqual(hdr['VERSION'], 6)
self.assertEqual(hdr['R0'], config.percolation_r0)
self.assertEqual(hdr['BETA'], config.percolation_beta)
self.assertEqual(hdr['STEPSIZE'], config.maskgal_rad_stepsize)
self.assertEqual(hdr['NMAG'], config.nmag)
self.assertEqual(hdr['NGALS'], config.maskgal_ngals)
self.assertEqual(hdr['CHISQMAX'], config.chisq_max)
self.assertEqual(hdr['LVALREF'], config.lval_reference)
self.assertEqual(hdr['EXTRA'], config.maskgal_dmag_extra)
self.assertEqual(hdr['ALPHA'], config.calib_lumfunc_alpha)
self.assertEqual(hdr['RSIG'], config.rsig)
self.assertEqual(hdr['ZREDERR'], config.maskgal_zred_err)
testing.assert_almost_equal(maskgals['r'][0: 3], [0.66900003, 0.119, 0.722])
testing.assert_almost_equal(maskgals['phi'][0: 3], [1.73098969, 2.53610063, 4.2362957])
testing.assert_almost_equal(maskgals['x'][0: 3], [-0.1067116, -0.09784444, -0.33090013])
testing.assert_almost_equal(maskgals['m'][0: 3], [0.46200001, 1.778, -1.43599999])
testing.assert_almost_equal(maskgals['chisq'][0: 3], [8.63599968, 2.28399992, 1.55799997])
testing.assert_almost_equal(maskgals['cwt'][0: 3], [0.02877194, 0.1822518, 0.17872778])
testing.assert_almost_equal(maskgals['nfw'][0: 3], [0.15366785, 0.32543495, 0.1454625])
testing.assert_almost_equal(maskgals['dzred'][0: 3], [-0.03090504, 0.00847131, -0.01800639])
testing.assert_almost_equal(maskgals['zwt'][0: 3], [6.0447073, 18.23568916, 13.30043507])
testing.assert_almost_equal(maskgals['lumwt'][0: 3], [0.39371657, 0.62304342, 0.01774093])
testing.assert_almost_equal(maskgals['theta_r'][0: 3, 3], [0.73237121, 1., 0.3299689])
testing.assert_almost_equal(maskgals['radbins'][0, 0: 3], [0.40000001, 0.5, 0.60000002])
testing.assert_almost_equal(maskgals['nin_orig'][0, 0: 3], [2213., 2663., 3066.])
testing.assert_almost_equal(maskgals['nin'][0, 0: 3], [2203.3347168, 2651.54467773, 3062.44628906])
def test_zred_runpixels(self):
"""
Test redmapper.ZredRunPixels, computing zreds for all the galaxies
in a pixelized galaxy catalog.
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
config.d.hpix = 2163
config.d.nside = 64
config.border = 0.0
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.zredfile = os.path.join(self.test_dir, 'zreds', 'testing_zreds_master_table.fit')
# FIXME: try an illegal one...
zredRunpix = ZredRunPixels(config)
zredRunpix.run()
# Check that the zred file has been built...
self.assertTrue(os.path.isfile(config.zredfile))
# Read in just the galaxies...
gals0 = GalaxyCatalog.from_galfile(config.galfile, nside=config.d.nside, hpix=config.d.hpix, border=config.border)
# And with the zreds...
gals = GalaxyCatalog.from_galfile(config.galfile, zredfile=config.zredfile, nside=config.d.nside, hpix=config.d.hpix, border=config.border)
# Confirm they're the same galaxies...
testing.assert_array_almost_equal(gals0.ra, gals.ra)
# Confirm the zreds are okay
self.assertGreater(np.min(gals.zred), 0.0)
self.assertGreater(np.min(gals.chisq), 0.0)
self.assertLess(np.max(gals.lkhd), 0.0)
zredfile = os.path.join(file_path, 'zreds_test', 'dr8_test_zreds_master_table.fit')
gals_compare = GalaxyCatalog.from_galfile(config.galfile, zredfile=zredfile,
nside=config.d.nside, hpix=config.d.hpix, border=config.border)
zredstr = RedSequenceColorPar(config.parfile)
mstar_input = zredstr.mstar(gals_compare.zred_uncorr)
mstar = zredstr.mstar(gals_compare.zred_uncorr)
ok, = np.where((gals_compare.refmag < (mstar_input - 2.5*np.log10(0.15))) |
(gals_compare.refmag < (mstar - 2.5*np.log10(0.15))))
delta_zred_uncorr = gals.zred_uncorr[ok] - gals_compare.zred_uncorr[ok]
use, = np.where(np.abs(delta_zred_uncorr) < 1e-3)
testing.assert_array_less(0.98, float(use.size) / float(ok.size))
def test_selectspecseeds(self):
"""
Run tests on redmapper.calibration.SelectSpecSeeds
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
def test_selectspecseeds(self):
"""
Run tests on redmapper.calibration.SelectSpecSeeds
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
def runTest(self):
"""
Run redmapper.DepthMap tests.
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
# Check the regular depth
depthstr = DepthMap(config)
RAs = np.array([140.00434405, 142.04090, 142.09242, 142.11448, 50.0])
Decs = np.array([63.47175301, 65.133844, 65.084844, 65.109541, 50.0])
comp_limmag = np.array([20.810108, 20.59153, 20.59663, 20.59663, -1.63750e+30], dtype='f4')
comp_exptime = np.array([78.849754, 63.56208, 63.56208, 63.56209, -1.63750e+30], dtype='f4')
comp_m50 = np.array([20.967576, 20.85170, 20.85677, 20.85677, -1.63750e+30], dtype='f4')
limmag, exptime, m50 = depthstr.get_depth_values(RAs, Decs)
testing.assert_almost_equal(limmag, comp_limmag, 4)
testing.assert_almost_equal(exptime, comp_exptime, 4)
testing.assert_almost_equal(m50, comp_m50, 4)
# And check the areas...
mags = np.array([20.0, 20.2, 20.4, 20.6, 20.8, 21.0])
areas_idl = np.array([3.29709, 3.29709, 3.29709, 3.29709, 2.86089, 0.0603447])
areas = depthstr.calc_areas(mags)
testing.assert_almost_equal(areas, areas_idl, 4)
config2 = Configuration(file_path + "/" + conf_filename)
config2.d.hpix = 582972
config2.d.nside = 1024
config2.border = 0.02
depthstr2 = DepthMap(config2)
limmag2, exptime2, m502 = depthstr2.get_depth_values(RAs, Decs)
comp_limmag[0] = hp.UNSEEN
comp_exptime[0] = hp.UNSEEN
comp_m50[0] = hp.UNSEEN
testing.assert_almost_equal(limmag2, comp_limmag, 4)
testing.assert_almost_equal(exptime2, comp_exptime, 4)
testing.assert_almost_equal(m502, comp_m50, 4)
config3 = Configuration(file_path + "/" + conf_filename)
config3.d.hpix = 8421
config3.d.nside = 128
config3.border = 0.0
depthstr3 = DepthMap(config3)
areas3 = depthstr3.calc_areas(np.array([20.0, 20.5, 21.0]))
testing.assert_almost_equal(areas3[0], 0.20457271, 6)
def test_readmask(self):
"""
Test reading and using of mask files in redmapper.HPMask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
# First, test a bare mask
config.mask_mode = 0
mask = get_mask(config)
maskgal_index = mask.select_maskgals_sample()
testing.assert_equal(hasattr(mask, 'maskgals'), True)
testing.assert_equal(isinstance(mask, Mask), True)
testing.assert_equal(isinstance(mask, HPMask), False)
# And the healpix mask
config.mask_mode = 3
mask = get_mask(config)
maskgal_index = mask.select_maskgals_sample()
testing.assert_equal(hasattr(mask, 'maskgals'), True)
testing.assert_equal(isinstance(mask, Mask), True)
testing.assert_equal(isinstance(mask, HPMask), True)
# When ready, add in test of gen_maskgals()
# Test the healpix configuration
testing.assert_equal(mask.nside,2048)
# Next test the compute_radmask() function
# Note: RA and DECs are in degrees here
RAs = np.array([140.00434405, 142.04090, 142.09242, 142.11448, 50.0])
Decs = np.array([63.47175301, 65.133844, 65.084844, 65.109541, 50.0])
comp = np.array([True, True, True, True, False])
testing.assert_equal(mask.compute_radmask(RAs, Decs), comp)
# And test that we're getting the right numbers from a sub-mask
config2 = Configuration(file_path + "/" + conf_filename)
config2.d.hpix = 582972
config2.d.nside = 1024
config2.border = 0.02
mask2 = get_mask(config2)
maskgal_index = mask2.select_maskgals_sample()
comp = np.array([False, True, True, True, False])
testing.assert_equal(mask2.compute_radmask(RAs, Decs), comp)
def test_zred_runcat(self):
"""
Test redmapper.ZredRunCatalog, computing zreds for all the galaxies in
a single catalog file.
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
outfile = os.path.join(self.test_dir, 'test_zred_out.fits')
tab = fitsio.read(config.galfile, ext=1, lower=True)
galfile = os.path.join(os.path.dirname(config.galfile),
tab[0]['filenames'][0].decode())
zredRuncat = ZredRunCatalog(config)
zredRuncat.run(galfile, outfile)
# This exercises the reading code
gals = GalaxyCatalog.from_galfile(galfile, zredfile=outfile)
self.assertGreater(np.min(gals.zred), 0.0)
self.assertGreater(np.min(gals.chisq), 0.0)
self.assertLess(np.max(gals.lkhd), 0.0)
# And compare to the "official" run...
config.zredfile = os.path.join(file_path, 'zreds_test',
'dr8_test_zreds_master_table.fit')
ztab = fitsio.read(config.zredfile, ext=1, lower=True)
zredfile = os.path.join(os.path.dirname(config.zredfile),
ztab[0]['filenames'][0].decode())
gals_compare = GalaxyCatalog.from_galfile(galfile, zredfile=zredfile)
zredstr = RedSequenceColorPar(config.parfile)
mstar_input = zredstr.mstar(gals_compare.zred_uncorr)
mstar = zredstr.mstar(gals_compare.zred_uncorr)
ok, = np.where(
(gals_compare.refmag < (mstar_input - 2.5 * np.log10(0.15)))
| (gals_compare.refmag < (mstar - 2.5 * np.log10(0.15))))
delta_zred_uncorr = gals.zred_uncorr[ok] - gals_compare.zred_uncorr[ok]
use, = np.where(np.abs(delta_zred_uncorr) < 1e-3)
testing.assert_array_less(0.98, float(use.size) / float(ok.size))
def _setup_cluster(self):
"""
Set up the cluster to run through the centering code.
"""
file_path = 'data_for_tests'
cluster = Cluster()
cluster.config = Configuration(
os.path.join(file_path, 'testconfig.yaml'))
tempcat = fitsio.read(
os.path.join(file_path, 'test_wcen_zred_data.fit'))
temp_neighbors = np.zeros(tempcat[0]['RAS'].size,
dtype=[('RA', 'f8'), ('DEC', 'f8'),
('DIST', 'f4'), ('R', 'f4'),
('P', 'f4'), ('PFREE', 'f4'),
('PMEM', 'f4'), ('MAG', 'f4', 5),
('MAG_ERR', 'f4', 5),
('REFMAG', 'f4'),
('REFMAG_ERR', 'f4'), ('CHISQ', 'f4'),
('ZRED', 'f4'), ('ZRED_E', 'f4'),
('ZRED_CHISQ', 'f4')])
temp_neighbors['RA'] = tempcat[0]['RAS']
temp_neighbors['DEC'] = tempcat[0]['DECS']
temp_neighbors['R'] = tempcat[0]['R']
temp_neighbors['P'] = tempcat[0]['PVALS']
temp_neighbors['PFREE'] = tempcat[0]['WVALS']
temp_neighbors['PMEM'] = tempcat[0]['WTVALS']
temp_neighbors['REFMAG'] = tempcat[0]['REFMAG_TOTAL']
temp_neighbors['ZRED'] = tempcat[0]['GZREDS']
temp_neighbors['ZRED_E'] = tempcat[0]['GZREDE']
temp_neighbors['ZRED_CHISQ'] = tempcat[0]['GCHISQ']
temp_neighbors['DIST'] = tempcat[0]['R'] / (
np.radians(1.) *
cluster.config.cosmo.Da(0, tempcat[0]['ZCLUSTER']))
neighbors = GalaxyCatalog(temp_neighbors)
cluster.set_neighbors(neighbors)
zred_filename = 'test_dr8_pars.fit'
cluster.zredstr = RedSequenceColorPar(os.path.join(
file_path, 'test_dr8_pars.fit'),
fine=True,
zrange=[0.25, 0.35])
cluster.bkg = Background(os.path.join(file_path, 'test_bkg.fit'))
cluster.zredbkg = ZredBackground(
os.path.join(file_path, 'test_bkg.fit'))
cluster.redshift = tempcat[0]['ZCLUSTER']
cluster.ra = tempcat[0]['RAC']
cluster.dec = tempcat[0]['DECC']
cluster.r_lambda = 1.0 * (tempcat[0]['LAMBDA'] / 100.0)**0.2
cluster.Lambda = tempcat[0]['LAMBDA']
cluster.scaleval = tempcat[0]['SCALEVAL']
return cluster
def test_zred_runcat(self):
"""
Test redmapper.ZredRunCatalog, computing zreds for all the galaxies in
a single catalog file.
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
outfile = os.path.join(self.test_dir, 'test_zred_out.fits')
tab = fitsio.read(config.galfile, ext=1, lower=True)
galfile = os.path.join(os.path.dirname(config.galfile), tab[0]['filenames'][0].decode())
zredRuncat = ZredRunCatalog(config)
zredRuncat.run(galfile, outfile)
# This exercises the reading code
gals = GalaxyCatalog.from_galfile(galfile, zredfile=outfile)
self.assertGreater(np.min(gals.zred), 0.0)
self.assertGreater(np.min(gals.chisq), 0.0)
self.assertLess(np.max(gals.lkhd), 0.0)
# And compare to the "official" run...
config.zredfile = os.path.join(file_path, 'zreds_test', 'dr8_test_zreds_master_table.fit')
ztab = fitsio.read(config.zredfile, ext=1, lower=True)
zredfile = os.path.join(os.path.dirname(config.zredfile), ztab[0]['filenames'][0].decode())
gals_compare = GalaxyCatalog.from_galfile(galfile, zredfile=zredfile)
zredstr = RedSequenceColorPar(config.parfile)
mstar_input = zredstr.mstar(gals_compare.zred_uncorr)
mstar = zredstr.mstar(gals_compare.zred_uncorr)
ok, = np.where((gals_compare.refmag < (mstar_input - 2.5*np.log10(0.15))) |
(gals_compare.refmag < (mstar - 2.5*np.log10(0.15))))
delta_zred_uncorr = gals.zred_uncorr[ok] - gals_compare.zred_uncorr[ok]
use, = np.where(np.abs(delta_zred_uncorr) < 1e-3)
testing.assert_array_less(0.98, float(use.size) / float(ok.size))
def runTest(self):
"""
Run redmapper.DepthMap tests.
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
# Check the regular depth
depthstr = DepthMap(config)
RAs = np.array([140.00434405, 142.04090, 142.09242, 142.11448, 50.0])
Decs = np.array([63.47175301, 65.133844, 65.084844, 65.109541, 50.0])
comp_limmag = np.array(
[20.810108, 20.59153, 20.59663, 20.59663, -1.63750e+30],
dtype='f4')
comp_exptime = np.array(
[78.849754, 63.56208, 63.56208, 63.56209, -1.63750e+30],
dtype='f4')
comp_m50 = np.array(
[20.967576, 20.85170, 20.85677, 20.85677, -1.63750e+30],
dtype='f4')
limmag, exptime, m50 = depthstr.get_depth_values(RAs, Decs)
testing.assert_almost_equal(limmag, comp_limmag, 4)
testing.assert_almost_equal(exptime, comp_exptime, 4)
testing.assert_almost_equal(m50, comp_m50, 4)
# And check the areas...
mags = np.array([20.0, 20.2, 20.4, 20.6, 20.8, 21.0])
areas_idl = np.array(
[3.29709, 3.29709, 3.29709, 3.29709, 2.86089, 0.0603447])
areas = depthstr.calc_areas(mags)
testing.assert_almost_equal(areas, areas_idl, 4)
config2 = Configuration(file_path + "/" + conf_filename)
config2.d.hpix = 582972
config2.d.nside = 1024
config2.border = 0.02
depthstr2 = DepthMap(config2)
limmag2, exptime2, m502 = depthstr2.get_depth_values(RAs, Decs)
comp_limmag[0] = hp.UNSEEN
comp_exptime[0] = hp.UNSEEN
comp_m50[0] = hp.UNSEEN
testing.assert_almost_equal(limmag2, comp_limmag, 4)
testing.assert_almost_equal(exptime2, comp_exptime, 4)
testing.assert_almost_equal(m502, comp_m50, 4)
config3 = Configuration(file_path + "/" + conf_filename)
config3.d.hpix = 8421
config3.d.nside = 128
config3.border = 0.0
depthstr3 = DepthMap(config3)
areas3 = depthstr3.calc_areas(np.array([20.0, 20.5, 21.0]))
testing.assert_almost_equal(areas3[0], 0.20457271, 6)
def test_depthfit(self):
"""
Test depth fitting with redmapper.DepthLim
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
gals = GalaxyCatalog.from_galfile(config.galfile)
np.random.seed(seed=12345)
# First creation of depth object
dlim = DepthLim(gals.refmag, gals.refmag_err)
# This has been inspected that it makes sense.
# Also, one should really be using a depth map
testing.assert_almost_equal(dlim.initpars['EXPTIME'], 104.782066, 0)
testing.assert_almost_equal(dlim.initpars['LIMMAG'], 20.64819717, 0)
# And take a subpixel
gals = GalaxyCatalog.from_galfile(config.galfile, hpix=8421, nside=128)
config.mask_mode = 0
mask = get_mask(config)
maskgal_index = mask.select_maskgals_sample()
mask.read_maskgals(config.maskgalfile)
dlim.calc_maskdepth(mask.maskgals, gals.refmag, gals.refmag_err)
pars, fail = calcErrorModel(gals.refmag, gals.refmag_err, calcErr=False)
testing.assert_almost_equal(pars['EXPTIME'], 63.73879623, 0)
testing.assert_almost_equal(pars['LIMMAG'], 20.68231583, 0)
# And make sure the maskgals are getting the right constant value
testing.assert_almost_equal(pars['EXPTIME'], mask.maskgals.exptime.min())
testing.assert_almost_equal(pars['EXPTIME'], mask.maskgals.exptime.max())
testing.assert_almost_equal(pars['LIMMAG'], mask.maskgals.limmag.min())
testing.assert_almost_equal(pars['LIMMAG'], mask.maskgals.limmag.max())
def runTest(self):
file_path = 'data_for_tests'
conffile = 'testconfig.yaml'
catfile = 'test_cluster_pos.fit'
config = Configuration(file_path + '/' + conffile)
config.catfile = file_path + '/' + catfile
runcat = RunCatalog(config)
runcat.run(do_percolation_masking=False)
testing.assert_equal(runcat.cat.mem_match_id, [1, 2, 3])
testing.assert_almost_equal(runcat.cat.Lambda,
[23.86299324, 17.3491192, 13.36757088])
#testing.assert_almost_equal(runcat.cat.lambda_e, [ 2.47804546, 2.00936174, 2.4651196])
testing.assert_almost_equal(runcat.cat.lambda_e,
[2.47804546, 2.0184479, 2.4651196])
#testing.assert_almost_equal(runcat.cat.z_lambda, [ 0.22786506, 0.32121494, 0.22311865])
testing.assert_almost_equal(runcat.cat.z_lambda,
[0.22786506, 0.3215729, 0.22311865])
#testing.assert_almost_equal(runcat.cat.z_lambda_e, [ 0.00629484, 0.01389629, 0.00969736])
testing.assert_almost_equal(runcat.cat.z_lambda_e,
[0.00629484, 0.0139078, 0.00969736])
runcat.run(do_percolation_masking=True)
testing.assert_equal(runcat.cat.mem_match_id, [1, 2, 3])
#testing.assert_almost_equal(runcat.cat.Lambda, [ 23.86299324, 17.39488411, -1.0])
testing.assert_almost_equal(runcat.cat.Lambda,
[23.86299324, 17.3491192, -1.0])
#testing.assert_almost_equal(runcat.cat.lambda_e, [ 2.47804546, 2.00936174, -1.0])
testing.assert_almost_equal(runcat.cat.lambda_e,
[2.47804546, 2.0184479, -1.0])
#testing.assert_almost_equal(runcat.cat.z_lambda, [ 0.22786506, 0.32121494, -1.0])
testing.assert_almost_equal(runcat.cat.z_lambda,
[0.22786506, 0.3215729, -1.0])
#testing.assert_almost_equal(runcat.cat.z_lambda_e, [ 0.00629484, 0.01389629, -1.0])
testing.assert_almost_equal(runcat.cat.z_lambda_e,
[0.00629484, 0.0139078, -1.0])
def test_run_colormem(self):
"""
Run tests of redmapper.RunColormem
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
# First, we need the red galaxy model
config.specfile_train = os.path.join(file_path, 'test_dr8_spec.fit')
config.zrange = [0.1,0.2]
config.redgalfile = config.redmapper_filename('test_redgals')
config.redgalmodelfile = config.redmapper_filename('test_redgalmodel')
selred = SelectSpecRedGalaxies(config)
selred.run()
# Main test...
config.zmemfile = config.redmapper_filename('test_zmem')
rcm = RunColormem(config)
rcm.run()
rcm.output_training()
# Check that the files are there...
self.assertTrue(os.path.isfile(config.zmemfile))
mem = fitsio.read(config.zmemfile, ext=1)
testing.assert_equal(mem.size, 16)
testing.assert_array_almost_equal(mem['pcol'][0:3], np.array([0.94829756, 0.83803916, 0.88315928]))
testing.assert_array_almost_equal(mem['z'][0:3], np.array([0.191279, 0.188257, 0.186945]))
def runTest(self):
"""
Run the ClusterFit test.
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
conf_filename = 'testconfig.yaml'
config = Configuration(file_path + '/' + conf_filename)
gals = GalaxyCatalog.from_galfile(config.galfile)
# temporary hack...
dist = esutil.coords.sphdist(142.12752, 65.103898, gals.ra, gals.dec)
mpc_scale = np.radians(1.) * config.cosmo.Da(0, 0.227865)
r = np.clip(mpc_scale * dist, 1e-6, None)
use, = np.where(r < 0.75)
st = np.argsort(r[use])
cbkg = ColorBackground(config.bkgfile_color, usehdrarea=True)
zredstr = RedSequenceColorPar(None, config=config)
cluster = Cluster(r0=0.5,
beta=0.0,
config=config,
cbkg=cbkg,
neighbors=gals[use[st]],
zredstr=zredstr)
cluster.ra = 142.12752
cluster.dec = 65.103898
cluster.redshift = 0.227865
cluster.update_neighbors_dist()
mask = HPMask(cluster.config)
maskgal_index = mask.select_maskgals_sample(maskgal_index=0)
depthstr = DepthMap(cluster.config)
mask.set_radmask(cluster)
depthstr.calc_maskdepth(mask.maskgals, cluster.ra, cluster.dec,
cluster.mpc_scale)
lam = cluster.calc_richness_fit(mask,
1,
centcolor_in=1.36503,
calc_err=False)
testing.assert_almost_equal(lam, 16.174486160, decimal=5)
testing.assert_almost_equal(cluster.neighbors.pcol[0:4],
np.array([0.94243, 0., 0.06338, 0.16077]),
5)
lam = cluster.calc_richness_fit(mask, 1, calc_err=False)
testing.assert_almost_equal(lam, 16.2049961, decimal=5)
def runTest(self):
"""
Run the redmapper.RunCatalog tests.
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
conffile = 'testconfig.yaml'
catfile = 'test_cluster_pos.fit'
config = Configuration(file_path + '/' + conffile)
config.catfile = file_path + '/' + catfile
runcat = RunCatalog(config)
runcat.run(do_percolation_masking=False)
testing.assert_equal(runcat.cat.mem_match_id, [1, 2, 3])
testing.assert_almost_equal(runcat.cat.Lambda,
[24.16809273, 26.85296822, 13.36757088])
testing.assert_almost_equal(runcat.cat.lambda_e,
[2.50003219, 4.83695221, 2.4651196])
testing.assert_almost_equal(runcat.cat.z_lambda,
[0.22785459, 0.32256541, 0.2176394])
testing.assert_almost_equal(runcat.cat.z_lambda_e,
[0.00631017, 0.01353213, 0.00984608])
runcat.run(do_percolation_masking=True)
testing.assert_equal(runcat.cat.mem_match_id, [1, 2, 3])
testing.assert_almost_equal(runcat.cat.Lambda,
[24.22911263, 26.85296822, -1.])
testing.assert_almost_equal(runcat.cat.lambda_e,
[2.50442076, 4.83695221, -1.])
testing.assert_almost_equal(runcat.cat.z_lambda,
[0.22785437, 0.32256407, -1.])
testing.assert_almost_equal(runcat.cat.z_lambda_e,
[0.00630675, 0.01353031, -1.])
def test_generatezredbkg(self):
"""
Test generation of a zred background file.
"""
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.bkgfile = os.path.join(config.outpath, '%s_testbkg.fit' % (config.d.outbase))
config.zrange = [0.1, 0.2]
# First test without a zred file ...
gen = ZredBackgroundGenerator(config)
self.assertRaises(RuntimeError, gen.run)
# And now fix it ...
config.zredfile = os.path.join('data_for_tests', 'zreds_test', 'dr8_test_zreds_master_table.fit')
gen = ZredBackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
zbkg = fitsio.read(config.bkgfile, ext='ZREDBKG')
# Some spot-testing...
# (The numbers have been checked to be consistent with the full run tested above
# but can't be directly compared because this is much noisier)
testing.assert_equal(zbkg[0]['sigma_g'].shape, (48, 10))
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 5], 620.0223999, decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][47, 8], 30501.8398438, decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 0], 384.3362732, decimal=5)
def test_readmask(self):
"""
Test reading and using of mask files in redmapper.HPMask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
# First, test a bare mask
config.mask_mode = 0
mask = get_mask(config)
maskgal_index = mask.select_maskgals_sample()
testing.assert_equal(hasattr(mask, 'maskgals'), True)
testing.assert_equal(isinstance(mask, Mask), True)
testing.assert_equal(isinstance(mask, HPMask), False)
# And the healpix mask
config.mask_mode = 3
mask = get_mask(config)
maskgal_index = mask.select_maskgals_sample()
testing.assert_equal(hasattr(mask, 'maskgals'), True)
testing.assert_equal(isinstance(mask, Mask), True)
testing.assert_equal(isinstance(mask, HPMask), True)
# When ready, add in test of gen_maskgals()
# Test the healpix configuration
testing.assert_equal(mask.nside, 2048)
# Next test the compute_radmask() function
# Note: RA and DECs are in degrees here
RAs = np.array([140.00434405, 142.04090, 142.09242, 142.11448, 50.0])
Decs = np.array([63.47175301, 65.133844, 65.084844, 65.109541, 50.0])
comp = np.array([True, True, True, True, False])
testing.assert_equal(mask.compute_radmask(RAs, Decs), comp)
# And test that we're getting the right numbers from a sub-mask
config2 = Configuration(file_path + "/" + conf_filename)
config2.d.hpix = 582972
config2.d.nside = 1024
config2.border = 0.02
mask2 = get_mask(config2)
maskgal_index = mask2.select_maskgals_sample()
comp = np.array([False, True, True, True, False])
testing.assert_equal(mask2.compute_radmask(RAs, Decs), comp)
def runTest(self):
"""
Run the apply_errormodels() test.
"""
file_path = 'data_for_tests'
conf_filename = 'testconfig.yaml'
config = Configuration(file_path + '/' + conf_filename)
mask = HPMask(config) #Create the mask
maskgal_index = mask.select_maskgals_sample()
#set all the necessary inputs from test file
mask.maskgals.exptime = 100.
mask.maskgals.limmag = 20.
mask.maskgals.zp[0] = 22.5
mask.maskgals.nsig[0] = 10.
#necessary as mask.maskgals.exptime has shape (6000,)
mag_in = np.full(6000, 1, dtype=float)
mag_in[:6] = np.array([16., 17., 18., 19., 20., 21.])
#test without noise
mag, mag_err = apply_errormodels(mask.maskgals, mag_in, nonoise=True)
idx = np.array([0, 1, 2, 3, 4, 5])
mag_idl = np.array([16., 17., 18., 19., 20., 21.])
mag_err_idl = np.array(
[0.00602535, 0.0107989, 0.0212915, 0.0463765, 0.108574, 0.264390])
testing.assert_almost_equal(mag[idx], mag_idl)
testing.assert_almost_equal(mag_err[idx], mag_err_idl, decimal=6)
#test with noise and set seed
seed = 0
random.seed(seed=seed)
mag, mag_err = apply_errormodels(mask.maskgals, mag_in)
idx = np.array([0, 1, 2, 3, 4, 5, 1257, 2333, 3876])
mag_test = np.array([
15.98942267, 16.99568733, 17.97935868, 18.90075284, 19.81409659,
21.29508236, 0.99999373, 1.00000663, 1.00000807
])
mag_err_test = np.array([
5.96693051e-03, 1.07560575e-02, 2.08905241e-02, 4.23251692e-02,
9.14877522e-02, 3.46958444e-01, 5.44154045e-06, 5.44160510e-06,
5.44161230e-06
])
testing.assert_almost_equal(mag[idx], mag_test)
testing.assert_almost_equal(mag_err[idx], mag_err_test)
def test_generatezredbkg(self):
"""
Test generation of a zred background file.
"""
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.bkgfile = os.path.join(config.outpath,
'%s_testbkg.fit' % (config.d.outbase))
config.zrange = [0.1, 0.2]
# First test without a zred file ...
gen = ZredBackgroundGenerator(config)
self.assertRaises(RuntimeError, gen.run)
# And now fix it ...
config.zredfile = os.path.join('data_for_tests', 'zreds_test',
'dr8_test_zreds_master_table.fit')
gen = ZredBackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
zbkg = fitsio.read(config.bkgfile, ext='ZREDBKG')
# Some spot-testing...
# (The numbers have been checked to be consistent with the full run tested above
# but can't be directly compared because this is much noisier)
testing.assert_equal(zbkg[0]['sigma_g'].shape, (48, 10))
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 5],
620.0223999,
decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][47, 8],
30501.8398438,
decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 0],
384.3362732,
decimal=5)
def runTest(self):
file_path = 'data_for_tests'
#The configuration
#Used by the mask, background and richness calculation
conf_filename = 'testconfig.yaml'
config = Configuration(file_path + '/' + conf_filename)
filename = 'test_cluster_members.fit'
neighbors = GalaxyCatalog.from_fits_file(file_path + '/' + filename)
zred_filename = 'test_dr8_pars.fit'
zredstr = RedSequenceColorPar(file_path + '/' + zred_filename,
fine=True)
bkg_filename = 'test_bkg.fit'
bkg = Background('%s/%s' % (file_path, bkg_filename))
cluster = Cluster(config=config,
zredstr=zredstr,
bkg=bkg,
neighbors=neighbors)
hdr = fitsio.read_header(file_path + '/' + filename, ext=1)
#cluster.z = hdr['Z']
#cluster.update_z(hdr['Z'])
cluster.redshift = hdr['Z']
richness_compare = hdr['LAMBDA']
richness_compare_err = hdr['LAMBDA_E']
cluster.ra = hdr['RA']
cluster.dec = hdr['DEC']
#Set up the mask
mask = HPMask(cluster.config) #Create the mask
mask.set_radmask(cluster)
#depthstr
depthstr = DepthMap(cluster.config)
depthstr.calc_maskdepth(mask.maskgals, cluster.ra, cluster.dec,
cluster.mpc_scale)
cluster.neighbors.dist = np.degrees(
cluster.neighbors.r / cluster.cosmo.Dl(0, cluster.redshift))
#set seed
seed = 0
random.seed(seed=seed)
# make a zlambda object
zlam = Zlambda(cluster)
z_lambda, z_lambda_e = zlam.calc_zlambda(cluster.redshift,
mask,
calc_err=True,
calcpz=True)
testing.assert_almost_equal(cluster.z_lambda, 0.22700983)
testing.assert_almost_equal(cluster.z_lambda_err, 0.00448909596)
# zlambda_err test
z_lambda_err = zlam._zlambda_calc_gaussian_err(cluster.z_lambda)
testing.assert_almost_equal(z_lambda_err, 0.006303830)
# and test the correction on its own
corr_filename = 'test_dr8_zlambdacorr.fit'
zlambda_corr = ZlambdaCorrectionPar(file_path + '/' + corr_filename,
30.0)
zlam_in = 0.227865
zlam_e_in = 0.00629995
zlam_out = 0.228654
zlam_e_out = 0.00840213
zlam_new, zlam_e_new = zlambda_corr.apply_correction(
24.5, zlam_in, zlam_e_in)
testing.assert_almost_equal(zlam_new, zlam_out, 5)
testing.assert_almost_equal(zlam_e_new, zlam_e_out, 5)
def test_run_colormem(self):
"""
Run tests of redmapper.RunColormem
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
# First, we need the red galaxy model
config.specfile_train = os.path.join(file_path, 'test_dr8_spec.fit')
config.zrange = [0.1, 0.2]
config.redgalfile = config.redmapper_filename('test_redgals')
config.redgalmodelfile = config.redmapper_filename('test_redgalmodel')
selred = SelectSpecRedGalaxies(config)
selred.run()
# Main test...
config.zmemfile = config.redmapper_filename('test_zmem')
rcm = RunColormem(config)
rcm.run()
rcm.output_training()
# Check that the files are there...
self.assertTrue(os.path.isfile(config.zmemfile))
mem = fitsio.read(config.zmemfile, ext=1)
testing.assert_equal(mem.size, 16)
testing.assert_array_almost_equal(
mem['pcol'][0:3], np.array([0.94829756, 0.83803916, 0.88315928]))
testing.assert_array_almost_equal(
mem['z'][0:3], np.array([0.191279, 0.188257, 0.186945]))
def runTest(self):
"""
Run the ColorBackground and ColorBackgroundGenerator tests.
"""
file_name = 'test_dr8_col_bkg.fit'
file_path = 'data_for_tests'
cbkg = ColorBackground('%s/%s' % (file_path, file_name))
col1 = np.array([0.572300, 1.39560])
col2 = np.array([0.7894, 0.9564])
refmags = np.array([17.587, 18.956])
refmagindex = np.array([258, 395])
col1index = np.array([1, 17])
col2index = np.array([15, 19])
idl_bkg1 = np.array([0.148366, 0.165678])
idl_bkg2 = np.array([0.00899471, 0.0201531])
idl_bkg12 = np.array([0.0111827, 0.0719981])
# Test color1
py_outputs = cbkg.lookup_diagonal(1, col1, refmags)
testing.assert_almost_equal(py_outputs, idl_bkg1, decimal=5)
# Test color2
py_outputs = cbkg.lookup_diagonal(2, col2, refmags)
testing.assert_almost_equal(py_outputs, idl_bkg2, decimal=5)
# Test off-diagonal
py_outputs = cbkg.lookup_offdiag(1, 2, col1, col2, refmags)
testing.assert_almost_equal(py_outputs, idl_bkg12, decimal=5)
# And a test sigma_g with the usehdrarea=True
cbkg2 = ColorBackground('%s/%s' % (file_path, file_name), usehdrarea=True)
col1 = np.array([0.572300, 1.39560, 1.0])
col2 = np.array([0.7894, 0.9564, 1.0])
refmags = np.array([17.587, 18.956, 25.0])
idl_sigma_g1 = np.array([123.382, 611.711, np.inf])
idl_sigma_g2 = np.array([8.48481, 82.8938, np.inf])
# Test color1
py_outputs = cbkg2.sigma_g_diagonal(1, col1, refmags)
testing.assert_almost_equal(py_outputs, idl_sigma_g1, decimal=3)
# Test color2
py_outputs = cbkg2.sigma_g_diagonal(2, col2, refmags)
testing.assert_almost_equal(py_outputs, idl_sigma_g2, decimal=3)
#####################################################
# Now a test of the generation of a color background
conf_filename = 'testconfig.yaml'
config = Configuration(file_path + "/" + conf_filename)
tfile = tempfile.mkstemp()
os.close(tfile[0])
config.bkgfile_color = tfile[1]
config.d.nside = 128
config.d.hpix = 8421
config.border = 0.0
cbg = ColorBackgroundGenerator(config, minrangecheck=5)
# Need to set clobber=True because the tempfile was created
cbg.run(clobber=True)
fits = fitsio.FITS(config.bkgfile_color)
# Make sure we have 11 extensions
testing.assert_equal(len(fits), 11)
# Check the 01_01 and 01_02
bkg11 = fits['01_01_REF'].read()
bkg11_compare = fitsio.read(file_path + "/test_dr8_bkg_zredc_sub.fits", ext='01_01_REF')
testing.assert_almost_equal(bkg11['BC'], bkg11_compare['BC'], 3)
testing.assert_almost_equal(bkg11['N'], bkg11_compare['N'], 3)
bkg12 = fits['01_02_REF'].read()
bkg12_compare = fitsio.read(file_path + "/test_dr8_bkg_zredc_sub.fits", ext='01_02_REF')
testing.assert_almost_equal(bkg12['BC'], bkg12_compare['BC'], 2)
testing.assert_almost_equal(bkg12['N'], bkg12_compare['N'], 4)
# And delete the tempfile
os.remove(config.bkgfile_color)
def runTest(self):
"""
Run the ColorBackground and ColorBackgroundGenerator tests.
"""
file_name = 'test_dr8_col_bkg.fit'
file_path = 'data_for_tests'
cbkg = ColorBackground('%s/%s' % (file_path, file_name))
col1 = np.array([0.572300, 1.39560])
col2 = np.array([0.7894, 0.9564])
refmags = np.array([17.587, 18.956])
refmagindex = np.array([258, 395])
col1index = np.array([1, 17])
col2index = np.array([15, 19])
idl_bkg1 = np.array([0.148366, 0.165678])
idl_bkg2 = np.array([0.00899471, 0.0201531])
idl_bkg12 = np.array([0.0111827, 0.0719981])
# Test color1
py_outputs = cbkg.lookup_diagonal(1, col1, refmags)
testing.assert_almost_equal(py_outputs, idl_bkg1, decimal=5)
# Test color2
py_outputs = cbkg.lookup_diagonal(2, col2, refmags)
testing.assert_almost_equal(py_outputs, idl_bkg2, decimal=5)
# Test off-diagonal
py_outputs = cbkg.lookup_offdiag(1, 2, col1, col2, refmags)
testing.assert_almost_equal(py_outputs, idl_bkg12, decimal=5)
# And a test sigma_g with the usehdrarea=True
cbkg2 = ColorBackground('%s/%s' % (file_path, file_name),
usehdrarea=True)
col1 = np.array([0.572300, 1.39560, 1.0])
col2 = np.array([0.7894, 0.9564, 1.0])
refmags = np.array([17.587, 18.956, 25.0])
idl_sigma_g1 = np.array([123.382, 611.711, np.inf])
idl_sigma_g2 = np.array([8.48481, 82.8938, np.inf])
# Test color1
py_outputs = cbkg2.sigma_g_diagonal(1, col1, refmags)
testing.assert_almost_equal(py_outputs, idl_sigma_g1, decimal=3)
# Test color2
py_outputs = cbkg2.sigma_g_diagonal(2, col2, refmags)
testing.assert_almost_equal(py_outputs, idl_sigma_g2, decimal=3)
#####################################################
# Now a test of the generation of a color background
conf_filename = 'testconfig.yaml'
config = Configuration(file_path + "/" + conf_filename)
tfile = tempfile.mkstemp()
os.close(tfile[0])
config.bkgfile_color = tfile[1]
config.d.nside = 128
config.d.hpix = 8421
config.border = 0.0
cbg = ColorBackgroundGenerator(config, minrangecheck=5)
# Need to set clobber=True because the tempfile was created
cbg.run(clobber=True)
fits = fitsio.FITS(config.bkgfile_color)
# Make sure we have 11 extensions
testing.assert_equal(len(fits), 11)
# Check the 01_01 and 01_02
bkg11 = fits['01_01_REF'].read()
bkg11_compare = fitsio.read(file_path + "/test_dr8_bkg_zredc_sub.fits",
ext='01_01_REF')
testing.assert_almost_equal(bkg11['BC'], bkg11_compare['BC'], 3)
testing.assert_almost_equal(bkg11['N'], bkg11_compare['N'], 3)
bkg12 = fits['01_02_REF'].read()
bkg12_compare = fitsio.read(file_path + "/test_dr8_bkg_zredc_sub.fits",
ext='01_02_REF')
testing.assert_almost_equal(bkg12['BC'], bkg12_compare['BC'], 2)
testing.assert_almost_equal(bkg12['N'], bkg12_compare['N'], 4)
# And delete the tempfile
os.remove(config.bkgfile_color)
def test_redmapper_run(self):
"""
Run test of redmapper.RedmapperRun.
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
# First, test the splitting
config.calib_run_nproc = 4
redmapper_run = RedmapperRun(config)
splits = redmapper_run._get_pixel_splits()
self.assertEqual(splits[0], 64)
testing.assert_array_equal(splits[1],
np.array([2163, 2296, 2297, 2434]))
# Now, this will just run on 1 but will test consolidation code
config.calib_run_nproc = 2
# Note you need these to be set to get same answer with nproc = 1 because
# of mask rounding
# config.d.hpix = 570
# config.d.nside = 32
config.seedfile = os.path.join(file_path, 'test_dr8_specseeds.fit')
config.zredfile = os.path.join(file_path, 'zreds_test',
'dr8_test_zreds_master_table.fit')
redmapper_run = RedmapperRun(config)
redmapper_run.run(specmode=True,
consolidate_like=True,
keepz=True,
seedfile=config.seedfile)
# Now let's check that we got the final file...
self.assertTrue(
os.path.isfile(
os.path.join(config.outpath,
'%s_final.fit' % (config.d.outbase))))
self.assertTrue(
os.path.isfile(
os.path.join(config.outpath,
'%s_final_members.fit' % (config.d.outbase))))
self.assertTrue(
os.path.isfile(
os.path.join(config.outpath,
'%s_like.fit' % (config.d.outbase))))
cat = Catalog.from_fits_file(
os.path.join(config.outpath, '%s_final.fit' % (config.d.outbase)))
# Spot checks to look for regressions
testing.assert_equal(cat.size, 24)
self.assertGreater(cat.Lambda.min(), 3.0)
testing.assert_array_almost_equal(
cat.Lambda[0:3], np.array([24.396917, 17.944063, 7.738485]))
# And check that the members are all accounted for...
mem = Catalog.from_fits_file(
os.path.join(config.outpath,
'%s_final_members.fit' % (config.d.outbase)))
a, b = esutil.numpy_util.match(cat.mem_match_id, mem.mem_match_id)
testing.assert_equal(a.size, mem.size)
def test_redmagic_calibrate(self):
"""
"""
np.random.seed(12345)
file_path = 'data_for_tests'
conf_filename = 'testconfig_redmagic.yaml'
config = Configuration(os.path.join(file_path, conf_filename))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
testgals = GalaxyCatalog.from_fits_file(
os.path.join('data_for_tests', 'redmagic_test',
'redmagic_test_input_gals.fit'))
testgals.add_fields([('mag', 'f4', 5), ('mag_err', 'f4', 5)])
redmagic_cal = RedmagicCalibrator(config)
# We have to have do_run=False here because we don't have a real
# galaxy training set with associated zreds!
redmagic_cal.run(gals=testgals, do_run=False)
# Read in the calibrated parameters
self.assertTrue(os.path.isfile(config.redmagicfile))
cal = fitsio.read(config.redmagicfile, ext=1)
# Check that they are what we think they should be
# (these checks are arbitrary, just to make sure nothing has changed)
testing.assert_almost_equal(
cal['cmax'][0, :], np.array([1.14588386, 3.89420298, -0.36792211]))
testing.assert_almost_equal(
cal['bias'][0, :], np.array([-0.09999912, -0.04537928,
0.01599778]))
testing.assert_almost_equal(
cal['eratio'][0, :], np.array([1.4999998, 1.48021495, 0.50000003]))
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 3)
# This is a hack of the volume limit mask to change from the one used for
# calibration to the one used for the run (which uses a different footprint
# because of reasons)
config_regular = Configuration(
os.path.join(file_path, 'testconfig.yaml'))
maskfile = config_regular.maskfile
config = Configuration(redmagic_cal.runfile)
cal, hdr = fitsio.read(config.redmagicfile, ext=1, header=True)
config.maskfile = maskfile
os.remove(cal['vmaskfile'][0].decode().rstrip())
mask = VolumeLimitMask(config, cal['etamin'], use_geometry=True)
# Now test the running, using the output file which has valid galaxies/zreds
run_redmagic = RunRedmagicTask(redmagic_cal.runfile)
run_redmagic.run()
# check that we have a redmagic catalog
rmcatfile = config.redmapper_filename('redmagic_%s' % ('highdens'))
self.assertTrue(os.path.isfile(rmcatfile))
# And a random catalog
rmrandfile = config.redmapper_filename('redmagic_%s_randoms' %
('highdens'))
self.assertTrue(
os.path.isfile(
config.redmapper_filename('redmagic_%s_randoms' %
('highdens'))))
# And check that the plot is there
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 4)
# And that we have the desired number of redmagic and randoms
red_cat = GalaxyCatalog.from_fits_file(rmcatfile)
rand_cat = GalaxyCatalog.from_fits_file(rmrandfile)
self.assertEqual(rand_cat.size, red_cat.size * 10)
# And confirm that all the randoms are in the footprint
zmax = mask.calc_zmax(rand_cat.ra, rand_cat.dec)
self.assertTrue(np.all(rand_cat.z < zmax))
def runTest(self):
file_path = 'data_for_tests'
cluster = Cluster()
conf_filename = 'testconfig.yaml'
cluster.config = Configuration(file_path + '/' + conf_filename)
filename = 'test_wcen_zred_data.fit'
tempcat = fitsio.read(file_path + '/' + filename, ext=1)
temp_neighbors = np.zeros(tempcat[0]['RAS'].size,
dtype=[('RA', 'f8'), ('DEC', 'f8'),
('DIST', 'f4'), ('R', 'f4'),
('P', 'f4'), ('PFREE', 'f4'),
('PMEM', 'f4'), ('MAG', 'f4', 5),
('MAG_ERR', 'f4', 5),
('REFMAG', 'f4'),
('REFMAG_ERR', 'f4'), ('CHISQ', 'f4'),
('ZRED', 'f4'), ('ZRED_E', 'f4'),
('ZRED_CHISQ', 'f4')])
temp_neighbors['RA'] = tempcat[0]['RAS']
temp_neighbors['DEC'] = tempcat[0]['DECS']
temp_neighbors['R'] = tempcat[0]['R']
temp_neighbors['P'] = tempcat[0]['PVALS']
temp_neighbors['PFREE'] = tempcat[0]['WVALS']
temp_neighbors['PMEM'] = tempcat[0]['WTVALS']
temp_neighbors['REFMAG'] = tempcat[0]['REFMAG_TOTAL']
temp_neighbors['ZRED'] = tempcat[0]['GZREDS']
temp_neighbors['ZRED_E'] = tempcat[0]['GZREDE']
temp_neighbors['ZRED_CHISQ'] = tempcat[0]['GCHISQ']
temp_neighbors['DIST'] = tempcat[0]['R'] / (
np.radians(1.) *
cluster.config.cosmo.Da(0, tempcat[0]['ZCLUSTER']))
neighbors = GalaxyCatalog(temp_neighbors)
cluster.set_neighbors(neighbors)
zred_filename = 'test_dr8_pars.fit'
cluster.zredstr = RedSequenceColorPar(file_path + '/' + zred_filename,
fine=True)
bkg_filename = 'test_bkg.fit'
cluster.bkg = Background('%s/%s' % (file_path, bkg_filename))
cluster.zredbkg = ZredBackground('%s/%s' % (file_path, bkg_filename))
cluster.redshift = tempcat[0]['ZCLUSTER']
cluster.ra = tempcat[0]['RAC']
cluster.dec = tempcat[0]['DECC']
cluster.r_lambda = 1.0 * (tempcat[0]['LAMBDA'] / 100.0)**0.2
cluster.Lambda = tempcat[0]['LAMBDA']
cluster.scaleval = tempcat[0]['SCALEVAL']
corr_filename = 'test_dr8_zlambdacorr.fit'
zlambda_corr = ZlambdaCorrectionPar(file_path + '/' + corr_filename,
30.0)
# And the meat of it...
cent = CenteringWcenZred(cluster, zlambda_corr=zlambda_corr)
cent.find_center()
testing.assert_almost_equal(cent.p_cen,
tempcat[0]['PCEN'][tempcat[0]['GOOD']], 5)
testing.assert_almost_equal(cent.q_cen,
tempcat[0]['QCEN'][tempcat[0]['GOOD']], 4)
testing.assert_almost_equal(cent.p_sat, tempcat[0]['PSAT'], 4)
testing.assert_almost_equal(cent.p_fg, tempcat[0]['PFG'], 4)
testing.assert_array_equal(cent.index,
tempcat[0]['USE'][tempcat[0]['GOOD']])
def runTest(self):
"""
Run the ClusterTest
"""
# all new...
random.seed(seed=12345)
file_path = 'data_for_tests'
cluster = Cluster()
conf_filename = 'testconfig.yaml'
cluster.config = Configuration(file_path + '/' + conf_filename)
filename = 'test_cluster_members.fit'
neighbors = GalaxyCatalog.from_fits_file(file_path + '/' + filename)
cluster.set_neighbors(neighbors)
zred_filename = 'test_dr8_pars.fit'
cluster.zredstr = RedSequenceColorPar(file_path + '/' + zred_filename,
fine=True)
bkg_filename = 'test_bkg.fit'
cluster.bkg = Background('%s/%s' % (file_path, bkg_filename))
hdr = fitsio.read_header(file_path + '/' + filename, ext=1)
cluster.redshift = hdr['Z']
richness_compare = hdr['LAMBDA']
richness_compare_err = hdr['LAMBDA_E']
scaleval_compare = hdr['SCALEVAL']
cpars_compare = np.array(
[hdr['CPARS0'], hdr['CPARS1'], hdr['CPARS2'], hdr['CPARS3']])
cval_compare = hdr['CVAL']
mstar_compare = hdr['MSTAR']
cluster.ra = hdr['RA']
cluster.dec = hdr['DEC']
mask = HPMask(cluster.config)
maskgal_index = mask.select_maskgals_sample(maskgal_index=0)
mask.set_radmask(cluster)
depthstr = DepthMap(cluster.config)
depthstr.calc_maskdepth(mask.maskgals, cluster.ra, cluster.dec,
cluster.mpc_scale)
# Test the NFW profile on its own
# (this works to 5 decimal places because of the 2*pi*r scaling)
nfw_python = cluster._calc_radial_profile()
testing.assert_almost_equal(nfw_python,
neighbors.nfw / (2. * np.pi * neighbors.r),
5)
# Test the background
# Note that this uses the input chisq values
bkg_python = cluster.calc_bkg_density(cluster.neighbors.r,
cluster.neighbors.chisq,
cluster.neighbors.refmag)
# this is cheating here...
to_test, = np.where(
(cluster.neighbors.refmag < cluster.bkg.refmagbins[-1]))
seed = 0
random.seed(seed=0)
richness = cluster.calc_richness(mask)
# these are regression tests. Various mask issues make the matching
# to idl for the time being
testing.assert_almost_equal(cluster.Lambda, 24.366407, 5)
testing.assert_almost_equal(cluster.lambda_e, 2.5137918, 5)
return
def test_maskgals(self):
"""
Test generation of maskgals file.
"""
# Note that due to historical reasons, this is testing the
# new generation of maskgals with some spot checks. Independently,
# it has been checked that the distributions are the same as for
# the old IDL code which was used to generate the reference used
# in the cluster tests.
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(os.path.join(file_path, conf_filename))
# For testing, and backwards compatibility, only make one
config.maskgal_nsamples = 1
config.mask_mode = 0
mask = get_mask(config)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
maskgalfile = os.path.join(self.test_dir, 'testmaskgal.fit')
random.seed(seed=12345)
# This will generate the file if it isn't there
mask.read_maskgals(maskgalfile)
maskgals, hdr = fitsio.read(maskgalfile, ext=1, header=True)
self.assertEqual(maskgals.size,
config.maskgal_ngals * config.maskgal_nsamples)
self.assertEqual(hdr['VERSION'], 6)
self.assertEqual(hdr['R0'], config.percolation_r0)
self.assertEqual(hdr['BETA'], config.percolation_beta)
self.assertEqual(hdr['STEPSIZE'], config.maskgal_rad_stepsize)
self.assertEqual(hdr['NMAG'], config.nmag)
self.assertEqual(hdr['NGALS'], config.maskgal_ngals)
self.assertEqual(hdr['CHISQMAX'], config.chisq_max)
self.assertEqual(hdr['LVALREF'], config.lval_reference)
self.assertEqual(hdr['EXTRA'], config.maskgal_dmag_extra)
self.assertEqual(hdr['ALPHA'], config.calib_lumfunc_alpha)
self.assertEqual(hdr['RSIG'], config.rsig)
self.assertEqual(hdr['ZREDERR'], config.maskgal_zred_err)
testing.assert_almost_equal(maskgals['r'][0:3],
[0.66900003, 0.119, 0.722])
testing.assert_almost_equal(maskgals['phi'][0:3],
[1.73098969, 2.53610063, 4.2362957])
testing.assert_almost_equal(maskgals['x'][0:3],
[-0.1067116, -0.09784444, -0.33090013])
testing.assert_almost_equal(maskgals['m'][0:3],
[0.46200001, 1.778, -1.43599999])
testing.assert_almost_equal(maskgals['chisq'][0:3],
[8.63599968, 2.28399992, 1.55799997])
testing.assert_almost_equal(maskgals['cwt'][0:3],
[0.02877194, 0.1822518, 0.17872778])
testing.assert_almost_equal(maskgals['nfw'][0:3],
[0.15366785, 0.32543495, 0.1454625])
testing.assert_almost_equal(maskgals['dzred'][0:3],
[-0.03090504, 0.00847131, -0.01800639])
testing.assert_almost_equal(maskgals['zwt'][0:3],
[6.0447073, 18.23568916, 13.30043507])
testing.assert_almost_equal(maskgals['lumwt'][0:3],
[0.39371657, 0.62304342, 0.01774093])
testing.assert_almost_equal(maskgals['theta_r'][0:3, 3],
[0.73237121, 1., 0.3299689])
testing.assert_almost_equal(maskgals['radbins'][0, 0:3],
[0.40000001, 0.5, 0.60000002])
testing.assert_almost_equal(maskgals['nin_orig'][0, 0:3],
[2213., 2663., 3066.])
testing.assert_almost_equal(
maskgals['nin'][0,
0:3], [2203.3347168, 2651.54467773, 3062.44628906])
def runTest(self):
"""
Run tests on redmapper.Zlambda
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
conf_filename = 'testconfig.yaml'
config = Configuration(file_path + '/' + conf_filename)
filename = 'test_cluster_members.fit'
neighbors = GalaxyCatalog.from_fits_file(file_path + '/' + filename)
zred_filename = 'test_dr8_pars.fit'
zredstr = RedSequenceColorPar(file_path + '/' + zred_filename,fine = True)
bkg_filename = 'test_bkg.fit'
bkg = Background('%s/%s' % (file_path, bkg_filename))
cluster = Cluster(config=config, zredstr=zredstr, bkg=bkg, neighbors=neighbors)
hdr=fitsio.read_header(file_path+'/'+filename,ext=1)
cluster.redshift = hdr['Z']
richness_compare = hdr['LAMBDA']
richness_compare_err = hdr['LAMBDA_E']
cluster.ra = hdr['RA']
cluster.dec = hdr['DEC']
#Set up the mask
mask = HPMask(cluster.config) #Create the mask
maskgal_index = mask.select_maskgals_sample(maskgal_index=0)
mask.set_radmask(cluster)
#depthstr
depthstr = DepthMap(cluster.config)
depthstr.calc_maskdepth(mask.maskgals, cluster.ra, cluster.dec, cluster.mpc_scale)
cluster.neighbors.dist = np.degrees(cluster.neighbors.r/cluster.cosmo.Dl(0,cluster.redshift))
# make a zlambda object
zlam = Zlambda(cluster)
z_lambda, z_lambda_e = zlam.calc_zlambda(cluster.redshift, mask, calc_err=True, calcpz=True)
# I am not sure why this isn't repeatable better than this
testing.assert_almost_equal(cluster.z_lambda, 0.22666427, 6)
testing.assert_almost_equal(cluster.z_lambda_err, 0.00443601, 4)
# zlambda_err test
z_lambda_err = zlam._zlambda_calc_gaussian_err(cluster.z_lambda)
testing.assert_almost_equal(z_lambda_err, 0.0063738347, 5)
# and test the correction on its own
corr_filename = 'test_dr8_zlambdacorr.fit'
zlambda_corr = ZlambdaCorrectionPar(file_path + '/' + corr_filename, zlambda_pivot=30.0)
zlam_in = 0.227865
zlam_e_in = 0.00629995
zlam_out = 0.228654
zlam_e_out = 0.00840213
zlam_new, zlam_e_new = zlambda_corr.apply_correction(24.5, zlam_in, zlam_e_in)
testing.assert_almost_equal(zlam_new, zlam_out, 5)
testing.assert_almost_equal(zlam_e_new, zlam_e_out, 5)
def test_redmagic_fitter(self):
np.random.seed(12345)
file_path = 'data_for_tests/redmagic_test'
# Read in the red-sequence parametrization
# Read in the input data for comparison (no afterburner)
calstr = fitsio.read(os.path.join(file_path, 'rcal_str_preab.fit'),
ext=1,
lower=True)
# Read in the input data for testing (no afterburner)
calstr2 = fitsio.read(os.path.join(file_path, 'rcal_str2.fit'),
ext=1,
lower=True)
# Make a zred structure for mstar...
config = Configuration(
os.path.join('data_for_tests', 'testconfig.yaml'))
zredstr = RedSequenceColorPar(None, config=config)
# Set up the fitter...
#randomn = np.random.normal(size=calstr2['z'][0, :].size)
# Old IDL code did not sample for the selection, I think this was wrong
randomn = np.zeros(calstr2['z'][0, :].size)
rmfitter = RedmagicParameterFitter(
calstr['nodes'][0, :],
calstr['corrnodes'][0, :],
calstr2['z'][0, :],
calstr2['z_err'][0, :],
calstr2['chisq'][0, :],
calstr2['mstar'][0, :],
calstr2['zcal'][0, :],
calstr2['zcal_e'][0, :],
calstr2['refmag'][0, :],
randomn,
calstr2['zmax'][0, :],
calstr['etamin'][0],
calstr['n0'][0],
calstr2['volume'][0, :],
calstr2['zrange'][0, :],
calstr2['zbinsize'][0],
zredstr,
maxchi=20.0,
ab_use=calstr2['afterburner_use'][0, :])
# These match the IDL values
testing.assert_almost_equal(rmfitter(calstr['cmax'][0, :]),
1.9331937798956758)
p0_cval = np.zeros(calstr['nodes'][0, :].size) + 2.0
testing.assert_almost_equal(rmfitter(p0_cval), 317.4524284321642)
cvals = rmfitter.fit(p0_cval)
# This does not match the IDL output, because this is doing a lot
# better job minimizing the function, at least in this test.
# I hope this is just because of the size of the testbed, which is
# really way too small for something like this.
testing.assert_almost_equal(
cvals, np.array([2.61657263, 2.20376531, 1.00663991]))
# Now we have to check the fitting with the afterburner
biasvals = np.zeros(rmfitter._corrnodes.size)
eratiovals = np.ones(rmfitter._corrnodes.size)
biasvals, eratiovals = rmfitter.fit_bias_eratio(
cvals, biasvals, eratiovals)
cvals = rmfitter.fit(cvals,
biaspars=biasvals,
eratiopars=eratiovals,
afterburner=True)
testing.assert_almost_equal(
cvals, np.array([3.39002141, 1.74421087, 0.93541002]))
testing.assert_almost_equal(
biasvals, np.array([0.00896487, -0.02456343, 0.02006761]))
testing.assert_almost_equal(
eratiovals, np.array([1.49999937, 1.01673233, 0.65657318]))
def runTest(self):
"""
Test redmapper.pipeline.RedmapperConsolidateTask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.consolidate_lambda_cuts = np.array([5.0, 20.0])
config.consolidate_vlim_lstars = np.array([])
# Make 3 fake simple catalogs...
random.seed(seed=12345)
nside = 4
test_arr = np.zeros(5, dtype=[('mem_match_id', 'i4'),
('ra', 'f8'),
('dec', 'f8'),
('lambda', 'f4'),
('lnlamlike', 'f4')])
# Do cat0, pixel 0
theta, phi = hp.pix2ang(nside, 0)
test_arr['mem_match_id'] = np.arange(test_arr.size) + 1
test_arr['lambda'][:] = 100.0
test_arr['lnlamlike'] = random.random(size=test_arr.size) * 100
test_arr['ra'][:] = np.degrees(phi)
test_arr['dec'][:] = 90.0 - np.degrees(theta)
cat0 = ClusterCatalog(test_arr)
cat0.to_fits_file(config.redmapper_filename('cat_4_00000_final'))
cat0.to_fits_file(config.redmapper_filename('cat_4_00000_final_members'))
# Do cat1, pixel 1
theta, phi = hp.pix2ang(nside, 1)
test_arr['mem_match_id'] = np.arange(test_arr.size) + 1
test_arr['lambda'][:] = 100.0
test_arr['lnlamlike'] = random.random(size=test_arr.size) * 100
test_arr['ra'][:] = np.degrees(phi)
test_arr['dec'][:] = 90.0 - np.degrees(theta)
cat1 = ClusterCatalog(test_arr)
cat1.to_fits_file(config.redmapper_filename('cat_4_00001_final'))
cat1.to_fits_file(config.redmapper_filename('cat_4_00001_final_members'))
# Do cat2, pixel 2
theta, phi = hp.pix2ang(nside, 2)
test_arr['mem_match_id'] = np.arange(test_arr.size) + 1
test_arr['lambda'][:] = 100.0
test_arr['lnlamlike'] = random.random(size=test_arr.size) * 100
test_arr['ra'][:] = np.degrees(phi)
test_arr['dec'][:] = 90.0 - np.degrees(theta)
cat2 = ClusterCatalog(test_arr)
cat2.to_fits_file(config.redmapper_filename('cat_4_00002_final'))
cat2.to_fits_file(config.redmapper_filename('cat_4_00002_final_members'))
# need to write config out in test directory...
config_file = config.redmapper_filename('testconfig', filetype='yaml')
config.output_yaml(config_file)
# Consolidate them together...
consol = RedmapperConsolidateTask(config_file)
consol.run(match_spec=False, do_plots=False)
# Check that the ordering is correct, etc.
catfile = config.redmapper_filename('redmapper_v%s_lgt20_catalog' % (config.version))
memfile = config.redmapper_filename('redmapper_v%s_lgt20_catalog_members' % (config.version))
self.assertTrue(os.path.isfile(catfile))
self.assertTrue(os.path.isfile(memfile))
cat = fitsio.read(catfile, ext=1)
self.assertEqual(cat.size, cat0.size + cat1.size + cat2.size)
# Sort by mem_match_id, these should be reverse sorted in lnlamlike
st = np.argsort(cat['mem_match_id'])
self.assertTrue(np.all(np.diff(cat['lnlamlike'][st[::-1]]) >= 0))
# And check that the ra/dec/lnlamlike match...
test0 = {}
for cluster in cat0:
test0[cluster.lnlamlike] = cluster.ra
ctr = 0
for i in range(cat.size):
if cat['lnlamlike'][i] in test0:
if cat['ra'][i] == test0[cat['lnlamlike'][i]]:
ctr += 1
self.assertEqual(ctr, cat0.size)
def runTest(self):
"""
First test the filters:
nfw, lum, and bkg
"""
# all new...
file_path = 'data_for_tests'
cluster = Cluster()
conf_filename = 'testconfig.yaml'
cluster.config = Configuration(file_path + '/' + conf_filename)
filename = 'test_cluster_members.fit'
neighbors = GalaxyCatalog.from_fits_file(file_path + '/' + filename)
cluster.set_neighbors(neighbors)
zred_filename = 'test_dr8_pars.fit'
cluster.zredstr = RedSequenceColorPar(file_path + '/' + zred_filename, fine=True)
bkg_filename = 'test_bkg.fit'
cluster.bkg = Background('%s/%s' % (file_path, bkg_filename))
hdr=fitsio.read_header(file_path+'/'+filename,ext=1)
#cluster.z = hdr['Z']
#cluster.update_z(hdr['Z'])
cluster.redshift = hdr['Z']
richness_compare = hdr['LAMBDA']
richness_compare_err = hdr['LAMBDA_E']
scaleval_compare = hdr['SCALEVAL']
cpars_compare = np.array([hdr['CPARS0'], hdr['CPARS1'], hdr['CPARS2'], hdr['CPARS3']])
cval_compare = hdr['CVAL']
mstar_compare = hdr['MSTAR']
cluster.ra = hdr['RA']
cluster.dec = hdr['DEC']
mask = HPMask(cluster.config)
mask.set_radmask(cluster)
depthstr = DepthMap(cluster.config)
depthstr.calc_maskdepth(mask.maskgals, cluster.ra, cluster.dec, cluster.mpc_scale)
# Test the NFW profile on its own
# (this works to 5 decimal places because of the 2*pi*r scaling)
nfw_python = cluster._calc_radial_profile()
testing.assert_almost_equal(nfw_python, neighbors.nfw/(2.*np.pi*neighbors.r),5)
# Test the luminosity
# The problem is that the IDL code has multiple ways of computing the index.
# And this should be fixed here. I don't want to duplicate dumb ideas.
#mstar = cluster.zredstr.mstar(cluster.z)
#testing.assert_almost_equal(mstar, mstar_compare, 3)
#maxmag = mstar - 2.5*np.log10(cluster.config.lval_reference)
#lum_python = cluster._calc_luminosity(maxmag)
#testing.assert_almost_equal(lum_python, neighbors.lumwt, 3)
# Test theta_i
#theta_i_python = calc_theta_i(neighbors.refmag, neighbors.refmag_err,
# maxmag, cluster.zredstr.limmag)
#testing.assert_almost_equal(theta_i_python, neighbors.theta_i, 3)
# Test the background
# Note that this uses the input chisq values
bkg_python = cluster.calc_bkg_density(cluster.neighbors.r,
cluster.neighbors.chisq,
cluster.neighbors.refmag)
# this is cheating here...
to_test, = np.where((cluster.neighbors.refmag < cluster.bkg.refmagbins[-1]))
#testing.assert_almost_equal(bkg_python[to_test], neighbors.bcounts[to_test], 3)
#testing.assert_allclose(bkg_python[to_test], neighbors.bcounts[to_test],
# rtol=1e-4, atol=0)
# skip this test for now. Blah.
# Now the cluster tests
# cluster.neighbors.dist = np.degrees(cluster.neighbors.r / cluster.cosmo.Da(0, cluster.redshift))
seed = 0
random.seed(seed = 0)
richness = cluster.calc_richness(mask)
# these are regression tests. Various mask issues make the matching
# to idl for the time being
testing.assert_almost_equal(cluster.Lambda, 23.86299324)
testing.assert_almost_equal(cluster.lambda_e, 2.4780307)
#testing.assert_almost_equal(cluster.neighbors.theta_i,
# neighbors.theta_i, 3)
#testing.assert_almost_equal(cluster.neighbors.theta_r,
# neighbors.theta_r, 3)
#testing.assert_almost_equal(cluster.neighbors.p,
# neighbors.p, 3)
return
def runTest(self):
"""
This tests the mask.py module. Since this
module is still in developement, so too are
these unit tests.
First test the red_maskgals() and gen_maskgals() functions.
Note: the gen_maskgals() function isn't written yet, so it
can't be tested. TODO
Next create a HPMask, check that it has maskgals and then
test to see if nside, offset, and npix are
the values that we expect them to be. These are
variables that tell us about how healpix is formatted.
Next test the fracgood array and its features, including
the fracgood_float and various fracgood entries.
Next test the compute_radmask() function which
finds if an array of (RA,DEC) pairs are in or out
of the mask. TODO
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
# First, test a bare mask
config.mask_mode = 0
mask = get_mask(config)
testing.assert_equal(hasattr(mask, 'maskgals'), True)
testing.assert_equal(isinstance(mask, Mask), True)
testing.assert_equal(isinstance(mask, HPMask), False)
# And the healpix mask
config.mask_mode = 3
mask = get_mask(config)
testing.assert_equal(hasattr(mask, 'maskgals'), True)
testing.assert_equal(isinstance(mask, Mask), True)
testing.assert_equal(isinstance(mask, HPMask), True)
# When ready, add in test of gen_maskgals()
# Test the healpix configuration
testing.assert_equal(mask.nside, 2048)
testing.assert_equal(mask.offset, 2100800)
testing.assert_equal(mask.npix, 548292)
# Next test that the fracgood is working properly
indices = [396440, 445445, 99547, 354028, 516163] #Random indices
true_fracgoods = np.array([0, 0, 0.828125, 0.796875,
0.828125]) #known fracgoods at indices
testing.assert_equal(mask.fracgood_float, 1)
testing.assert_equal(mask.fracgood.shape[0], 548292)
testing.assert_equal(mask.fracgood[indices], true_fracgoods)
# Next test the compute_radmask() function
# Note: RA and DECs are in degrees here
RAs = np.array([142.10934, 142.04090, 142.09242, 142.11448, 50.0])
Decs = np.array([65.022666, 65.133844, 65.084844, 65.109541, 50.0])
comp = np.array([True, True, True, True, False])
testing.assert_equal(mask.compute_radmask(RAs, Decs), comp)
# And test that we're getting the right numbers from a sub-mask
config2 = Configuration(file_path + "/" + conf_filename)
config2.hpix = 582972
config2.nside = 1024
config2.border = 0.02
mask2 = get_mask(config2)
comp = np.array([False, True, True, True, False])
testing.assert_equal(mask2.compute_radmask(RAs, Decs), comp)
def test_galaxycatalog_create(self):
"""
Run `redmapper.GalaxyCatalogMaker` tests.
"""
# Make a test directory
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
info_dict = {'LIM_REF': 21.0,
'REF_IND': 3,
'AREA': 25.0,
'NMAG': 5,
'MODE': 'SDSS',
'ZP': 22.5,
'U_IND': 0,
'G_IND': 1,
'R_IND': 2,
'I_IND': 3,
'Z_IND': 3}
# Test 1: read in catalog, write it to a single file, and then
# try to split it up
# make sure that it makes it properly, and that the output number
# of files is the same as input number of files.
configfile = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(configfile)
gals = GalaxyCatalog.from_galfile(config.galfile)
tab = Entry.from_fits_file(config.galfile)
maker = GalaxyCatalogMaker(os.path.join(self.test_dir, 'test_working'), info_dict, nside=tab.nside)
maker.append_galaxies(gals._ndarray)
maker.finalize_catalog()
tab2 = Entry.from_fits_file(os.path.join(self.test_dir, 'test_working_master_table.fit'))
self.assertEqual(tab.nside, tab2.nside)
self.assertEqual(tab.filenames.size, tab2.filenames.size)
for filename in tab2.filenames:
self.assertTrue(os.path.isfile(os.path.join(self.test_dir, filename.decode())))
# Test 2: Make a catalog that has an incomplete dtype
dtype = [('id', 'i8'),
('ra', 'f8')]
maker = GalaxyCatalogMaker(os.path.join(self.test_dir, 'test'), info_dict)
testgals = np.zeros(10, dtype=dtype)
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
# Test 3: make a catalog that has the wrong number of magnitudes
dtype = GalaxyCatalogMaker.get_galaxy_dtype(3)
testgals = np.zeros(10, dtype=dtype)
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
# Test 4: make a catalog that has some NaNs
dtype = GalaxyCatalogMaker.get_galaxy_dtype(info_dict['NMAG'])
testgals = np.ones(10, dtype=dtype)
testgals['mag'][0, 1] = np.nan
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
# Test 5: make a catalog that has ra/dec out of range
testgals = np.ones(10, dtype=dtype)
testgals['ra'][1] = -1.0
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
testgals = np.ones(10, dtype=dtype)
testgals['dec'][1] = -100.0
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
# Test 6: make a catalog that has mag > 90.0
testgals = np.ones(10, dtype=dtype)
testgals['mag'][0, 1] = 100.0
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
# Test 7: make a catalog that has mag_err == 0.0
testgals = np.ones(10, dtype=dtype)
testgals['mag_err'][0, 1] = 0.0
self.assertRaises(RuntimeError, maker.append_galaxies, testgals)
def test_redmagic_calibrate(self):
"""
"""
np.random.seed(12345)
file_path = 'data_for_tests'
conf_filename = 'testconfig_redmagic.yaml'
config = Configuration(os.path.join(file_path, conf_filename))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
testgals = GalaxyCatalog.from_fits_file(os.path.join('data_for_tests', 'redmagic_test', 'redmagic_test_input_gals.fit'))
testgals.add_fields([('mag', 'f4', 5), ('mag_err', 'f4', 5)])
redmagic_cal = RedmagicCalibrator(config)
# We have to have do_run=False here because we don't have a real
# galaxy training set with associated zreds!
redmagic_cal.run(gals=testgals, do_run=False)
# Read in the calibrated parameters
self.assertTrue(os.path.isfile(config.redmagicfile))
cal = fitsio.read(config.redmagicfile, ext=1)
# Check that they are what we think they should be
# (these checks are arbitrary, just to make sure nothing has changed)
testing.assert_almost_equal(cal['cmax'][0, :], np.array([1.14588386, 3.89420298, -0.36792211]))
testing.assert_almost_equal(cal['bias'][0, :], np.array([-0.09999912, -0.04537928, 0.01599778]))
testing.assert_almost_equal(cal['eratio'][0, :], np.array([1.4999998, 1.48021495, 0.50000003]))
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 3)
# This is a hack of the volume limit mask to change from the one used for
# calibration to the one used for the run (which uses a different footprint
# because of reasons)
config_regular = Configuration(os.path.join(file_path, 'testconfig.yaml'))
maskfile = config_regular.maskfile
config = Configuration(redmagic_cal.runfile)
cal, hdr = fitsio.read(config.redmagicfile, ext=1, header=True)
config.maskfile = maskfile
os.remove(cal['vmaskfile'][0].decode().rstrip())
mask = VolumeLimitMask(config, cal['etamin'], use_geometry=True)
# Now test the running, using the output file which has valid galaxies/zreds
run_redmagic = RunRedmagicTask(redmagic_cal.runfile)
run_redmagic.run()
# check that we have a redmagic catalog
rmcatfile = config.redmapper_filename('redmagic_%s' % ('highdens'))
self.assertTrue(os.path.isfile(rmcatfile))
# And a random catalog
rmrandfile = config.redmapper_filename('redmagic_%s_randoms' % ('highdens'))
self.assertTrue(os.path.isfile(config.redmapper_filename('redmagic_%s_randoms' % ('highdens'))))
# And check that the plot is there
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 4)
# And that we have the desired number of redmagic and randoms
red_cat = GalaxyCatalog.from_fits_file(rmcatfile)
rand_cat = GalaxyCatalog.from_fits_file(rmrandfile)
self.assertEqual(rand_cat.size, red_cat.size * 10)
# And confirm that all the randoms are in the footprint
zmax = mask.calc_zmax(rand_cat.ra, rand_cat.dec)
self.assertTrue(np.all(rand_cat.z < zmax))
def runTest(self):
file_path = 'data_for_tests'
conffile = 'testconfig.yaml'
config = Configuration(file_path + '/' + conffile)
gals_all = GalaxyCatalog.from_galfile(config.galfile)
zred_filename = 'test_dr8_pars.fit'
zredstr = RedSequenceColorPar(file_path + '/' + zred_filename, fine=True)
bkg_filename = 'test_bkg.fit'
bkg = Background('%s/%s' % (file_path, bkg_filename))
mask = HPMask(config)
depthstr = DepthMap(config)
#cosmo = Cosmo()
testcatfile = 'test_cluster_pos.fit'
cat = ClusterCatalog.from_catfile(file_path + '/' + testcatfile,
zredstr=zredstr,
config=config,
bkg=bkg)
# test single neighbors...
c0 = cat[0]
c0.find_neighbors(0.2, gals_all)
c1 = cat[1]
c1.find_neighbors(0.2, gals_all)
testing.assert_equal(c0.neighbors.size, 580)
testing.assert_equal(c1.neighbors.size, 298)
testing.assert_array_less(c0.neighbors.dist, 0.2)
testing.assert_array_less(c1.neighbors.dist, 0.2)
# and multi-match...
i0, i1, dist = gals_all.match_many(cat.ra, cat.dec, 0.2)
u0, = np.where(i0 == 0)
testing.assert_equal(c0.neighbors.size, u0.size)
u1, = np.where(i0 == 1)
testing.assert_equal(c1.neighbors.size, u1.size)
# and compute the richness on the first one...
mask.set_radmask(c0)
depthstr.calc_maskdepth(mask.maskgals, c0.ra, c0.dec, c0.mpc_scale)
richness = c0.calc_richness(mask)
# Make sure the numbers were propagated to the parent catalog
testing.assert_equal(richness, cat.Lambda[0])
testing.assert_equal(c0.Lambda_e, cat.Lambda_e[0])
testing.assert_equal(c0.scaleval, cat.scaleval[0])
# And make sure the numbers are correct
testing.assert_almost_equal(richness, 23.86299324)
def test_zred_runpixels(self):
"""
Test redmapper.ZredRunPixels, computing zreds for all the galaxies
in a pixelized galaxy catalog.
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
config.d.hpix = 2163
config.d.nside = 64
config.border = 0.0
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.zredfile = os.path.join(self.test_dir, 'zreds',
'testing_zreds_master_table.fit')
# FIXME: try an illegal one...
zredRunpix = ZredRunPixels(config)
zredRunpix.run()
# Check that the zred file has been built...
self.assertTrue(os.path.isfile(config.zredfile))
# Read in just the galaxies...
gals0 = GalaxyCatalog.from_galfile(config.galfile,
nside=config.d.nside,
hpix=config.d.hpix,
border=config.border)
# And with the zreds...
gals = GalaxyCatalog.from_galfile(config.galfile,
zredfile=config.zredfile,
nside=config.d.nside,
hpix=config.d.hpix,
border=config.border)
# Confirm they're the same galaxies...
testing.assert_array_almost_equal(gals0.ra, gals.ra)
# Confirm the zreds are okay
self.assertGreater(np.min(gals.zred), 0.0)
self.assertGreater(np.min(gals.chisq), 0.0)
self.assertLess(np.max(gals.lkhd), 0.0)
zredfile = os.path.join(file_path, 'zreds_test',
'dr8_test_zreds_master_table.fit')
gals_compare = GalaxyCatalog.from_galfile(config.galfile,
zredfile=zredfile,
nside=config.d.nside,
hpix=config.d.hpix,
border=config.border)
zredstr = RedSequenceColorPar(config.parfile)
mstar_input = zredstr.mstar(gals_compare.zred_uncorr)
mstar = zredstr.mstar(gals_compare.zred_uncorr)
ok, = np.where(
(gals_compare.refmag < (mstar_input - 2.5 * np.log10(0.15)))
| (gals_compare.refmag < (mstar - 2.5 * np.log10(0.15))))
delta_zred_uncorr = gals.zred_uncorr[ok] - gals_compare.zred_uncorr[ok]
use, = np.where(np.abs(delta_zred_uncorr) < 1e-3)
testing.assert_array_less(0.98, float(use.size) / float(ok.size))