def runTest(self):
"""
Run tests of redmapper.VolumeLimitMask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
vlim = VolumeLimitMask(config, config.vlim_lstar)
# And test the values...
ras = np.array([140.0, 141.0, 150.0])
decs = np.array([65.25, 65.6, 30.0])
zmax = vlim.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax, [0.338, 0.341, 0.0])
# And compute a geometry mask
vlim_geom = VolumeLimitMask(config, config.vlim_lstar + 1.0, use_geometry=True)
zmax_geom = vlim_geom.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax_geom, [config.zrange[1], config.zrange[1], 0.0])
def test_generatebkg(self):
"""
Test generation of a background file.
"""
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
config.calib_nproc = 1
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.bkgfile = os.path.join(config.outpath, '%s_testbkg.fit' % (config.d.outbase))
config.zrange = [0.1, 0.2]
gen = BackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
bkg = fitsio.read(config.bkgfile, ext='CHISQBKG')
# Some spot-testing...
testing.assert_equal(bkg[0]['sigma_g'].shape, (48, 40, 5))
testing.assert_equal(bkg[0]['sigma_lng'].shape, (48, 40, 5))
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 20, 2], 2.8444533)
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 10, 3], 7.4324579)
testing.assert_almost_equal(bkg[0]['sigma_lng'][30, 10, 3], 3.7618985)
testing.assert_almost_equal(bkg[0]['sigma_lng'][45, 10, 3], 0.0)
def runTest(self):
"""
Run tests of redmapper.VolumeLimitMask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
vlim = VolumeLimitMask(config, config.vlim_lstar)
# And test the values...
ras = np.array([140.0, 141.0, 150.0])
decs = np.array([65.25, 65.6, 30.0])
zmax = vlim.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax, [0.338, 0.341, 0.0])
# And compute a geometry mask
vlim_geom = VolumeLimitMask(config,
config.vlim_lstar + 1.0,
use_geometry=True)
zmax_geom = vlim_geom.calc_zmax(ras, decs, get_fracgood=False)
testing.assert_almost_equal(zmax_geom,
[config.zrange[1], config.zrange[1], 0.0])
def test_generatebkg(self):
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = test_dir
config.bkgfile = os.path.join(config.outpath, '%s_testbkg.fit' % (config.outbase))
config.zrange = [0.1, 0.2]
gen = BackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
bkg = fitsio.read(config.bkgfile, ext='CHISQBKG')
# Some spot-testing...
testing.assert_equal(bkg[0]['sigma_g'].shape, (48, 40, 5))
testing.assert_equal(bkg[0]['sigma_lng'].shape, (48, 40, 5))
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 20, 2], 2.8444533)
testing.assert_almost_equal(bkg[0]['sigma_g'][30, 10, 3], 7.4324579)
testing.assert_almost_equal(bkg[0]['sigma_lng'][30, 10, 3], 3.7618985)
testing.assert_almost_equal(bkg[0]['sigma_lng'][45, 10, 3], 0.0)
if os.path.exists(test_dir):
shutil.rmtree(test_dir, True)
def test_maskgals(self):
"""
Test generation of maskgals file.
"""
# Note that due to historical reasons, this is testing the
# new generation of maskgals with some spot checks. Independently,
# it has been checked that the distributions are the same as for
# the old IDL code which was used to generate the reference used
# in the cluster tests.
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(os.path.join(file_path, conf_filename))
# For testing, and backwards compatibility, only make one
config.maskgal_nsamples = 1
config.mask_mode = 0
mask = get_mask(config)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
maskgalfile = os.path.join(self.test_dir, 'testmaskgal.fit')
random.seed(seed=12345)
# This will generate the file if it isn't there
mask.read_maskgals(maskgalfile)
maskgals, hdr = fitsio.read(maskgalfile, ext=1, header=True)
self.assertEqual(maskgals.size, config.maskgal_ngals * config.maskgal_nsamples)
self.assertEqual(hdr['VERSION'], 6)
self.assertEqual(hdr['R0'], config.percolation_r0)
self.assertEqual(hdr['BETA'], config.percolation_beta)
self.assertEqual(hdr['STEPSIZE'], config.maskgal_rad_stepsize)
self.assertEqual(hdr['NMAG'], config.nmag)
self.assertEqual(hdr['NGALS'], config.maskgal_ngals)
self.assertEqual(hdr['CHISQMAX'], config.chisq_max)
self.assertEqual(hdr['LVALREF'], config.lval_reference)
self.assertEqual(hdr['EXTRA'], config.maskgal_dmag_extra)
self.assertEqual(hdr['ALPHA'], config.calib_lumfunc_alpha)
self.assertEqual(hdr['RSIG'], config.rsig)
self.assertEqual(hdr['ZREDERR'], config.maskgal_zred_err)
testing.assert_almost_equal(maskgals['r'][0: 3], [0.66900003, 0.119, 0.722])
testing.assert_almost_equal(maskgals['phi'][0: 3], [1.73098969, 2.53610063, 4.2362957])
testing.assert_almost_equal(maskgals['x'][0: 3], [-0.1067116, -0.09784444, -0.33090013])
testing.assert_almost_equal(maskgals['m'][0: 3], [0.46200001, 1.778, -1.43599999])
testing.assert_almost_equal(maskgals['chisq'][0: 3], [8.63599968, 2.28399992, 1.55799997])
testing.assert_almost_equal(maskgals['cwt'][0: 3], [0.02877194, 0.1822518, 0.17872778])
testing.assert_almost_equal(maskgals['nfw'][0: 3], [0.15366785, 0.32543495, 0.1454625])
testing.assert_almost_equal(maskgals['dzred'][0: 3], [-0.03090504, 0.00847131, -0.01800639])
testing.assert_almost_equal(maskgals['zwt'][0: 3], [6.0447073, 18.23568916, 13.30043507])
testing.assert_almost_equal(maskgals['lumwt'][0: 3], [0.39371657, 0.62304342, 0.01774093])
testing.assert_almost_equal(maskgals['theta_r'][0: 3, 3], [0.73237121, 1., 0.3299689])
testing.assert_almost_equal(maskgals['radbins'][0, 0: 3], [0.40000001, 0.5, 0.60000002])
testing.assert_almost_equal(maskgals['nin_orig'][0, 0: 3], [2213., 2663., 3066.])
testing.assert_almost_equal(maskgals['nin'][0, 0: 3], [2203.3347168, 2651.54467773, 3062.44628906])
def test_selectspecseeds(self):
"""
Run tests on redmapper.calibration.SelectSpecSeeds
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
def test_selectspecseeds(self):
"""
Run tests on redmapper.calibration.SelectSpecSeeds
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
def test_zred_runcat(self):
"""
Test redmapper.ZredRunCatalog, computing zreds for all the galaxies in
a single catalog file.
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
outfile = os.path.join(self.test_dir, 'test_zred_out.fits')
tab = fitsio.read(config.galfile, ext=1, lower=True)
galfile = os.path.join(os.path.dirname(config.galfile),
tab[0]['filenames'][0].decode())
zredRuncat = ZredRunCatalog(config)
zredRuncat.run(galfile, outfile)
# This exercises the reading code
gals = GalaxyCatalog.from_galfile(galfile, zredfile=outfile)
self.assertGreater(np.min(gals.zred), 0.0)
self.assertGreater(np.min(gals.chisq), 0.0)
self.assertLess(np.max(gals.lkhd), 0.0)
# And compare to the "official" run...
config.zredfile = os.path.join(file_path, 'zreds_test',
'dr8_test_zreds_master_table.fit')
ztab = fitsio.read(config.zredfile, ext=1, lower=True)
zredfile = os.path.join(os.path.dirname(config.zredfile),
ztab[0]['filenames'][0].decode())
gals_compare = GalaxyCatalog.from_galfile(galfile, zredfile=zredfile)
zredstr = RedSequenceColorPar(config.parfile)
mstar_input = zredstr.mstar(gals_compare.zred_uncorr)
mstar = zredstr.mstar(gals_compare.zred_uncorr)
ok, = np.where(
(gals_compare.refmag < (mstar_input - 2.5 * np.log10(0.15)))
| (gals_compare.refmag < (mstar - 2.5 * np.log10(0.15))))
delta_zred_uncorr = gals.zred_uncorr[ok] - gals_compare.zred_uncorr[ok]
use, = np.where(np.abs(delta_zred_uncorr) < 1e-3)
testing.assert_array_less(0.98, float(use.size) / float(ok.size))
def test_zred_runcat(self):
"""
Test redmapper.ZredRunCatalog, computing zreds for all the galaxies in
a single catalog file.
"""
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
outfile = os.path.join(self.test_dir, 'test_zred_out.fits')
tab = fitsio.read(config.galfile, ext=1, lower=True)
galfile = os.path.join(os.path.dirname(config.galfile), tab[0]['filenames'][0].decode())
zredRuncat = ZredRunCatalog(config)
zredRuncat.run(galfile, outfile)
# This exercises the reading code
gals = GalaxyCatalog.from_galfile(galfile, zredfile=outfile)
self.assertGreater(np.min(gals.zred), 0.0)
self.assertGreater(np.min(gals.chisq), 0.0)
self.assertLess(np.max(gals.lkhd), 0.0)
# And compare to the "official" run...
config.zredfile = os.path.join(file_path, 'zreds_test', 'dr8_test_zreds_master_table.fit')
ztab = fitsio.read(config.zredfile, ext=1, lower=True)
zredfile = os.path.join(os.path.dirname(config.zredfile), ztab[0]['filenames'][0].decode())
gals_compare = GalaxyCatalog.from_galfile(galfile, zredfile=zredfile)
zredstr = RedSequenceColorPar(config.parfile)
mstar_input = zredstr.mstar(gals_compare.zred_uncorr)
mstar = zredstr.mstar(gals_compare.zred_uncorr)
ok, = np.where((gals_compare.refmag < (mstar_input - 2.5*np.log10(0.15))) |
(gals_compare.refmag < (mstar - 2.5*np.log10(0.15))))
delta_zred_uncorr = gals.zred_uncorr[ok] - gals_compare.zred_uncorr[ok]
use, = np.where(np.abs(delta_zred_uncorr) < 1e-3)
testing.assert_array_less(0.98, float(use.size) / float(ok.size))
def test_generatezredbkg(self):
"""
Test generation of a zred background file.
"""
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.bkgfile = os.path.join(config.outpath,
'%s_testbkg.fit' % (config.d.outbase))
config.zrange = [0.1, 0.2]
# First test without a zred file ...
gen = ZredBackgroundGenerator(config)
self.assertRaises(RuntimeError, gen.run)
# And now fix it ...
config.zredfile = os.path.join('data_for_tests', 'zreds_test',
'dr8_test_zreds_master_table.fit')
gen = ZredBackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
zbkg = fitsio.read(config.bkgfile, ext='ZREDBKG')
# Some spot-testing...
# (The numbers have been checked to be consistent with the full run tested above
# but can't be directly compared because this is much noisier)
testing.assert_equal(zbkg[0]['sigma_g'].shape, (48, 10))
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 5],
620.0223999,
decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][47, 8],
30501.8398438,
decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 0],
384.3362732,
decimal=5)
def test_run_colormem(self):
"""
Run tests of redmapper.RunColormem
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
# First, we need the red galaxy model
config.specfile_train = os.path.join(file_path, 'test_dr8_spec.fit')
config.zrange = [0.1, 0.2]
config.redgalfile = config.redmapper_filename('test_redgals')
config.redgalmodelfile = config.redmapper_filename('test_redgalmodel')
selred = SelectSpecRedGalaxies(config)
selred.run()
# Main test...
config.zmemfile = config.redmapper_filename('test_zmem')
rcm = RunColormem(config)
rcm.run()
rcm.output_training()
# Check that the files are there...
self.assertTrue(os.path.isfile(config.zmemfile))
mem = fitsio.read(config.zmemfile, ext=1)
testing.assert_equal(mem.size, 16)
testing.assert_array_almost_equal(
mem['pcol'][0:3], np.array([0.94829756, 0.83803916, 0.88315928]))
testing.assert_array_almost_equal(
mem['z'][0:3], np.array([0.191279, 0.188257, 0.186945]))
def test_run_colormem(self):
"""
Run tests of redmapper.RunColormem
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
# First, we need the red galaxy model
config.specfile_train = os.path.join(file_path, 'test_dr8_spec.fit')
config.zrange = [0.1,0.2]
config.redgalfile = config.redmapper_filename('test_redgals')
config.redgalmodelfile = config.redmapper_filename('test_redgalmodel')
selred = SelectSpecRedGalaxies(config)
selred.run()
# Main test...
config.zmemfile = config.redmapper_filename('test_zmem')
rcm = RunColormem(config)
rcm.run()
rcm.output_training()
# Check that the files are there...
self.assertTrue(os.path.isfile(config.zmemfile))
mem = fitsio.read(config.zmemfile, ext=1)
testing.assert_equal(mem.size, 16)
testing.assert_array_almost_equal(mem['pcol'][0:3], np.array([0.94829756, 0.83803916, 0.88315928]))
testing.assert_array_almost_equal(mem['z'][0:3], np.array([0.191279, 0.188257, 0.186945]))
def test_generatezredbkg(self):
"""
Test generation of a zred background file.
"""
config_file = os.path.join('data_for_tests', 'testconfig.yaml')
config = Configuration(config_file)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.bkgfile = os.path.join(config.outpath, '%s_testbkg.fit' % (config.d.outbase))
config.zrange = [0.1, 0.2]
# First test without a zred file ...
gen = ZredBackgroundGenerator(config)
self.assertRaises(RuntimeError, gen.run)
# And now fix it ...
config.zredfile = os.path.join('data_for_tests', 'zreds_test', 'dr8_test_zreds_master_table.fit')
gen = ZredBackgroundGenerator(config)
gen.run(clobber=True)
self.assertTrue(os.path.isfile(config.bkgfile))
zbkg = fitsio.read(config.bkgfile, ext='ZREDBKG')
# Some spot-testing...
# (The numbers have been checked to be consistent with the full run tested above
# but can't be directly compared because this is much noisier)
testing.assert_equal(zbkg[0]['sigma_g'].shape, (48, 10))
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 5], 620.0223999, decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][47, 8], 30501.8398438, decimal=5)
testing.assert_almost_equal(zbkg[0]['sigma_g'][30, 0], 384.3362732, decimal=5)
def test_redmagic_calibrate(self):
"""
"""
np.random.seed(12345)
file_path = 'data_for_tests'
conf_filename = 'testconfig_redmagic.yaml'
config = Configuration(os.path.join(file_path, conf_filename))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
testgals = GalaxyCatalog.from_fits_file(
os.path.join('data_for_tests', 'redmagic_test',
'redmagic_test_input_gals.fit'))
testgals.add_fields([('mag', 'f4', 5), ('mag_err', 'f4', 5)])
redmagic_cal = RedmagicCalibrator(config)
# We have to have do_run=False here because we don't have a real
# galaxy training set with associated zreds!
redmagic_cal.run(gals=testgals, do_run=False)
# Read in the calibrated parameters
self.assertTrue(os.path.isfile(config.redmagicfile))
cal = fitsio.read(config.redmagicfile, ext=1)
# Check that they are what we think they should be
# (these checks are arbitrary, just to make sure nothing has changed)
testing.assert_almost_equal(
cal['cmax'][0, :], np.array([1.14588386, 3.89420298, -0.36792211]))
testing.assert_almost_equal(
cal['bias'][0, :], np.array([-0.09999912, -0.04537928,
0.01599778]))
testing.assert_almost_equal(
cal['eratio'][0, :], np.array([1.4999998, 1.48021495, 0.50000003]))
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 3)
# This is a hack of the volume limit mask to change from the one used for
# calibration to the one used for the run (which uses a different footprint
# because of reasons)
config_regular = Configuration(
os.path.join(file_path, 'testconfig.yaml'))
maskfile = config_regular.maskfile
config = Configuration(redmagic_cal.runfile)
cal, hdr = fitsio.read(config.redmagicfile, ext=1, header=True)
config.maskfile = maskfile
os.remove(cal['vmaskfile'][0].decode().rstrip())
mask = VolumeLimitMask(config, cal['etamin'], use_geometry=True)
# Now test the running, using the output file which has valid galaxies/zreds
run_redmagic = RunRedmagicTask(redmagic_cal.runfile)
run_redmagic.run()
# check that we have a redmagic catalog
rmcatfile = config.redmapper_filename('redmagic_%s' % ('highdens'))
self.assertTrue(os.path.isfile(rmcatfile))
# And a random catalog
rmrandfile = config.redmapper_filename('redmagic_%s_randoms' %
('highdens'))
self.assertTrue(
os.path.isfile(
config.redmapper_filename('redmagic_%s_randoms' %
('highdens'))))
# And check that the plot is there
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 4)
# And that we have the desired number of redmagic and randoms
red_cat = GalaxyCatalog.from_fits_file(rmcatfile)
rand_cat = GalaxyCatalog.from_fits_file(rmrandfile)
self.assertEqual(rand_cat.size, red_cat.size * 10)
# And confirm that all the randoms are in the footprint
zmax = mask.calc_zmax(rand_cat.ra, rand_cat.dec)
self.assertTrue(np.all(rand_cat.z < zmax))
def runTest(self):
"""
Test redmapper.pipeline.RedmapperConsolidateTask
"""
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(file_path + "/" + conf_filename)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
config.consolidate_lambda_cuts = np.array([5.0, 20.0])
config.consolidate_vlim_lstars = np.array([])
# Make 3 fake simple catalogs...
random.seed(seed=12345)
nside = 4
test_arr = np.zeros(5, dtype=[('mem_match_id', 'i4'),
('ra', 'f8'),
('dec', 'f8'),
('lambda', 'f4'),
('lnlamlike', 'f4')])
# Do cat0, pixel 0
theta, phi = hp.pix2ang(nside, 0)
test_arr['mem_match_id'] = np.arange(test_arr.size) + 1
test_arr['lambda'][:] = 100.0
test_arr['lnlamlike'] = random.random(size=test_arr.size) * 100
test_arr['ra'][:] = np.degrees(phi)
test_arr['dec'][:] = 90.0 - np.degrees(theta)
cat0 = ClusterCatalog(test_arr)
cat0.to_fits_file(config.redmapper_filename('cat_4_00000_final'))
cat0.to_fits_file(config.redmapper_filename('cat_4_00000_final_members'))
# Do cat1, pixel 1
theta, phi = hp.pix2ang(nside, 1)
test_arr['mem_match_id'] = np.arange(test_arr.size) + 1
test_arr['lambda'][:] = 100.0
test_arr['lnlamlike'] = random.random(size=test_arr.size) * 100
test_arr['ra'][:] = np.degrees(phi)
test_arr['dec'][:] = 90.0 - np.degrees(theta)
cat1 = ClusterCatalog(test_arr)
cat1.to_fits_file(config.redmapper_filename('cat_4_00001_final'))
cat1.to_fits_file(config.redmapper_filename('cat_4_00001_final_members'))
# Do cat2, pixel 2
theta, phi = hp.pix2ang(nside, 2)
test_arr['mem_match_id'] = np.arange(test_arr.size) + 1
test_arr['lambda'][:] = 100.0
test_arr['lnlamlike'] = random.random(size=test_arr.size) * 100
test_arr['ra'][:] = np.degrees(phi)
test_arr['dec'][:] = 90.0 - np.degrees(theta)
cat2 = ClusterCatalog(test_arr)
cat2.to_fits_file(config.redmapper_filename('cat_4_00002_final'))
cat2.to_fits_file(config.redmapper_filename('cat_4_00002_final_members'))
# need to write config out in test directory...
config_file = config.redmapper_filename('testconfig', filetype='yaml')
config.output_yaml(config_file)
# Consolidate them together...
consol = RedmapperConsolidateTask(config_file)
consol.run(match_spec=False, do_plots=False)
# Check that the ordering is correct, etc.
catfile = config.redmapper_filename('redmapper_v%s_lgt20_catalog' % (config.version))
memfile = config.redmapper_filename('redmapper_v%s_lgt20_catalog_members' % (config.version))
self.assertTrue(os.path.isfile(catfile))
self.assertTrue(os.path.isfile(memfile))
cat = fitsio.read(catfile, ext=1)
self.assertEqual(cat.size, cat0.size + cat1.size + cat2.size)
# Sort by mem_match_id, these should be reverse sorted in lnlamlike
st = np.argsort(cat['mem_match_id'])
self.assertTrue(np.all(np.diff(cat['lnlamlike'][st[::-1]]) >= 0))
# And check that the ra/dec/lnlamlike match...
test0 = {}
for cluster in cat0:
test0[cluster.lnlamlike] = cluster.ra
ctr = 0
for i in range(cat.size):
if cat['lnlamlike'][i] in test0:
if cat['ra'][i] == test0[cat['lnlamlike'][i]]:
ctr += 1
self.assertEqual(ctr, cat0.size)
def test_redmapper_run(self):
"""
Run test of redmapper.RedmapperRun.
"""
random.seed(seed=12345)
file_path = 'data_for_tests'
configfile = 'testconfig.yaml'
config = Configuration(os.path.join(file_path, configfile))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
# First, test the splitting
config.calib_run_nproc = 4
redmapper_run = RedmapperRun(config)
splits = redmapper_run._get_pixel_splits()
self.assertEqual(splits[0], 64)
testing.assert_array_equal(splits[1],
np.array([2163, 2296, 2297, 2434]))
# Now, this will just run on 1 but will test consolidation code
config.calib_run_nproc = 2
# Note you need these to be set to get same answer with nproc = 1 because
# of mask rounding
# config.d.hpix = 570
# config.d.nside = 32
config.seedfile = os.path.join(file_path, 'test_dr8_specseeds.fit')
config.zredfile = os.path.join(file_path, 'zreds_test',
'dr8_test_zreds_master_table.fit')
redmapper_run = RedmapperRun(config)
redmapper_run.run(specmode=True,
consolidate_like=True,
keepz=True,
seedfile=config.seedfile)
# Now let's check that we got the final file...
self.assertTrue(
os.path.isfile(
os.path.join(config.outpath,
'%s_final.fit' % (config.d.outbase))))
self.assertTrue(
os.path.isfile(
os.path.join(config.outpath,
'%s_final_members.fit' % (config.d.outbase))))
self.assertTrue(
os.path.isfile(
os.path.join(config.outpath,
'%s_like.fit' % (config.d.outbase))))
cat = Catalog.from_fits_file(
os.path.join(config.outpath, '%s_final.fit' % (config.d.outbase)))
# Spot checks to look for regressions
testing.assert_equal(cat.size, 24)
self.assertGreater(cat.Lambda.min(), 3.0)
testing.assert_array_almost_equal(
cat.Lambda[0:3], np.array([24.396917, 17.944063, 7.738485]))
# And check that the members are all accounted for...
mem = Catalog.from_fits_file(
os.path.join(config.outpath,
'%s_final_members.fit' % (config.d.outbase)))
a, b = esutil.numpy_util.match(cat.mem_match_id, mem.mem_match_id)
testing.assert_equal(a.size, mem.size)
def test_maskgals(self):
"""
Test generation of maskgals file.
"""
# Note that due to historical reasons, this is testing the
# new generation of maskgals with some spot checks. Independently,
# it has been checked that the distributions are the same as for
# the old IDL code which was used to generate the reference used
# in the cluster tests.
file_path = "data_for_tests"
conf_filename = "testconfig.yaml"
config = Configuration(os.path.join(file_path, conf_filename))
# For testing, and backwards compatibility, only make one
config.maskgal_nsamples = 1
config.mask_mode = 0
mask = get_mask(config)
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
maskgalfile = os.path.join(self.test_dir, 'testmaskgal.fit')
random.seed(seed=12345)
# This will generate the file if it isn't there
mask.read_maskgals(maskgalfile)
maskgals, hdr = fitsio.read(maskgalfile, ext=1, header=True)
self.assertEqual(maskgals.size,
config.maskgal_ngals * config.maskgal_nsamples)
self.assertEqual(hdr['VERSION'], 6)
self.assertEqual(hdr['R0'], config.percolation_r0)
self.assertEqual(hdr['BETA'], config.percolation_beta)
self.assertEqual(hdr['STEPSIZE'], config.maskgal_rad_stepsize)
self.assertEqual(hdr['NMAG'], config.nmag)
self.assertEqual(hdr['NGALS'], config.maskgal_ngals)
self.assertEqual(hdr['CHISQMAX'], config.chisq_max)
self.assertEqual(hdr['LVALREF'], config.lval_reference)
self.assertEqual(hdr['EXTRA'], config.maskgal_dmag_extra)
self.assertEqual(hdr['ALPHA'], config.calib_lumfunc_alpha)
self.assertEqual(hdr['RSIG'], config.rsig)
self.assertEqual(hdr['ZREDERR'], config.maskgal_zred_err)
testing.assert_almost_equal(maskgals['r'][0:3],
[0.66900003, 0.119, 0.722])
testing.assert_almost_equal(maskgals['phi'][0:3],
[1.73098969, 2.53610063, 4.2362957])
testing.assert_almost_equal(maskgals['x'][0:3],
[-0.1067116, -0.09784444, -0.33090013])
testing.assert_almost_equal(maskgals['m'][0:3],
[0.46200001, 1.778, -1.43599999])
testing.assert_almost_equal(maskgals['chisq'][0:3],
[8.63599968, 2.28399992, 1.55799997])
testing.assert_almost_equal(maskgals['cwt'][0:3],
[0.02877194, 0.1822518, 0.17872778])
testing.assert_almost_equal(maskgals['nfw'][0:3],
[0.15366785, 0.32543495, 0.1454625])
testing.assert_almost_equal(maskgals['dzred'][0:3],
[-0.03090504, 0.00847131, -0.01800639])
testing.assert_almost_equal(maskgals['zwt'][0:3],
[6.0447073, 18.23568916, 13.30043507])
testing.assert_almost_equal(maskgals['lumwt'][0:3],
[0.39371657, 0.62304342, 0.01774093])
testing.assert_almost_equal(maskgals['theta_r'][0:3, 3],
[0.73237121, 1., 0.3299689])
testing.assert_almost_equal(maskgals['radbins'][0, 0:3],
[0.40000001, 0.5, 0.60000002])
testing.assert_almost_equal(maskgals['nin_orig'][0, 0:3],
[2213., 2663., 3066.])
testing.assert_almost_equal(
maskgals['nin'][0,
0:3], [2203.3347168, 2651.54467773, 3062.44628906])
def test_redmagic_calibrate(self):
"""
"""
np.random.seed(12345)
file_path = 'data_for_tests'
conf_filename = 'testconfig_redmagic.yaml'
config = Configuration(os.path.join(file_path, conf_filename))
self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestRedmapper-')
config.outpath = self.test_dir
testgals = GalaxyCatalog.from_fits_file(os.path.join('data_for_tests', 'redmagic_test', 'redmagic_test_input_gals.fit'))
testgals.add_fields([('mag', 'f4', 5), ('mag_err', 'f4', 5)])
redmagic_cal = RedmagicCalibrator(config)
# We have to have do_run=False here because we don't have a real
# galaxy training set with associated zreds!
redmagic_cal.run(gals=testgals, do_run=False)
# Read in the calibrated parameters
self.assertTrue(os.path.isfile(config.redmagicfile))
cal = fitsio.read(config.redmagicfile, ext=1)
# Check that they are what we think they should be
# (these checks are arbitrary, just to make sure nothing has changed)
testing.assert_almost_equal(cal['cmax'][0, :], np.array([1.14588386, 3.89420298, -0.36792211]))
testing.assert_almost_equal(cal['bias'][0, :], np.array([-0.09999912, -0.04537928, 0.01599778]))
testing.assert_almost_equal(cal['eratio'][0, :], np.array([1.4999998, 1.48021495, 0.50000003]))
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 3)
# This is a hack of the volume limit mask to change from the one used for
# calibration to the one used for the run (which uses a different footprint
# because of reasons)
config_regular = Configuration(os.path.join(file_path, 'testconfig.yaml'))
maskfile = config_regular.maskfile
config = Configuration(redmagic_cal.runfile)
cal, hdr = fitsio.read(config.redmagicfile, ext=1, header=True)
config.maskfile = maskfile
os.remove(cal['vmaskfile'][0].decode().rstrip())
mask = VolumeLimitMask(config, cal['etamin'], use_geometry=True)
# Now test the running, using the output file which has valid galaxies/zreds
run_redmagic = RunRedmagicTask(redmagic_cal.runfile)
run_redmagic.run()
# check that we have a redmagic catalog
rmcatfile = config.redmapper_filename('redmagic_%s' % ('highdens'))
self.assertTrue(os.path.isfile(rmcatfile))
# And a random catalog
rmrandfile = config.redmapper_filename('redmagic_%s_randoms' % ('highdens'))
self.assertTrue(os.path.isfile(config.redmapper_filename('redmagic_%s_randoms' % ('highdens'))))
# And check that the plot is there
pngs = glob.glob(os.path.join(self.test_dir, '*.png'))
self.assertEqual(len(pngs), 4)
# And that we have the desired number of redmagic and randoms
red_cat = GalaxyCatalog.from_fits_file(rmcatfile)
rand_cat = GalaxyCatalog.from_fits_file(rmrandfile)
self.assertEqual(rand_cat.size, red_cat.size * 10)
# And confirm that all the randoms are in the footprint
zmax = mask.calc_zmax(rand_cat.ra, rand_cat.dec)
self.assertTrue(np.all(rand_cat.z < zmax))