def test_generate_healpix_map_single(self): """ Test the generation of a healpix map from a sparse map for a single-value field """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 value = np.random.random(n_rand) # Create a HEALPix map healpix_map = np.zeros(hpg.nside_to_npixel(nside_map), dtype=np.float64) + hpg.UNSEEN idx = hpg.angle_to_pixel(nside_map, ra, dec) healpix_map[idx] = value # Create a HealSparseMap sparse_map = healsparse.HealSparseMap(nside_coverage=nside_coverage, healpix_map=healpix_map) hp_out = sparse_map.generate_healpix_map(nside=nside_map) testing.assert_almost_equal(healpix_map, hp_out) if not has_healpy: return # Now check that it works specifying a different resolution nside_map2 = 32 hp_out = sparse_map.generate_healpix_map(nside=nside_map2) # Let's compare with the original downgraded healpix_map = hp.ud_grade(healpix_map, nside_out=nside_map2, order_in='NESTED', order_out='NESTED') testing.assert_almost_equal(healpix_map, hp_out)
def test_read_parquet_coverage(self): """ Test reading healSparseCoverage from a parquet dataset. """ nside_coverage = 32 nside_map = 64 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') fname = os.path.join(self.test_dir, 'healsparse_map_test.hsparquet') sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype=np.float32) sparse_map[0:20000] = np.random.random(size=20000).astype(np.float32) sparse_map.write(fname, format='parquet') # Generate a coverage mask from the 0: 20000 cov_mask_test = np.zeros(hpg.nside_to_npixel(nside_coverage), dtype=np.bool_) ra, dec = hpg.pixel_to_angle(nside_map, np.arange(20000)) ipnest = np.unique(hpg.angle_to_pixel(nside_coverage, ra, dec)) cov_mask_test[ipnest] = True cov_map = healsparse.HealSparseCoverage.read(fname) # Ensure that the coverage mask is what we think it should be testing.assert_array_equal(cov_map.coverage_mask, cov_mask_test) # Ensure that we can address the cov_map by index testing.assert_array_equal(cov_map[:], cov_map._cov_index_map) testing.assert_array_equal(cov_map[0:100], cov_map._cov_index_map[0:100]) testing.assert_array_equal([cov_map[0]], [cov_map._cov_index_map[0]])
def test_parquet_read_outoforder(self): """ Test reading parquet maps that have been written with out-of-order pixels """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') # Create an empty map sparse_map = healsparse.HealSparseMap.make_empty( nside_coverage, nside_map, np.float64) fname = os.path.join(self.test_dir, 'healsparse_map_outoforder.hsparquet') # Fill it out of order pixel = np.arange(4000, 20000) values = np.random.random(pixel.size) sparse_map.update_values_pix(pixel, values) pixel2 = np.arange(1000) values2 = np.random.random(pixel2.size) sparse_map.update_values_pix(pixel2, values2) sparse_map.write(fname, format='parquet') # And read it in... sparse_map = healsparse.HealSparseMap.read(fname) # Test some values ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_map = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN test_map[pixel] = values test_map[pixel2] = values2 testing.assert_almost_equal(sparse_map.get_values_pix(ipnest), test_map[ipnest]) # Read in the first two and the Nth pixel # These pixels are chosen because they are covered by the random test points sparse_map_small = healsparse.HealSparseMap.read(fname, pixels=[0, 1, 3179]) # Test some values ipnest_cov = np.right_shift(ipnest, sparse_map_small._cov_map.bit_shift) test_values_small = test_map[ipnest] outside_small, = np.where((ipnest_cov != 0) & (ipnest_cov != 1) & (ipnest_cov != 3179)) test_values_small[outside_small] = hpg.UNSEEN testing.assert_almost_equal(sparse_map_small.get_values_pix(ipnest), test_values_small)
def test_degrade_map_recarray(self): """ Test HealSparse.degrade functionality with recarray quantities """ random.seed(seed=12345) nside_coverage = 32 nside_map = 1024 nside_new = 256 dtype = [('col1', 'f8'), ('col2', 'f8'), ('col3', 'i4')] sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1') pixel = np.arange(20000) values = np.zeros_like(pixel, dtype=dtype) values['col1'] = random.random(size=pixel.size) values['col2'] = random.random(size=pixel.size) values['col3'] = random.poisson(size=pixel.size, lam=2) sparse_map.update_values_pix(pixel, values) ra, dec = hpg.pixel_to_angle(nside_map, pixel) # Make the test values hpmap_col1 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap_col2 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap_col3 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap_col1[pixel] = values['col1'] hpmap_col2[pixel] = values['col2'] hpmap_col3[pixel] = values['col3'] # Degrade healpix maps hpmap_col1 = hp.ud_grade(hpmap_col1, nside_out=nside_new, order_in='NESTED', order_out='NESTED') hpmap_col2 = hp.ud_grade(hpmap_col2, nside_out=nside_new, order_in='NESTED', order_out='NESTED') hpmap_col3 = hp.ud_grade(hpmap_col3, nside_out=nside_new, order_in='NESTED', order_out='NESTED') ipnest_test = hpg.angle_to_pixel(nside_new, ra, dec) # Degrade the old map new_map = sparse_map.degrade(nside_out=nside_new) testing.assert_almost_equal(new_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpmap_col1[ipnest_test]) testing.assert_almost_equal(new_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpmap_col2[ipnest_test]) testing.assert_almost_equal(new_map.get_values_pos(ra, dec, lonlat=True)['col3'], hpmap_col3[ipnest_test]) # Test degrade-on-read self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') fname = os.path.join(self.test_dir, 'test_recarray_degrade.hs') sparse_map.write(fname) new_map2 = healsparse.HealSparseMap.read(fname, degrade_nside=nside_new) testing.assert_almost_equal(new_map2.get_values_pos(ra, dec, lonlat=True)['col1'], hpmap_col1[ipnest_test]) testing.assert_almost_equal(new_map2.get_values_pos(ra, dec, lonlat=True)['col2'], hpmap_col2[ipnest_test]) testing.assert_almost_equal(new_map2.get_values_pos(ra, dec, lonlat=True)['col3'], hpmap_col3[ipnest_test])
def test_upgrade_map_recarray(self): """ Test upgrade functionality with a recarray. """ random.seed(seed=12345) nside_coverage = 32 nside_map = 256 nside_new = 1024 dtype = [('col1', 'f8'), ('col2', 'f8'), ('col3', 'i4')] sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1') pixel = np.arange(20000) values = np.zeros_like(pixel, dtype=dtype) values['col1'] = random.random(size=pixel.size) values['col2'] = random.random(size=pixel.size) values['col3'] = random.poisson(size=pixel.size, lam=2) sparse_map.update_values_pix(pixel, values) ra, dec = hpg.pixel_to_angle(nside_map, pixel) # Make the test values hpmap_col1 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap_col2 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap_col3 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap_col1[pixel] = values['col1'] hpmap_col2[pixel] = values['col2'] hpmap_col3[pixel] = values['col3'] # Upgrade healpix maps hpmap_col1 = hp.ud_grade(hpmap_col1, nside_out=nside_new, order_in='NESTED', order_out='NESTED') hpmap_col2 = hp.ud_grade(hpmap_col2, nside_out=nside_new, order_in='NESTED', order_out='NESTED') hpmap_col3 = hp.ud_grade(hpmap_col3, nside_out=nside_new, order_in='NESTED', order_out='NESTED') ipnest_test = hpg.angle_to_pixel(nside_new, ra, dec) # Upgrade the old map new_map = sparse_map.upgrade(nside_out=nside_new) testing.assert_almost_equal( new_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpmap_col1[ipnest_test]) testing.assert_almost_equal( new_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpmap_col2[ipnest_test]) testing.assert_almost_equal( new_map.get_values_pos(ra, dec, lonlat=True)['col3'], hpmap_col3[ipnest_test])
def test_generate_healpix_map_recarray(self): """ Testing the generation of a healpix map from recarray healsparsemap we also test the pixel and position lookup """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 value = np.random.random(n_rand) # Make sure our pixels are unique ipnest = hpg.angle_to_pixel(nside_map, ra, dec) _, uind = np.unique(ipnest, return_index=True) ra = ra[uind] dec = dec[uind] value = value[uind] # Create empty healpix map healpix_map = np.zeros(hpg.nside_to_npixel(nside_map), dtype='f4') + hpg.UNSEEN healpix_map2 = np.zeros(hpg.nside_to_npixel(nside_map), dtype='f8') + hpg.UNSEEN healpix_map[hpg.angle_to_pixel(nside_map, ra, dec)] = value healpix_map2[hpg.angle_to_pixel(nside_map, ra, dec)] = value # Create an empty map dtype = [('col1', 'f4'), ('col2', 'f8')] self.assertRaises(RuntimeError, healsparse.HealSparseMap.make_empty, nside_coverage, nside_map, dtype) # Generate empty map that will be updated sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1') pixel = hpg.angle_to_pixel(nside_map, ra, dec) values = np.zeros_like(pixel, dtype=dtype) values['col1'] = value values['col2'] = value # Update values works with the HEALPix-like indexing scheme sparse_map.update_values_pix(pixel, values) hp_out1 = sparse_map.generate_healpix_map(nside=nside_map, key='col1') hp_out2 = sparse_map.generate_healpix_map(nside=nside_map, key='col2') testing.assert_almost_equal(healpix_map, hp_out1) testing.assert_almost_equal(healpix_map2, hp_out2)
def test_healpix_explicit_int_write(self): """Test writing healpix partial (explicit) maps (integer).""" random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') # Generate a map full_map = np.zeros(hpg.nside_to_npixel(nside_map), dtype=np.int32) full_map[0:10000] = 4 full_map[20000:30000] = 5 ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_values = full_map[ipnest] filename = os.path.join(self.test_dir, 'healsparse_healpix_int_partial_map.fits') sparse_map = healsparse.HealSparseMap(healpix_map=full_map, nside_coverage=nside_coverage, nest=True, sentinel=0) with self.assertWarns(UserWarning): sparse_map.write(filename, format='healpix') # Read in with healsparse and make sure it is the same. sparse_map2 = healsparse.HealSparseMap.read( filename, nside_coverage=nside_coverage) np.testing.assert_array_equal(sparse_map2.valid_pixels, sparse_map.valid_pixels) testing.assert_almost_equal(sparse_map2.get_values_pix(ipnest), test_values) # Read in with healpy and make sure it is the same. full_map2 = hp.read_map(filename, nest=True) testing.assert_array_equal((full_map2 > hpg.UNSEEN).nonzero()[0], sparse_map.valid_pixels) # healpy will convert all the BAD_DATA to UNSEEN good, = (test_values > 0).nonzero() bad, = (test_values == 0).nonzero() testing.assert_array_equal(full_map2[ipnest[good]], test_values[good]) testing.assert_array_almost_equal(full_map2[ipnest[bad]], hpg.UNSEEN)
def test_healpix_explicit_read(self): """Test reading healpix partial (explicit) maps.""" random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') # Generate a random map full_map = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN full_map[0:20000] = np.random.random(size=20000) ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_values = full_map[ipnest] filename = os.path.join(self.test_dir, 'healpix_map_ring_explicit.fits') full_map_ring = hp.reorder(full_map, n2r=True) hp.write_map(filename, full_map_ring, dtype=np.float64, partial=True) # Read it with healsparse sparse_map = healsparse.HealSparseMap.read( filename, nside_coverage=nside_coverage) # Check that we can do a basic lookup testing.assert_almost_equal(sparse_map.get_values_pix(ipnest), test_values) filename = os.path.join(self.test_dir, 'healpix_map_nest_explicit.fits') hp.write_map(filename, full_map, dtype=np.float64, nest=True, partial=True) # Read it with healsparse sparse_map = healsparse.HealSparseMap.read( filename, nside_coverage=nside_coverage) # Check that we can do a basic lookup testing.assert_almost_equal(sparse_map.get_values_pix(ipnest), test_values)
def test_read_fits_coverage(self): """ Test reading healSparseCoverage from a fits file. """ nside_coverage = 32 nside_map = 64 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') fname = os.path.join(self.test_dir, 'healsparse_map_test.hsp') sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype=np.float32) sparse_map[0:20000] = np.random.random(size=20000).astype(np.float32) sparse_map.write(fname) # Generate a coverage mask from the 0: 20000 cov_mask_test = np.zeros(hpg.nside_to_npixel(nside_coverage), dtype=np.bool_) ra, dec = hpg.pixel_to_angle(nside_map, np.arange(20000)) ipnest = np.unique(hpg.angle_to_pixel(nside_coverage, ra, dec)) cov_mask_test[ipnest] = True cov_map = healsparse.HealSparseCoverage.read(fname) # Ensure that the coverage mask is what we think it should be testing.assert_array_equal(cov_map.coverage_mask, cov_mask_test) # Ensure that we can address the cov_map by index testing.assert_array_equal(cov_map[:], cov_map._cov_index_map) testing.assert_array_equal(cov_map[0:100], cov_map._cov_index_map[0:100]) testing.assert_array_equal([cov_map[0]], [cov_map._cov_index_map[0]]) if not has_healpy: return # Make a healpy file and make sure we can't read it test_map = np.zeros(hpg.nside_to_npixel(nside_coverage)) fname = os.path.join(self.test_dir, 'healpy_map_test.fits') hp.write_map(fname, test_map) self.assertRaises(RuntimeError, healsparse.HealSparseCoverage.read, fname)
def test_healpix_explicit_write(self): """Test writing healpix partial (explicit) maps (floating point).""" random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') # Generate a random map full_map = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN full_map[0:20000] = np.random.random(size=20000) ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_values = full_map[ipnest] filename = os.path.join(self.test_dir, 'healsparse_healpix_partial_map.fits') sparse_map = healsparse.HealSparseMap(healpix_map=full_map, nside_coverage=nside_coverage, nest=True) sparse_map.write(filename, format='healpix') # Read in with healsparse and make sure it is the same. sparse_map2 = healsparse.HealSparseMap.read( filename, nside_coverage=nside_coverage) np.testing.assert_array_equal(sparse_map2.valid_pixels, sparse_map.valid_pixels) testing.assert_array_almost_equal(sparse_map2.get_values_pix(ipnest), test_values) # Read in with healpy and make sure it is the same. full_map2 = hp.read_map(filename, nest=True) testing.assert_array_equal((full_map2 > hpg.UNSEEN).nonzero()[0], sparse_map.valid_pixels) testing.assert_array_almost_equal(full_map2[ipnest], test_values)
def test_generate_healpix_map_ring(self): """ Test the generation of a healpixmap in ring type """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 value = np.random.random(n_rand) # Create a HEALPix map healpix_map = np.zeros(hpg.nside_to_npixel(nside_map), dtype=np.float64) + hpg.UNSEEN idx = hpg.angle_to_pixel(nside_map, ra, dec) healpix_map[idx] = value # Create a HealSparseMap sparse_map = healsparse.HealSparseMap(nside_coverage=nside_coverage, healpix_map=healpix_map) hp_out_ring = sparse_map.generate_healpix_map(nside=nside_map, nest=False) healpix_map_ring = hp.reorder(healpix_map, n2r=True) testing.assert_almost_equal(healpix_map_ring, hp_out_ring)
def test_lookup(self): """ Test lookup functionality """ np.random.seed(12345) nside_coverage = 32 nside_map = 1024 full_map = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN full_map[0:200000] = np.random.random(size=200000) sparse_map = healsparse.HealSparseMap(healpix_map=full_map, nside_coverage=nside_coverage) n_rand = 100000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 theta, phi = hpg.lonlat_to_thetaphi(ra, dec) ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_values = full_map[ipnest] # Test the pixel lookup comp_values = sparse_map.get_values_pix(ipnest) testing.assert_array_almost_equal(comp_values, test_values) # Test pixel lookup (valid pixels) # Note that this tests all the downstream functions valid_mask = sparse_map.get_values_pix(ipnest, valid_mask=True) testing.assert_array_equal(valid_mask, comp_values > hpg.UNSEEN) # Test pixel lookup (ring) ipring = hpg.nest_to_ring(nside_map, ipnest) comp_values = sparse_map.get_values_pix(ipring, nest=False) testing.assert_array_almost_equal(comp_values, test_values) # Test pixel lookup (higher nside) comp_values = sparse_map.get_values_pix(hpg.angle_to_pixel( 4096, ra, dec), nside=4096) testing.assert_array_almost_equal(comp_values, test_values) # Test pixel lookup (lower nside) lowres_pix = hpg.angle_to_pixel(256, ra, dec) self.assertRaises(ValueError, sparse_map.get_values_pix, lowres_pix, nside=256) # Test the theta/phi lookup comp_values = sparse_map.get_values_pos(theta, phi, lonlat=False) testing.assert_array_almost_equal(comp_values, test_values) # Test the ra/dec lookup comp_values = sparse_map.get_values_pos(ra, dec, lonlat=True) testing.assert_array_almost_equal(comp_values, test_values) # Test the list of valid pixels valid_pixels = sparse_map.valid_pixels testing.assert_array_equal(valid_pixels, np.where(full_map > hpg.UNSEEN)[0]) # Test the position of valid pixels ra_sp, dec_sp = sparse_map.valid_pixels_pos(lonlat=True) _ra_sp, _dec_sp = hpg.pixel_to_angle( nside_map, np.where(full_map > hpg.UNSEEN)[0]) testing.assert_array_almost_equal(ra_sp, _ra_sp) testing.assert_array_almost_equal(dec_sp, _dec_sp) # Test position of valid pixels and valid pixels valid_pixels, ra_sp, dec_sp = sparse_map.valid_pixels_pos( lonlat=True, return_pixels=True) _ra_sp, _dec_sp = hpg.pixel_to_angle( nside_map, np.where(full_map > hpg.UNSEEN)[0]) testing.assert_array_almost_equal(ra_sp, _ra_sp) testing.assert_array_almost_equal(dec_sp, _dec_sp) testing.assert_array_equal(valid_pixels, np.where(full_map > hpg.UNSEEN)[0])
def test_build_maps_single(self): """ Test building a map for a single-value field """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 # Create an empty map sparse_map = healsparse.HealSparseMap.make_empty( nside_coverage, nside_map, np.float64) # Look up all the values, make sure they're all UNSEEN testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True), hpg.UNSEEN) # Fail to append because of wrong dtype pixel = np.arange(4000, 20000) values = np.ones_like(pixel, dtype=np.float32) self.assertRaises(ValueError, sparse_map.update_values_pix, pixel, values) # Append a bunch of pixels values = np.ones_like(pixel, dtype=np.float64) sparse_map.update_values_pix(pixel, values) # Make a healpix map for comparison hpmap = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmap[pixel] = values ipnest_test = hpg.angle_to_pixel(nside_map, ra, dec, nest=True) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True), hpmap[ipnest_test]) # Replace the pixels values += 1 sparse_map.update_values_pix(pixel, values) hpmap[pixel] = values testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True), hpmap[ipnest_test]) # Replace and append more pixels # Note that these are lower-number pixels, so the map is out of order pixel2 = np.arange(3000) + 2000 values2 = np.ones_like(pixel2, dtype=np.float64) sparse_map.update_values_pix(pixel2, values2) hpmap[pixel2] = values2 testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True), hpmap[ipnest_test]) # Test making empty maps sparse_map2 = healsparse.HealSparseMap.make_empty_like(sparse_map) self.assertEqual(sparse_map2.nside_coverage, sparse_map.nside_coverage) self.assertEqual(sparse_map2.nside_sparse, sparse_map.nside_sparse) self.assertEqual(sparse_map2.dtype, sparse_map.dtype) self.assertEqual(sparse_map2._sentinel, sparse_map._sentinel) sparse_map2b = healsparse.HealSparseMap.make_empty_like( sparse_map, cov_pixels=[0, 2]) self.assertEqual(sparse_map2b.nside_coverage, sparse_map.nside_coverage) self.assertEqual(sparse_map2b.nside_sparse, sparse_map.nside_sparse) self.assertEqual(sparse_map2b.dtype, sparse_map.dtype) self.assertEqual(sparse_map2b._sentinel, sparse_map._sentinel) self.assertEqual(len(sparse_map2b._sparse_map), sparse_map2._cov_map.nfine_per_cov * 3) testing.assert_array_equal(sparse_map2b._sparse_map, sparse_map._sentinel) sparse_map2 = healsparse.HealSparseMap.make_empty_like( sparse_map, nside_coverage=16) self.assertEqual(sparse_map2.nside_coverage, 16) self.assertEqual(sparse_map2.nside_sparse, sparse_map.nside_sparse) self.assertEqual(sparse_map2.dtype, sparse_map.dtype) self.assertEqual(sparse_map2._sentinel, sparse_map._sentinel) sparse_map2 = healsparse.HealSparseMap.make_empty_like( sparse_map, nside_sparse=128) self.assertEqual(sparse_map2.nside_coverage, sparse_map.nside_coverage) self.assertEqual(sparse_map2.nside_sparse, 128) self.assertEqual(sparse_map2.dtype, sparse_map.dtype) self.assertEqual(sparse_map2._sentinel, sparse_map._sentinel) sparse_map2 = healsparse.HealSparseMap.make_empty_like(sparse_map, dtype=np.int32, sentinel=0) self.assertEqual(sparse_map2.nside_coverage, sparse_map.nside_coverage) self.assertEqual(sparse_map2.nside_sparse, sparse_map.nside_sparse) self.assertEqual(sparse_map2.dtype, np.int32)
def test_fits_read_outoforder_recarray(self): """ Test fits reading of recarray maps that have been written out-of-order """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 nRand = 1000 ra = np.random.random(nRand) * 360.0 dec = np.random.random(nRand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') fname = os.path.join(self.test_dir, 'healsparse_map_recarray_outoforder.hsp') # Create an empty map dtype = [('col1', 'f8'), ('col2', 'f8')] sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1') pixel = np.arange(4000, 20000) values = np.zeros_like(pixel, dtype=dtype) values['col1'] = np.random.random(size=pixel.size) values['col2'] = np.random.random(size=pixel.size) sparse_map.update_values_pix(pixel, values) pixel2 = np.arange(1000) values2 = np.zeros_like(pixel2, dtype=dtype) values2['col1'] = np.random.random(size=pixel2.size) values2['col2'] = np.random.random(size=pixel2.size) sparse_map.update_values_pix(pixel2, values2) sparse_map.write(fname) # And read it in... sparse_map = healsparse.HealSparseMap.read(fname) # Test some values ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_map_col1 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN test_map_col1[pixel] = values['col1'] test_map_col1[pixel2] = values2['col1'] test_map_col2 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN test_map_col2[pixel] = values['col2'] test_map_col2[pixel2] = values2['col2'] testing.assert_almost_equal( sparse_map.get_values_pix(ipnest)['col1'], test_map_col1[ipnest]) testing.assert_almost_equal( sparse_map.get_values_pix(ipnest)['col2'], test_map_col2[ipnest]) # These pixels are chosen because they are covered by the random test points sparse_map_small = healsparse.HealSparseMap.read(fname, pixels=[0, 1, 3179]) ipnest_cov = np.right_shift(ipnest, sparse_map_small._cov_map.bit_shift) test_values_small_col1 = test_map_col1[ipnest] test_values_small_col2 = test_map_col2[ipnest] outside_small, = np.where((ipnest_cov != 0) & (ipnest_cov != 1) & (ipnest_cov != 3179)) test_values_small_col1[outside_small] = hpg.UNSEEN test_values_small_col2[outside_small] = hpg.UNSEEN testing.assert_almost_equal( sparse_map_small.get_values_pix(ipnest)['col1'], test_values_small_col1) testing.assert_almost_equal( sparse_map_small.get_values_pix(ipnest)['col2'], test_values_small_col2)
def test_fits_writeread(self): """ Test healsparse fits i/o functionality """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') # Generate a random map full_map = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN full_map[0: 20000] = np.random.random(size=20000) ipnest = hpg.angle_to_pixel(nside_map, ra, dec) test_values = full_map[ipnest] # Test that we can do a basic set sparse_map = healsparse.HealSparseMap(healpix_map=full_map, nside_coverage=nside_coverage, nest=True) sparse_map[30000: 30005] = np.zeros(5, dtype=np.float64) # Write it to healsparse format sparse_map.write(os.path.join(self.test_dir, 'healsparse_map.hs')) # Read in healsparse format (full map) sparse_map = healsparse.HealSparseMap.read(os.path.join(self.test_dir, 'healsparse_map.hs')) # Check that we can do a basic lookup testing.assert_almost_equal(sparse_map.get_values_pix(ipnest), test_values) # Check that we can do a basic set sparse_map[30000: 30005] = np.zeros(5, dtype=np.float64) # Try to read in healsparse format, non-unique pixels self.assertRaises(RuntimeError, healsparse.HealSparseMap.read, os.path.join(self.test_dir, 'healsparse_map.hs'), pixels=[0, 0]) # Read in healsparse format (two pixels) sparse_map_small = healsparse.HealSparseMap.read(os.path.join(self.test_dir, 'healsparse_map.hs'), pixels=[0, 1]) # Test the coverage map only has two pixels cov_mask = sparse_map_small.coverage_mask self.assertEqual(cov_mask.sum(), 2) # Test lookup of values in those two pixels ipnestCov = np.right_shift(ipnest, sparse_map_small._cov_map.bit_shift) outside_small, = np.where(ipnestCov > 1) test_values2 = test_values.copy() test_values2[outside_small] = hpg.UNSEEN testing.assert_almost_equal(sparse_map_small.get_values_pix(ipnest), test_values2) # Read in healsparse format (all pixels) sparse_map_full = healsparse.HealSparseMap.read( os.path.join(self.test_dir, 'healsparse_map.hs'), pixels=np.arange(hpg.nside_to_npixel(nside_coverage)) ) testing.assert_almost_equal(sparse_map_full.get_values_pix(ipnest), test_values)
def test_build_maps_recarray(self): """ Testing building a map for a recarray """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 # Create an empty map dtype = [('col1', 'f4'), ('col2', 'f8')] self.assertRaises(RuntimeError, healsparse.HealSparseMap.make_empty, nside_coverage, nside_map, dtype) sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1') # Look up all the values, make sure they're all UNSEEN testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpg.UNSEEN) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpg.UNSEEN) pixel = np.arange(4000, 20000) values = np.zeros_like(pixel, dtype=dtype) values['col1'] = 1.0 values['col2'] = 2.0 sparse_map.update_values_pix(pixel, values) # Make healpix maps for comparison hpmapCol1 = np.zeros(hpg.nside_to_npixel(nside_map), dtype=np.float32) + hpg.UNSEEN hpmapCol2 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmapCol1[pixel] = values['col1'] hpmapCol2[pixel] = values['col2'] ipnest_test = hpg.angle_to_pixel(nside_map, ra, dec, nest=True) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpmapCol1[ipnest_test]) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpmapCol2[ipnest_test]) # Replace the pixels values['col1'] += 1 values['col2'] += 1 sparse_map.update_values_pix(pixel, values) hpmapCol1[pixel] = values['col1'] hpmapCol2[pixel] = values['col2'] testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpmapCol1[ipnest_test]) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpmapCol2[ipnest_test]) # Replace and append more pixels # Note that these are lower-number pixels, so the map is out of order pixel2 = np.arange(3000) + 2000 values2 = np.zeros_like(pixel2, dtype=dtype) values2['col1'] = 1.0 values2['col2'] = 2.0 sparse_map.update_values_pix(pixel2, values2) hpmapCol1[pixel2] = values2['col1'] hpmapCol2[pixel2] = values2['col2'] testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpmapCol1[ipnest_test]) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpmapCol2[ipnest_test]) # Test making empty maps sparse_map2 = healsparse.HealSparseMap.make_empty_like(sparse_map) self.assertEqual(sparse_map2.nside_coverage, sparse_map.nside_coverage) self.assertEqual(sparse_map2.nside_sparse, sparse_map.nside_sparse) self.assertEqual(sparse_map2.dtype, sparse_map.dtype) self.assertEqual(sparse_map2._sentinel, sparse_map._sentinel) sparse_map2b = healsparse.HealSparseMap.make_empty_like( sparse_map, cov_pixels=[0, 2]) self.assertEqual(sparse_map2b.nside_coverage, sparse_map.nside_coverage) self.assertEqual(sparse_map2b.nside_sparse, sparse_map.nside_sparse) self.assertEqual(sparse_map2b.dtype, sparse_map.dtype) self.assertEqual(sparse_map2b._sentinel, sparse_map._sentinel) self.assertEqual(len(sparse_map2b._sparse_map), sparse_map2._cov_map.nfine_per_cov * 3) testing.assert_array_equal(sparse_map2b._sparse_map['col1'], sparse_map._sentinel) testing.assert_array_equal(sparse_map2b._sparse_map['col2'], hpg.UNSEEN)
def test_make_wide_mask_map(self): """ Test making a wide mask map. """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 # Test expected errors on creating maps # Create empty maps to test bit width sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=7) self.assertTrue(sparse_map.is_wide_mask_map) self.assertEqual(sparse_map.wide_mask_maxbits, 8) self.assertEqual(sparse_map._sparse_map.shape, (4, 1)) self.assertEqual(sparse_map._sentinel, 0) # Set bits and retrieve them pixel = np.arange(4000, 20000) sparse_map.set_bits_pix(pixel, [4]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [4]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [6]), False) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [4, 6]), True) testing.assert_array_equal(sparse_map.valid_pixels, pixel) testing.assert_equal(sparse_map.n_valid, len(pixel)) pospix = hpg.angle_to_pixel(nside_map, ra, dec) inds = np.searchsorted(pixel, pospix) b, = np.where((inds > 0) & (inds < pixel.size)) comp_arr = np.zeros(pospix.size, dtype=np.bool_) comp_arr[b] = True testing.assert_array_equal( sparse_map.check_bits_pos(ra, dec, [4], lonlat=True), comp_arr) sparse_map.clear_bits_pix(pixel, [4]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [4]), False) sparse_map.set_bits_pix(pixel, [4, 6]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [4]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [6]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [7]), False) testing.assert_array_equal(sparse_map.valid_pixels, pixel) testing.assert_equal(sparse_map.n_valid, len(pixel)) # This just makes sure that the size is correct sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=8) self.assertEqual(sparse_map.wide_mask_maxbits, 8) # And now a double-wide to test # Note that 9 will create a 16 bit mask sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=9) self.assertEqual(sparse_map.wide_mask_maxbits, 16) sparse_map.set_bits_pix(pixel, [12]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [12]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [4]), False) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [15]), False) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [4, 12]), True) sparse_map.set_bits_pix(pixel, [2, 3, 5, 15]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [2]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [3]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [5]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [15]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [14]), False) # Clear a bit in the lower field, ensure upper field is untouched. sparse_map.clear_bits_pix(pixel, [5]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [5]), False) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [15]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [14]), False) testing.assert_array_equal(sparse_map.valid_pixels, pixel) testing.assert_equal(sparse_map.n_valid, len(pixel)) # Clear multiple bits in the lower field, ensure upper field is untouched. sparse_map.clear_bits_pix(pixel, [2, 3]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [2]), False) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [3]), False) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [15]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [14]), False) testing.assert_array_equal(sparse_map.valid_pixels, pixel) testing.assert_equal(sparse_map.n_valid, len(pixel)) # This makes sure the inferred size is correct sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=128) self.assertEqual(sparse_map.wide_mask_maxbits, 128) # And do a triple-wide sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=20) self.assertEqual(sparse_map.wide_mask_maxbits, 24) sparse_map.set_bits_pix(pixel, [5, 10, 20]) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [5]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [10]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [20]), True) testing.assert_array_equal(sparse_map.check_bits_pix(pixel, [21]), False) testing.assert_array_equal(sparse_map.valid_pixels, pixel) testing.assert_equal(sparse_map.n_valid, len(pixel))
def test_wide_mask_map_parquet_io(self): """ Test parquet i/o with wide mask maps. """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 n_rand = 1000 ra = np.random.random(n_rand) * 360.0 dec = np.random.random(n_rand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') # Test with single-wide sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=8) pixel = np.arange(4000, 20000) sparse_map.set_bits_pix(pixel, [5]) fname = os.path.join(self.test_dir, 'healsparse_map.hsparquet') sparse_map.write(fname, format='parquet') sparse_map_in = healsparse.HealSparseMap.read(fname) self.assertTrue(sparse_map_in.is_wide_mask_map) self.assertEqual(sparse_map_in.wide_mask_maxbits, 8) self.assertEqual(sparse_map_in._sparse_map.shape[1], 1) self.assertEqual(sparse_map_in._wide_mask_width, 1) self.assertEqual(sparse_map_in._sentinel, 0) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [5]), True) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [7]), False) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [5, 7]), True) pospix = hpg.angle_to_pixel(nside_map, ra, dec) inds = np.searchsorted(pixel, pospix) b, = np.where((inds > 0) & (inds < pixel.size)) comp_arr = np.zeros(pospix.size, dtype=np.bool_) comp_arr[b] = True testing.assert_array_equal( sparse_map_in.check_bits_pos(ra, dec, [5], lonlat=True), comp_arr) # And read a partial map sparse_map_in_partial = healsparse.HealSparseMap.read( fname, pixels=[1000, 1002]) self.assertTrue(sparse_map_in_partial.is_wide_mask_map) self.assertEqual(sparse_map_in_partial.wide_mask_maxbits, 8) self.assertEqual(sparse_map_in_partial._sparse_map.shape[1], 1) self.assertEqual(sparse_map_in_partial._wide_mask_width, 1) self.assertEqual(sparse_map_in_partial._sentinel, 0) cov_pixels = sparse_map._cov_map.cov_pixels(pixel) pixel_sub = pixel[(cov_pixels == 1000) | (cov_pixels == 1002)] testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [5]), True) testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [7]), False) testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [5, 7]), True) # Test with double-wide sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, WIDE_MASK, wide_mask_maxbits=16) pixel = np.arange(4000, 20000) sparse_map.set_bits_pix(pixel, [5, 10]) fname = os.path.join(self.test_dir, 'healsparse_map2.hsparquet') sparse_map.write(fname, format='parquet') sparse_map_in = healsparse.HealSparseMap.read(fname) self.assertTrue(sparse_map_in.is_wide_mask_map) self.assertEqual(sparse_map_in.wide_mask_maxbits, 16) self.assertEqual(sparse_map_in._sparse_map.shape[1], 2) self.assertEqual(sparse_map_in._wide_mask_width, 2) self.assertEqual(sparse_map_in._sentinel, 0) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [5]), True) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [10]), True) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [4]), False) testing.assert_array_equal(sparse_map_in.check_bits_pix(pixel, [12]), False) # And read a partial double-wide map sparse_map_in_partial = healsparse.HealSparseMap.read( fname, pixels=[1000, 1002]) self.assertTrue(sparse_map_in_partial.is_wide_mask_map) self.assertEqual(sparse_map_in_partial.wide_mask_maxbits, 16) self.assertEqual(sparse_map_in_partial._sparse_map.shape[1], 2) self.assertEqual(sparse_map_in_partial._wide_mask_width, 2) self.assertEqual(sparse_map_in_partial._sentinel, 0) cov_pixels = sparse_map._cov_map.cov_pixels(pixel) pixel_sub = pixel[(cov_pixels == 1000) | (cov_pixels == 1002)] testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [5]), True) testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [10]), True) testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [4]), False) testing.assert_array_equal( sparse_map_in_partial.check_bits_pix(pixel_sub, [12]), False)
def test_parquet_writeread_recarray(self): """ Test parquet recarray writing and reading. """ random.seed(seed=12345) nside_coverage = 32 nside_map = 64 nRand = 1000 ra = np.random.random(nRand) * 360.0 dec = np.random.random(nRand) * 180.0 - 90.0 self.test_dir = tempfile.mkdtemp(dir='./', prefix='TestHealSparse-') fname = os.path.join(self.test_dir, 'healsparse_map_recarray.hsparquet') dtype = [('col1', 'f8'), ('col2', 'f8')] sparse_map = healsparse.HealSparseMap.make_empty(nside_coverage, nside_map, dtype, primary='col1') pixel = np.arange(20000) values = np.zeros_like(pixel, dtype=dtype) values['col1'] = np.random.random(size=pixel.size) values['col2'] = np.random.random(size=pixel.size) sparse_map.update_values_pix(pixel, values) sparse_map.write(fname, format='parquet') # Make the test values hpmapCol1 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmapCol2 = np.zeros(hpg.nside_to_npixel(nside_map)) + hpg.UNSEEN hpmapCol1[pixel] = values['col1'] hpmapCol2[pixel] = values['col2'] ipnest_test = hpg.angle_to_pixel(nside_map, ra, dec) # Read in the map sparse_map = healsparse.HealSparseMap.read(fname) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col1'], hpmapCol1[ipnest_test]) testing.assert_almost_equal( sparse_map.get_values_pos(ra, dec, lonlat=True)['col2'], hpmapCol2[ipnest_test]) # Test the list of valid pixels valid_pixels = sparse_map.valid_pixels testing.assert_equal(valid_pixels, pixel) # Read in a partial map... sparse_map_small = healsparse.HealSparseMap.read(fname, pixels=[0, 1]) # Test the coverage map only has two pixels cov_mask = sparse_map_small.coverage_mask self.assertEqual(cov_mask.sum(), 2) # Test lookup of values in these two pixels ipnest_cov = np.right_shift(ipnest_test, sparse_map_small._cov_map.bit_shift) outside_small, = np.where(ipnest_cov > 1) # column1 is the "primary" column and will return UNSEEN test_values1b = hpmapCol1[ipnest_test].copy() test_values1b[outside_small] = hpg.UNSEEN # column2 is not the primary column and will also return UNSEEN test_values2b = hpmapCol2[ipnest_test].copy() test_values2b[outside_small] = hpg.UNSEEN testing.assert_almost_equal( sparse_map_small.get_values_pix(ipnest_test)['col1'], test_values1b) testing.assert_almost_equal( sparse_map_small.get_values_pix(ipnest_test)['col2'], test_values2b)