def test_str(self):
"""Verify yaml-ness of string representation"""
bitmask = BitMask('ccdmask', yaml.load(str(self.ccdmask)))
self.assertEqual(bitmask.names(), self.ccdmask.names())
for name in bitmask.names():
self.assertEqual(bitmask[name].mask, self.ccdmask[name].mask)
self.assertEqual(bitmask[name].comment, self.ccdmask[name].comment)
self.assertEqual(bitmask[name].bitnum, self.ccdmask[name].bitnum)
self.assertEqual(bitmask[name]._extra, self.ccdmask[name]._extra)
def test_str(self):
"""Verify yaml-ness of string representation"""
bitmask = BitMask('ccdmask', yaml.load(str(self.ccdmask)))
self.assertEqual(bitmask.names(), self.ccdmask.names())
for name in bitmask.names():
self.assertEqual(bitmask[name].mask, self.ccdmask[name].mask)
self.assertEqual(bitmask[name].comment, self.ccdmask[name].comment)
self.assertEqual(bitmask[name].bitnum, self.ccdmask[name].bitnum)
self.assertEqual(bitmask[name]._extra, self.ccdmask[name]._extra)
def test_badname(self):
"""Test raising AttributeError for bad names.
"""
with self.assertRaises(AttributeError):
x = self.ccdmask.BLATFOO
# Attribute already in use
with self.assertRaises(AttributeError):
bit = _MaskBit('BAD', 0, "comment", extra={'real': 'foo'})
# has extra that isn't a dict
with self.assertRaises(ValueError):
blat = BitMask('ccdmask', _baddef1)
def create_bpt_mask(survey, faprgrm, specred='everest'):
table = join_zcat_fastspec(survey=survey, faprgrm=faprgrm, specred=specred)
nii_bpt, sf_nii, agn_nii, liner_nii, composite_nii, quiescent_nii = Classify_NII_BPT(
table)
sii_bpt, sf_sii, agn_sii, liner_sii, quiescent_sii = Classify_SII_BPT(
table)
oi_bpt, sf_oi, agn_oi, liner_oi = Classify_OI_BPT(table)
file = open('../Sandbox/agn_masks.yaml', 'r')
bpt_defs = yaml.safe_load(file)
bpt_type = BitMask('bpt_type', bpt_defs)
bpt_mask = np.zeros(len(table))
## If anyone of the emission line fluxes is zero, then there is no bpt_mask (bpt_mask = 0)
## [NII]-BPT masks
bpt_mask = nii_bpt.data * bpt_type.NII_BPT_AV ## All the emission lines have S/N >= 3
bpt_mask |= sf_nii.data * bpt_type.NII_SF ## [NII] - Star Forming
bpt_mask |= agn_nii.data * bpt_type.NII_SY ## [NII] - Seyfert
bpt_mask |= liner_nii.data * bpt_type.NII_LINER ## [NII] - LINER
bpt_mask |= composite_nii.data * bpt_type.NII_COMP ## [NII] - Composite
bpt_mask |= quiescent_nii.data * bpt_type.NII_QUIES ## [NII] - Quiescent -- S/N one of the lines < 3
## [SII]-BPT masks
bpt_mask |= sii_bpt.data * bpt_type.SII_BPT_AV ## All the emission lines have S/N >= 3
bpt_mask |= sf_sii.data * bpt_type.SII_SF ## [SII] - Star Forming
bpt_mask |= agn_sii.data * bpt_type.SII_SY ## [SII] - Seyfert
bpt_mask |= liner_sii.data * bpt_type.SII_LINER ## [SII] - LINER
bpt_mask |= quiescent_sii.data * bpt_type.SII_QUIES ## [SII] - Quiescent -- S/N for one of the lines < 3
## [OI]-BPT masks
bpt_mask |= oi_bpt.data * bpt_type.OI_BPT_AV ## Except [OI] - other emission lines have S/N >= 3
bpt_mask |= sf_oi.data * bpt_type.OI_SF ## [OI] - Star Forming
bpt_mask |= agn_oi.data * bpt_type.OI_SY ## [OI] - Seyfert
bpt_mask |= liner_oi.data * bpt_type.OI_LINER ## [OI] - LINER
bptmask_column = Column(bpt_mask, name='BPT_MASK')
table.add_column(bptmask_column)
return (table)
## Retain only the science targets; Dropping standards etc.
_assigned = _assigned[_assigned['FA_TYPE'].quantity == 1]
print('Assigned science: {}'.format(len(_assigned)))
## Load one tile, currently.
Tile = 74057
sv_mtl = Table(fits.open(scratch + '/BGS/SV-ASSIGN/mtls/svmtl_{:06d}.fits'.format(Tile))[1].data)
## Limit to only BGS.
sv_mtl = sv_mtl[sv_mtl['SV1_BGS_TARGET'].quantity.value > 0]
print('Total targets in Tile {}: {}.'.format(Tile, len(sv_mtl)))
##
_bitdefs = load_mask_bits("sv1")
bgs_mask = BitMask('sv1_bgs_mask', _bitdefs)
types = bgs_mask.names()
bits = [bgs_mask.bitnum(x) for x in types]
## Those that were assigned in this tile.
assigned = join(_assigned, sv_mtl, keys=['BRICKID', 'BRICK_OBJID'], join_type='inner')
## SV Target class.
splits = {}
title = ''
##
for _type in ['BGS_FAINT', 'BGS_BRIGHT', 'BGS_FAINT_EXT', 'BGS_LOWQ', 'BGS_FIBMAG']:
sample = _type.split('_')
sample.pop(0)
from desitarget.targetmask import load_mask_bits
from desiutil.bitmask import BitMask
from desimodel.io import load_desiparams, load_platescale
scratch = os.environ['CSCRATCH']
tiles = Table(
fits.open(
'/global/cscratch1/sd/mjwilson/BGS/SV-ASSIGN/tiles/BGS_SV_30_3x_superset60_Sep2019.fits'
)[1].data)
utiles = np.unique(tiles['TILEID'].quantity)
##
_bitdefs = load_mask_bits("sv1")
desi_mask = BitMask('sv1_desi_mask', _bitdefs)
bgs_mask = BitMask('sv1_bgs_mask', _bitdefs)
types = bgs_mask.names()
bits = [bgs_mask.bitnum(x) for x in types]
## L428 of https://github.com/desihub/desimodel/blob/master/py/desimodel/focalplane/geometry.py
params = load_desiparams()
fiber_dia = params['fibers']['diameter_um']
#- Platescales in um/arcsec
ps = load_platescale()
## Add in GAMA labels.
_fits = fits.open(
'/global/cscratch1/sd/mjwilson/BGS/SV-ASSIGN/truth/legacy/ls-GAMA-south.fits'
"""
desitarget.contammask
=====================
"""
try:
from desiutil.bitmask import BitMask
except ImportError:
#
# This can happen during documentation builds.
#
def BitMask(*args): return None
_bitdefs = None
def _load_bits():
"""Load bit definitions from yaml file.
"""
global _bitdefs
import yaml
from pkg_resources import resource_filename
if _bitdefs is None:
_filepath = resource_filename('desitarget', "data/contammask.yaml")
with open(_filepath) as fx:
_bitdefs = yaml.load(fx)
return
#- convert to BitMask objects
if _bitdefs is None:
_load_bits()
contam_mask = BitMask('contam_mask', _bitdefs)
)[1].data)
_assigned.sort('BRICK_OBJID')
## Retain only the science targets; Dropping standards etc.
_assigned = _assigned[_assigned['FA_TYPE'].quantity == 1]
## Load one tile, currently.
sv_mtl = Table(
fits.open(scratch + '/BGS/SV-ASSIGN/mtls/svmtl_074057.fits')[1].data)
## Limit to only BGS.
sv_mtl = sv_mtl[sv_mtl['SV1_BGS_TARGET'].quantity.value > 0]
##
_bitdefs = load_mask_bits("sv1")
bgs_mask = BitMask('sv1_bgs_mask', _bitdefs)
types = bgs_mask.names()
bits = [bgs_mask.bitnum(x) for x in types]
##
assigned = join(_assigned,
sv_mtl,
keys=['BRICKID', 'BRICK_OBJID'],
join_type='inner')
assert len(_assigned) == len(assigned)
splits = {}
title = ''
for _type in [
if state not in priorities[bitname]:
priorities[bitname][state] = 1
else:
priorities[bitname] = dict()
#- add to the extra info dictionary for this target mask
for bitdef in _bitdefs[maskname]:
bitname = bitdef[0]
bitdef[3]['priorities'] = priorities[bitname]
return
#- convert to BitMask objects
if _bitdefs is None:
_load_bits()
desi_mask = BitMask('desi_mask', _bitdefs)
mws_mask = BitMask('mws_mask', _bitdefs)
bgs_mask = BitMask('bgs_mask', _bitdefs)
obsconditions = BitMask('obsconditions', _bitdefs)
obsmask = BitMask('obsmask', _bitdefs)
targetid_mask = BitMask('targetid_mask', _bitdefs)
#-------------------------------------------------------------------------
#- Do some error checking that the bitmasks are consistent with each other
# import sys
# error = False
# for mask in desi_target, mws_target, bgs_target:
# for bitname in targetmask.names:
# if targetmask[bitname]
# if bitname not in priorities.keys():
# print >> sys.stderr, "ERROR: no priority defined for "+bitname
def setUp(self):
self.ccdmask = BitMask('ccdmask', _bitdefs)
def setUp(self):
self.ccdmask = BitMask('ccdmask', _bitdefs)
class TestBitMask(unittest.TestCase):
"""Test desiutil.bitmask.
"""
def setUp(self):
self.ccdmask = BitMask('ccdmask', _bitdefs)
def tearDown(self):
pass
def test_names(self):
"""Test consistency for names to bits to masks.
"""
m = self.ccdmask
for name in m.names():
self.assertEqual(m.mask(name), 2**m.bitnum(name),
'Failed matching mask to bitnum for ' + name)
self.assertEqual(m.mask(name), m.mask(m.bitnum(name)),
'Failed matching mask to name for ' + name)
self.assertEqual(
m.bitname(m.bitnum(name)), name,
'Failed bit name->num->name roundtrip for ' + name)
self.assertEqual(m[name], m[name].mask)
self.assertEqual(m.bitname(name), m[name].name)
self.assertEqual(m.bitnum(name), m[name].bitnum)
c = m.comment(name)
names = m.names(m.COSMIC | m.BAD | 2**13)
self.assertTrue('COSMIC' in names)
self.assertTrue('BAD' in names)
self.assertTrue('UNKNOWN13' in names)
def test_mask(self):
"""Test options for blat.mask().
"""
for i in range(4):
self.assertEqual(self.ccdmask.mask(i), 2**i)
m = self.ccdmask
self.assertEqual(m.mask('BAD|COSMIC'), m.BAD | m.COSMIC)
def test_access(self):
"""Miscellaneous stuff that should work.
"""
self.assertEqual(self.ccdmask._name, 'ccdmask')
self.assertEqual(self.ccdmask['HOT'].blat, 'foo')
self.assertEqual(self.ccdmask.HOT.blat, 'foo')
self.assertEqual(self.ccdmask.HOT.name, 'HOT')
self.assertEqual(self.ccdmask.HOT.bitnum, 1)
self.assertEqual(self.ccdmask.HOT.mask, 2)
self.assertEqual(self.ccdmask.HOT, 2)
self.assertEqual(self.ccdmask.HOT.comment, "Hot pixel")
self.ccdmask.names()
def test_badname(self):
"""Test raising AttributeError for bad names.
"""
with self.assertRaises(AttributeError):
x = self.ccdmask.BLATFOO
# Attribute already in use
with self.assertRaises(AttributeError):
bit = _MaskBit('BAD', 0, "comment", extra={'real': 'foo'})
# has extra that isn't a dict
with self.assertRaises(ValueError):
blat = BitMask('ccdmask', _baddef1)
def test_str(self):
"""Verify yaml-ness of string representation"""
bitmask = BitMask('ccdmask', yaml.load(str(self.ccdmask)))
self.assertEqual(bitmask.names(), self.ccdmask.names())
for name in bitmask.names():
self.assertEqual(bitmask[name].mask, self.ccdmask[name].mask)
self.assertEqual(bitmask[name].comment, self.ccdmask[name].comment)
self.assertEqual(bitmask[name].bitnum, self.ccdmask[name].bitnum)
self.assertEqual(bitmask[name]._extra, self.ccdmask[name]._extra)
def test_print(self):
"""Test string representations.
"""
ccdmask_repr = """ccdmask:
- [BAD , 0, "Pre-determined bad pixel (any reason)"]
- [HOT , 1, "Hot pixel"]
- [DEAD , 2, "Dead pixel"]
- [SATURATED , 3, "Saturated pixel from object"]
- [COSMIC , 4, "Cosmic ray"]"""
bit_str = (
("BAD bit 0 mask 0x1 - Pre-determined bad pixel " +
"(any reason)"), "HOT bit 1 mask 0x2 - Hot pixel",
"DEAD bit 2 mask 0x4 - Dead pixel",
"SATURATED bit 3 mask 0x8 - Saturated pixel from object",
"COSMIC bit 4 mask 0x10 - Cosmic ray")
bit_repr = ('1', '2', '4', '8', '16')
blat = repr(self.ccdmask)
self.assertEqual(blat, _bitdefyaml)
for i, name in enumerate(self.ccdmask.names()):
self.assertEqual(str(self.ccdmask[name]), bit_str[i])
self.assertEqual(repr(self.ccdmask[name]), bit_repr[i])
tiles = Table(
fits.open(
'/global/cscratch1/sd/mjwilson/BGS/SV-ASSIGN/tiles/BGS_SV_30_3x_superset60_Sep2019.fits'
)[1].data)
utiles = np.unique(tiles['TILEID'].quantity)
result = Table(names=[
'TARGETID', 'FA_TYPE', 'BRICK_OBJID', 'TARGET_RA', 'TARGET_DEC', 'BRICKID'
],
dtype=('i8', 'i1', 'i4', 'f8', 'f8', 'i4'))
##
_bitdefs = load_mask_bits("sv1")
desi_mask = BitMask('sv1_desi_mask', _bitdefs)
bgs_mask = BitMask('sv1_bgs_mask', _bitdefs)
types = bgs_mask.names()
bits = [bgs_mask.bitnum(x) for x in types]
## L428 of https://github.com/desihub/desimodel/blob/master/py/desimodel/focalplane/geometry.py
params = load_desiparams()
fiber_dia = params['fibers']['diameter_um']
#- Platescales in um/arcsec
ps = load_platescale()
for tile in utiles:
print('Solving for Tile {}.'.format(tile))
- [MANYBADCOL, 4, ">10% of pixels are bad columns"]
- [MANYREJECTED, 5, ">10% of pixels rejected in extraction"]
#- Spectral pixel mask: bits that apply to individual spectral bins
specmask:
- [SOMEBADPIX, 0, "Some input pixels were masked or ivar=0"]
- [ALLBADPIX, 1, "All input pixels were masked or ivar=0"]
- [COSMIC, 2, "Input pixels included a masked cosmic"]
- [LOWFLAT, 3, "Fiber flat < 0.5"]
- [BADFIBERFLAT, 4, "Bad fiber flat solution"]
- [BRIGHTSKY, 5, "Bright sky level (details TBD)"]
- [BADSKY, 6, "Bad sky model"]
#- zmask: reasons why redshift fitting failed
""")
#-------------------------------------------------------------------------
#- The actual masks
try:
specmask = BitMask('specmask', _bitdefs)
ccdmask = BitMask('ccdmask', _bitdefs)
fibermask = BitMask('fibermask', _bitdefs)
except TypeError:
#
# This is needed to allow documentation to build even if desiutil is not
# installed.
#
specmask = object()
ccdmask = object()
fibermask = object()
"""
desitarget.sv1.sv1_targetmask
=============================
This looks more like a script than an actual module.
"""
from desiutil.bitmask import BitMask
from desitarget.targetmask import load_mask_bits
_bitdefs = load_mask_bits("sv1")
try:
desi_mask = BitMask('sv1_desi_mask', _bitdefs)
mws_mask = BitMask('sv1_mws_mask', _bitdefs)
bgs_mask = BitMask('sv1_bgs_mask', _bitdefs)
obsmask = BitMask('sv1_obsmask', _bitdefs)
except TypeError:
desi_mask = object()
mws_mask = object()
bgs_mask = object()
obsmask = object()
"""
desitarget.cmx.cmx_targetmask
=============================
This looks more like a script than an actual module.
"""
from desiutil.bitmask import BitMask
from desitarget.targetmask import load_mask_bits
_bitdefs = load_mask_bits("cmx")
try:
cmx_mask = BitMask('cmx_mask', _bitdefs)
cmx_obsmask = BitMask('cmx_obsmask', _bitdefs)
except TypeError:
cmx_mask = object()
cmx_obsmask = object()
class TestBitMask(unittest.TestCase):
"""Test desiutil.bitmask.
"""
def setUp(self):
self.ccdmask = BitMask('ccdmask', _bitdefs)
def tearDown(self):
pass
def test_names(self):
"""Test consistency for names to bits to masks.
"""
m = self.ccdmask
for name in m.names():
self.assertEqual(m.mask(name), 2**m.bitnum(name),
'Failed matching mask to bitnum for ' + name)
self.assertEqual(m.mask(name), m.mask(m.bitnum(name)),
'Failed matching mask to name for ' + name)
self.assertEqual(m.bitname(m.bitnum(name)), name,
'Failed bit name->num->name roundtrip for ' +
name)
self.assertEqual(m[name], m[name].mask)
self.assertEqual(m.bitname(name), m[name].name)
self.assertEqual(m.bitnum(name), m[name].bitnum)
c = m.comment(name)
names = m.names(m.COSMIC | m.BAD | 2**13)
self.assertTrue('COSMIC' in names)
self.assertTrue('BAD' in names)
self.assertTrue('UNKNOWN13' in names)
def test_mask(self):
"""Test options for blat.mask().
"""
for i in range(4):
self.assertEqual(self.ccdmask.mask(i), 2**i)
m = self.ccdmask
self.assertEqual(m.mask('BAD|COSMIC'), m.BAD | m.COSMIC)
def test_access(self):
"""Miscellaneous stuff that should work.
"""
self.assertEqual(self.ccdmask._name, 'ccdmask')
self.assertEqual(self.ccdmask['HOT'].blat, 'foo')
self.assertEqual(self.ccdmask.HOT.blat, 'foo')
self.assertEqual(self.ccdmask.HOT.name, 'HOT')
self.assertEqual(self.ccdmask.HOT.bitnum, 1)
self.assertEqual(self.ccdmask.HOT.mask, 2)
self.assertEqual(self.ccdmask.HOT, 2)
self.assertEqual(self.ccdmask.HOT.comment, "Hot pixel")
self.ccdmask.names()
def test_badname(self):
"""Test raising AttributeError for bad names.
"""
with self.assertRaises(AttributeError):
x = self.ccdmask.BLATFOO
# Attribute already in use
with self.assertRaises(AttributeError):
bit = _MaskBit('BAD', 0, "comment", extra={'real': 'foo'})
# has extra that isn't a dict
with self.assertRaises(ValueError):
blat = BitMask('ccdmask', _baddef1)
def test_str(self):
"""Verify yaml-ness of string representation"""
bitmask = BitMask('ccdmask', yaml.load(str(self.ccdmask)))
self.assertEqual(bitmask.names(), self.ccdmask.names())
for name in bitmask.names():
self.assertEqual(bitmask[name].mask, self.ccdmask[name].mask)
self.assertEqual(bitmask[name].comment, self.ccdmask[name].comment)
self.assertEqual(bitmask[name].bitnum, self.ccdmask[name].bitnum)
self.assertEqual(bitmask[name]._extra, self.ccdmask[name]._extra)
def test_print(self):
"""Test string representations.
"""
ccdmask_repr = """ccdmask:
- [BAD , 0, "Pre-determined bad pixel (any reason)"]
- [HOT , 1, "Hot pixel"]
- [DEAD , 2, "Dead pixel"]
- [SATURATED , 3, "Saturated pixel from object"]
- [COSMIC , 4, "Cosmic ray"]"""
bit_str = (
("BAD bit 0 mask 0x1 - Pre-determined bad pixel " +
"(any reason)"),
"HOT bit 1 mask 0x2 - Hot pixel",
"DEAD bit 2 mask 0x4 - Dead pixel",
"SATURATED bit 3 mask 0x8 - Saturated pixel from object",
"COSMIC bit 4 mask 0x10 - Cosmic ray")
bit_repr = ('1', '2', '4', '8', '16')
blat = repr(self.ccdmask)
self.assertEqual(blat, _bitdefyaml)
for i, name in enumerate(self.ccdmask.names()):
self.assertEqual(str(self.ccdmask[name]), bit_str[i])
self.assertEqual(repr(self.ccdmask[name]), bit_repr[i])