def read_image(filename):
"""
Returns desispec.image.Image object from input file
"""
log = get_logger()
t0 = time.time()
with fits.open(filename, uint=True, memmap=False) as fx:
image = native_endian(fx['IMAGE'].data).astype(np.float64)
ivar = native_endian(fx['IVAR'].data).astype(np.float64)
mask = native_endian(fx['MASK'].data).astype(np.uint16)
camera = fx['IMAGE'].header['CAMERA'].lower()
meta = fx['IMAGE'].header
if 'READNOISE' in fx:
readnoise = native_endian(fx['READNOISE'].data).astype(np.float64)
else:
readnoise = fx['IMAGE'].header['RDNOISE']
duration = time.time() - t0
log.info(iotime.format('read', filename, duration))
return Image(image,
ivar,
mask=mask,
readnoise=readnoise,
camera=camera,
meta=meta)
def read_fiberflat(filename):
"""Read fiberflat from filename
Args:
filename (str): Name of fiberflat file, or (night, expid, camera) tuple
Returns:
FiberFlat object with attributes
fiberflat, ivar, mask, meanspec, wave, header
Notes:
fiberflat, ivar, mask are 2D [nspec, nwave]
meanspec and wave are 1D [nwave]
"""
#- check if outfile is (night, expid, camera) tuple instead
if isinstance(filename, (tuple, list)) and len(filename) == 3:
night, expid, camera = filename
filename = findfile('fiberflat', night, expid, camera)
header = fits.getheader(filename, 0)
fiberflat = native_endian(fits.getdata(filename, 0))
ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8'))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8'))
wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8'))
return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def read_fiberflat(filename):
"""Read fiberflat from filename
Args:
filename (str): Name of fiberflat file, or (night, expid, camera) tuple
Returns:
FiberFlat object with attributes
fiberflat, ivar, mask, meanspec, wave, header
Notes:
fiberflat, ivar, mask are 2D [nspec, nwave]
meanspec and wave are 1D [nwave]
"""
#- check if outfile is (night, expid, camera) tuple instead
if isinstance(filename, (tuple, list)) and len(filename) == 3:
night, expid, camera = filename
filename = findfile('fiberflat', night, expid, camera)
header = fits.getheader(filename, 0)
fiberflat = native_endian(fits.getdata(filename, 0)).astype('f8')
ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8'))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8'))
wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8'))
return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def read_frame(filename, nspec=None):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file, or (night, expid, camera) tuple where
night = string YEARMMDD
expid = integer exposure ID
camera = b0, r1, .. z9
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('frame', night, expid, camera)
if not os.path.isfile(filename):
raise IOError("cannot open" + filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8'))
if 'FIBERMAP' in fx:
fibermap = fx['FIBERMAP'].data
else:
fibermap = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
resolution_data = resolution_data[0:nspec]
# return flux,ivar,wave,resolution_data, hdr
return Frame(wave,
flux,
ivar,
mask,
resolution_data,
meta=hdr,
fibermap=fibermap)
def read_frame(filename, nspec=None):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file, or (night, expid, camera) tuple where
night = string YEARMMDD
expid = integer exposure ID
camera = b0, r1, .. z9
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('frame', night, expid, camera)
if not os.path.isfile(filename) :
raise IOError("cannot open"+filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8'))
if 'FIBERMAP' in fx:
fibermap = fx['FIBERMAP'].data
else:
fibermap = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
resolution_data = resolution_data[0:nspec]
# return flux,ivar,wave,resolution_data, hdr
return Frame(wave, flux, ivar, mask, resolution_data, meta=hdr, fibermap=fibermap)
def read_image(filename):
"""
Returns desispec.image.Image object from input file
"""
fx = fits.open(filename, uint=True, memmap=False)
image = native_endian(fx['IMAGE'].data).astype(np.float64)
ivar = native_endian(fx['IVAR'].data).astype(np.float64)
mask = native_endian(fx['MASK'].data).astype(np.uint16)
camera = fx['IMAGE'].header['CAMERA'].lower()
meta = fx['IMAGE'].header
if 'READNOISE' in fx:
readnoise = native_endian(fx['READNOISE'].data).astype(np.float64)
else:
readnoise = fx['IMAGE'].header['RDNOISE']
return Image(image, ivar, mask=mask, readnoise=readnoise,
camera=camera, meta=meta)
def read_zbest(filename):
"""Returns a desispec.zfind.ZfindBase object with contents from filename.
"""
from desispec.io.util import native_endian
fx = fits.open(filename, memmap=False)
zbest = fx['ZBEST'].data
if 'WAVELENGTH' in fx:
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
model = fx['MODEL'].data
zf = ZfindBase(wave, flux, ivar, results=zbest)
zf.model = model
else:
zf = ZfindBase(None, None, None, results=zbest)
fx.close()
return zf
def read_zbest(filename):
"""Returns a desispec.zfind.ZfindBase object with contents from filename.
"""
from desispec.io.util import native_endian
fx = fits.open(filename, memmap=False)
zbest = fx['ZBEST'].data
if 'WAVELENGTH' in fx:
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
model = fx['MODEL'].data
zf = ZfindBase(wave, flux, ivar, results=zbest)
zf.model = model
else:
zf = ZfindBase(None, None, None, results=zbest)
fx.close()
return zf
def read_sky(filename):
"""Read sky model and return SkyModel object with attributes
wave, flux, ivar, mask, header.
skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave]
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('sky', night, expid, camera)
hdr = fits.getheader(filename, 0)
wave = native_endian(fits.getdata(filename, "WAVELENGTH"))
skyflux = native_endian(fits.getdata(filename, "SKY"))
ivar = native_endian(fits.getdata(filename, "IVAR"))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr)
return skymodel
def read_sky(filename) :
"""Read sky model and return SkyModel object with attributes
wave, flux, ivar, mask, header.
skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave]
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('sky', night, expid, camera)
hdr = fits.getheader(filename, 0)
wave = native_endian(fits.getdata(filename, "WAVELENGTH"))
skyflux = native_endian(fits.getdata(filename, "SKY"))
ivar = native_endian(fits.getdata(filename, "IVAR"))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr)
return skymodel
def read_sky(filename) :
"""Read sky model and return SkyModel object with attributes
wave, flux, ivar, mask, header.
skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave]
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('sky', night, expid, camera)
fx = fits.open(filename, memmap=False, uint=True)
hdr = fx[0].header
wave = native_endian(fx["WAVELENGTH"].data.astype('f8'))
skyflux = native_endian(fx["SKY"].data.astype('f8'))
ivar = native_endian(fx["IVAR"].data.astype('f8'))
mask = native_endian(fx["MASK"].data)
fx.close()
skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr)
return skymodel
def read_cosmics(filename, expid=1, shape=None, jitter=True):
"""
Reads a dark image with cosmics from the input filename.
The input might have multiple dark images; use the `expid%n` image where
`n` is the number of images in the input cosmics file.
Args:
filename : FITS filename with EXTNAME=IMAGE-*, IVAR-*, MASK-* HDUs
expid : integer, use `expid % n` image where `n` is number of images
Optional:
shape : (ny, nx) tuple for output image shape
jitter : If True (default), apply random flips and rolls so you
don't get the exact same cosmics every time
Returns:
`desisim.image.Image` object with attributes pix, ivar, mask
"""
fx = fits.open(filename)
imagekeys = list()
for i in range(len(fx)):
if fx[i].name.startswith('IMAGE-'):
imagekeys.append(fx[i].name.split('-', 1)[1])
assert len(imagekeys) > 0, 'No IMAGE-* extensions found in '+filename
i = expid % len(imagekeys)
pix = native_endian(fx['IMAGE-'+imagekeys[i]].data.astype(np.float64))
ivar = native_endian(fx['IVAR-'+imagekeys[i]].data.astype(np.float64))
mask = native_endian(fx['MASK-'+imagekeys[i]].data)
meta = fx['IMAGE-'+imagekeys[i]].header
meta['CRIMAGE'] = (imagekeys[i], 'input cosmic ray image')
if shape is not None:
if len(shape) != 2: raise ValueError('Invalid shape {}'.format(shape))
newpix = np.empty(shape, dtype=np.float64)
ny = min(shape[0], pix.shape[0])
nx = min(shape[1], pix.shape[1])
newpix[0:ny, 0:nx] = pix[0:ny, 0:nx]
pix = _resize(pix, shape)
ivar = _resize(ivar, shape)
mask = _resize(mask, shape)
if jitter:
#- Randomly flip left-right and/or up-down
if np.random.uniform(0, 1) > 0.5:
pix = np.fliplr(pix)
ivar = np.fliplr(ivar)
mask = np.fliplr(mask)
meta['CRFLIPLR'] = (True, 'Input cosmics image flipped Left/Right')
else:
meta['CRFLIPLR'] = (False, 'Input cosmics image NOT flipped Left/Right')
if np.random.uniform(0, 1) > 0.5:
pix = np.flipud(pix)
ivar = np.flipud(ivar)
mask = np.flipud(mask)
meta['CRFLIPUD'] = (True, 'Input cosmics image flipped Up/Down')
else:
meta['CRFLIPUD'] = (True, 'Input cosmics image NOT flipped Up/Down')
#- Randomly roll image a bit
nx, ny = np.random.randint(-200, 200, size=2)
pix = np.roll(np.roll(pix, ny, axis=0), nx, axis=1)
ivar = np.roll(np.roll(ivar, ny, axis=0), nx, axis=1)
mask = np.roll(np.roll(mask, ny, axis=0), nx, axis=1)
meta['CRSHIFTX'] = (nx, 'Input cosmics image shift in x')
meta['CRSHIFTY'] = (nx, 'Input cosmics image shift in y')
else:
meta['CRFLIPLR'] = (False, 'Input cosmics image NOT flipped Left/Right')
meta['CRFLIPUD'] = (True, 'Input cosmics image NOT flipped Up/Down')
meta['CRSHIFTX'] = (0, 'Input cosmics image shift in x')
meta['CRSHIFTY'] = (0, 'Input cosmics image shift in y')
#- RDNOISEn -> average RDNOISE
if 'RDNOISE' not in meta:
x = meta['RDNOISE0']+meta['RDNOISE1']+meta['RDNOISE2']+meta['RDNOISE3']
meta['RDNOISE'] = x / 4.0
return Image(pix, ivar, mask, meta=meta)
def read_cosmics(filename, expid=1, shape=None, jitter=True):
"""
Reads a dark image with cosmics from the input filename.
The input might have multiple dark images; use the `expid%n` image where
`n` is the number of images in the input cosmics file.
Args:
filename : FITS filename with EXTNAME=IMAGE-*, IVAR-*, MASK-* HDUs
expid : integer, use `expid % n` image where `n` is number of images
Optional:
shape : (ny, nx) tuple for output image shape
jitter : If True (default), apply random flips and rolls so you
don't get the exact same cosmics every time
Returns:
`desisim.image.Image` object with attributes pix, ivar, mask
"""
fx = fits.open(filename)
imagekeys = list()
for i in range(len(fx)):
if fx[i].name.startswith('IMAGE-'):
imagekeys.append(fx[i].name.split('-', 1)[1])
assert len(imagekeys) > 0, 'No IMAGE-* extensions found in ' + filename
i = expid % len(imagekeys)
pix = native_endian(fx['IMAGE-' + imagekeys[i]].data.astype(np.float64))
ivar = native_endian(fx['IVAR-' + imagekeys[i]].data.astype(np.float64))
mask = native_endian(fx['MASK-' + imagekeys[i]].data)
meta = fx['IMAGE-' + imagekeys[i]].header
meta['CRIMAGE'] = (imagekeys[i], 'input cosmic ray image')
if shape is not None:
if len(shape) != 2: raise ValueError('Invalid shape {}'.format(shape))
newpix = np.empty(shape, dtype=np.float64)
ny = min(shape[0], pix.shape[0])
nx = min(shape[1], pix.shape[1])
newpix[0:ny, 0:nx] = pix[0:ny, 0:nx]
pix = _resize(pix, shape)
ivar = _resize(ivar, shape)
mask = _resize(mask, shape)
if jitter:
#- Randomly flip left-right and/or up-down
if np.random.uniform(0, 1) > 0.5:
pix = np.fliplr(pix)
ivar = np.fliplr(ivar)
mask = np.fliplr(mask)
meta['CRFLIPLR'] = (True, 'Input cosmics image flipped Left/Right')
else:
meta['CRFLIPLR'] = (False,
'Input cosmics image NOT flipped Left/Right')
if np.random.uniform(0, 1) > 0.5:
pix = np.flipud(pix)
ivar = np.flipud(ivar)
mask = np.flipud(mask)
meta['CRFLIPUD'] = (True, 'Input cosmics image flipped Up/Down')
else:
meta['CRFLIPUD'] = (True,
'Input cosmics image NOT flipped Up/Down')
#- Randomly roll image a bit
nx, ny = np.random.randint(-200, 200, size=2)
pix = np.roll(np.roll(pix, ny, axis=0), nx, axis=1)
ivar = np.roll(np.roll(ivar, ny, axis=0), nx, axis=1)
mask = np.roll(np.roll(mask, ny, axis=0), nx, axis=1)
meta['CRSHIFTX'] = (nx, 'Input cosmics image shift in x')
meta['CRSHIFTY'] = (nx, 'Input cosmics image shift in y')
else:
meta['CRFLIPLR'] = (False,
'Input cosmics image NOT flipped Left/Right')
meta['CRFLIPUD'] = (True, 'Input cosmics image NOT flipped Up/Down')
meta['CRSHIFTX'] = (0, 'Input cosmics image shift in x')
meta['CRSHIFTY'] = (0, 'Input cosmics image shift in y')
#- RDNOISEn -> average RDNOISE
if 'RDNOISE' not in meta:
x = meta['RDNOISE0'] + meta['RDNOISE1'] + meta['RDNOISE2'] + meta[
'RDNOISE3']
meta['RDNOISE'] = x / 4.0
return Image(pix, ivar, mask, meta=meta)
def read_spectra(infile, single=False, coadd=None):
"""Read Spectra object from FITS file.
This reads data written by the write_spectra function. A new Spectra
object is instantiated and returned.
Args:
infile (str): path to read
single (bool): if True, keep spectra as single precision in memory.
coadd (array-like): if set, coadd all spectra from the provided targetids.
Returns (Spectra):
The object containing the data read from disk.
"""
ftype = np.float64
if single:
ftype = np.float32
infile = os.path.abspath(infile)
if not os.path.isfile(infile):
raise FileNotFoundError("{} is not a file!".format(infile))
# initialize data objects
bands = []
fmap = None
wave = None
flux = None
ivar = None
mask = None
res = None
extra = None
scores = None
with fits.open(infile, mode="readonly") as hdulist:
nhdu = len(hdulist)
# load the metadata.
meta = hdulist[0].header
# For efficiency, go through the HDUs in disk-order. Use the
# extension name to determine where to put the data. We don't
# explicitly copy the data, since that will be done when constructing
# the Spectra object.
for h in range(1, nhdu):
name = hdulist[h].header["EXTNAME"]
if name == "FIBERMAP":
fmap = encode_table(Table(hdulist[h].data, copy=True).as_array())
elif name == "SCORES":
scores = encode_table(Table(hdulist[h].data, copy=True).as_array())
else:
# Find the band based on the name
mat = re.match(r"(.*)_(.*)", name)
if mat is None:
raise RuntimeError("FITS extension name {} does not contain the band".format(name))
band = mat.group(1).lower()
type = mat.group(2)
if band not in bands:
bands.append(band)
if type == "WAVELENGTH":
if wave is None:
wave = {}
wave[band] = native_endian(hdulist[h].data.astype(ftype))
elif type == "FLUX":
if flux is None:
flux = {}
flux[band] = native_endian(hdulist[h].data.astype(ftype))
elif type == "IVAR":
if ivar is None:
ivar = {}
ivar[band] = native_endian(hdulist[h].data.astype(ftype))
elif type == "MASK":
if mask is None:
mask = {}
mask[band] = native_endian(hdulist[h].data.astype(np.uint32))
elif type == "RESOLUTION":
if res is None:
res = {}
res[band] = native_endian(hdulist[h].data.astype(ftype))
else:
# this must be an "extra" HDU
if extra is None:
extra = {}
if band not in extra:
extra[band] = {}
extra[band][type] = native_endian(hdulist[h].data.astype(ftype))
if coadd is not None:
uniq, indices = np.unique(coadd, return_index=True)
targetids = uniq[indices.argsort()]
ntargets = len(targetids)
cwave = dict()
cflux = dict()
civar = dict()
crdat = dict()
cmask = dict()
for channel in bands:
cwave[channel] = wave[channel].copy()
nwave = len(cwave[channel])
cflux[channel] = np.zeros((ntargets, nwave))
civar[channel] = np.zeros((ntargets, nwave))
ndiag = res[channel].shape[1]
crdat[channel] = np.zeros((ntargets, ndiag, nwave))
cmask[channel] = np.zeros((ntargets, nwave), dtype=mask[channel].dtype)
#- Loop over targets, coadding all spectra for each target
fibermap = Table(dtype=fmap.dtype)
for i, targetid in enumerate(targetids):
ii = np.where(fmap['TARGETID'] == targetid)[0]
fibermap.add_row(fmap[ii[0]])
for channel in bands:
if len(ii) > 1:
outwave, outflux, outivar, outrdat = _coadd(
wave[channel],
flux[channel][ii],
ivar[channel][ii],
res[channel][ii]
)
outmask = mask[channel][ii[0]]
for j in range(1, len(ii)):
outmask |= mask[channel][ii[j]]
else:
outwave, outflux, outivar, outrdat = (
wave[channel],
flux[channel][ii[0]],
ivar[channel][ii[0]],
res[channel][ii[0]]
)
outmask = mask[channel][ii[0]]
cflux[channel][i] = outflux
civar[channel][i] = outivar
crdat[channel][i] = outrdat
cmask[channel][i] = outmask
return Spectra(bands, cwave, cflux, civar, mask=cmask, resolution_data=crdat,
fibermap=fibermap, meta=meta, extra=extra, single=single,
scores=scores)
# Construct the Spectra object from the data. If there are any
# inconsistencies in the sizes of the arrays read from the file,
# they will be caught by the constructor.
return Spectra(bands, wave, flux, ivar, mask=mask, resolution_data=res,
fibermap=fmap, meta=meta, extra=extra, single=single,
scores=scores)
def read_qframe(filename, nspec=None, skip_resolution=False):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file
skip_resolution: bool, option
Speed up read time (>5x) by avoiding the Resolution matrix
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
log = get_logger()
if not os.path.isfile(filename) :
raise IOError("cannot open"+filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if wave.shape != flux.shape :
log.error("{} is not a valid QFrame file because wave.shape != flux.shape".format(filename))
return None
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
if 'SIGMA' in fx:
sigma = native_endian(fx['SIGMA'].data.astype('f8'))
else:
sigma = None
if 'FIBERMAP' in fx:
fibermap = fx['FIBERMAP'].data
fibers = fibermap['FIBER']
else:
fibermap = None
fibers = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
if mask is not None:
mask = mask[0:nspec]
if fibermap is not None:
fibermap = fibermap[:][0:nspec]
if fibers is not None:
fibers = fibers[:][0:nspec]
# return flux,ivar,wave,resolution_data, hdr
qframe = QFrame(wave, flux, ivar, mask=mask, sigma=sigma, meta=hdr, fibermap=fibermap, fibers=fibers)
# Return
return qframe
def read_cosmics(filename, expid=1, shape=None, jitter=True):
"""
Reads a dark image with cosmics from the input filename.
The input might have multiple dark images; use the `expid%n` image where
`n` is the number of images in the input cosmics file.
Args:
filename : FITS filename with EXTNAME=IMAGE-*, IVAR-*, MASK-* HDUs
expid : integer, use `expid % n` image where `n` is number of images
shape : (ny, nx, optional) tuple for output image shape
jitter (bool, optional): If True (default), apply random flips and rolls so you
don't get the exact same cosmics every time
Returns:
`desisim.image.Image` object with attributes pix, ivar, mask
"""
fx = fits.open(filename)
imagekeys = list()
for i in range(len(fx)):
if fx[i].name.startswith('IMAGE-'):
imagekeys.append(fx[i].name.split('-', 1)[1])
assert len(imagekeys) > 0, 'No IMAGE-* extensions found in ' + filename
i = expid % len(imagekeys)
pix = native_endian(fx['IMAGE-' + imagekeys[i]].data.astype(np.float64))
ivar = native_endian(fx['IVAR-' + imagekeys[i]].data.astype(np.float64))
mask = native_endian(fx['MASK-' + imagekeys[i]].data)
meta = fx['IMAGE-' + imagekeys[i]].header
meta['CRIMAGE'] = (imagekeys[i], 'input cosmic ray image')
#- De-trend each amplifier
nx = pix.shape[1] // 2
ny = pix.shape[0] // 2
kernel_size = min(201, ny // 3, nx // 3)
pix[0:ny, 0:nx] -= spline_medfilt2d(pix[0:ny, 0:nx], kernel_size)
pix[0:ny, nx:2 * nx] -= spline_medfilt2d(pix[0:ny, nx:2 * nx], kernel_size)
pix[ny:2 * ny, 0:nx] -= spline_medfilt2d(pix[ny:2 * ny, 0:nx], kernel_size)
pix[ny:2 * ny, nx:2 * nx] -= spline_medfilt2d(pix[ny:2 * ny, nx:2 * nx],
kernel_size)
if shape is not None:
if len(shape) != 2: raise ValueError('Invalid shape {}'.format(shape))
pix = _resize(pix, shape)
ivar = _resize(ivar, shape)
mask = _resize(mask, shape)
if jitter:
#- Randomly flip left-right and/or up-down
if np.random.uniform(0, 1) > 0.5:
pix = np.fliplr(pix)
ivar = np.fliplr(ivar)
mask = np.fliplr(mask)
meta['CRFLIPLR'] = (True, 'Input cosmics image flipped Left/Right')
else:
meta['CRFLIPLR'] = (False,
'Input cosmics image NOT flipped Left/Right')
if np.random.uniform(0, 1) > 0.5:
pix = np.flipud(pix)
ivar = np.flipud(ivar)
mask = np.flipud(mask)
meta['CRFLIPUD'] = (True, 'Input cosmics image flipped Up/Down')
else:
meta['CRFLIPUD'] = (False,
'Input cosmics image NOT flipped Up/Down')
#- Randomly roll image a bit
nx, ny = np.random.randint(-100, 100, size=2)
pix = np.roll(np.roll(pix, ny, axis=0), nx, axis=1)
ivar = np.roll(np.roll(ivar, ny, axis=0), nx, axis=1)
mask = np.roll(np.roll(mask, ny, axis=0), nx, axis=1)
meta['CRSHIFTX'] = (nx, 'Input cosmics image shift in x')
meta['CRSHIFTY'] = (nx, 'Input cosmics image shift in y')
else:
meta['CRFLIPLR'] = (False,
'Input cosmics image NOT flipped Left/Right')
meta['CRFLIPUD'] = (False, 'Input cosmics image NOT flipped Up/Down')
meta['CRSHIFTX'] = (0, 'Input cosmics image shift in x')
meta['CRSHIFTY'] = (0, 'Input cosmics image shift in y')
del meta['RDNOISE0']
#- Amp 1 lower left
nx = pix.shape[1] // 2
ny = pix.shape[0] // 2
iixy = np.s_[0:ny, 0:nx]
cx = pix[iixy][mask[iixy] == 0]
mean, median, std = sigma_clipped_stats(cx, sigma=3, iters=5)
meta['RDNOISE1'] = std
#- Amp 2 lower right
iixy = np.s_[0:ny, nx:2 * nx]
cx = pix[iixy][mask[iixy] == 0]
mean, median, std = sigma_clipped_stats(cx, sigma=3, iters=5)
meta['RDNOISE2'] = std
#- Amp 3 upper left
iixy = np.s_[ny:2 * ny, 0:nx]
mean, median, std = sigma_clipped_stats(pix[iixy], sigma=3, iters=5)
meta['RDNOISE3'] = std
#- Amp 4 upper right
iixy = np.s_[ny:2 * ny, nx:2 * nx]
mean, median, std = sigma_clipped_stats(pix[iixy], sigma=3, iters=5)
meta['RDNOISE4'] = std
fx.close()
return Image(pix, ivar, mask, meta=meta)
def read_simspec(filename,
cameras=None,
comm=None,
readflux=True,
readphot=True):
"""
Read a simspec file and return a SimSpec object
Args:
filename: input simspec file name
Options:
cameras: camera name or list of names, e.g. b0, r1, z9
comm: MPI communicator
readflux: if True (default), include flux
readphot: if True (default), include per-camera photons
"""
if comm is not None:
rank, size = comm.rank, comm.size
else:
rank, size = 0, 1
if cameras is None:
#- Build the potential cameras list based upon the fibermap
if rank == 0:
fibermap = fits.getdata(filename, 'FIBERMAP')
cameras = fibers2cameras(fibermap['FIBER'])
if comm is not None:
cameras = comm.bcast(cameras, root=0)
elif isinstance(cameras, str):
cameras = [
cameras,
]
#- Read and broadcast data that are common across cameras
header = flavor = truth = fibermap = obsconditions = None
wave = flux = skyflux = None
if rank == 0:
with fits.open(filename, memmap=False) as fx:
header = fx[0].header.copy()
flavor = header['FLAVOR']
if 'WAVE' in fx and readflux:
wave = native_endian(fx['WAVE'].data.copy())
if 'FLUX' in fx and readflux:
flux = native_endian(fx['FLUX'].data.astype('f8'))
if 'SKYFLUX' in fx and readflux:
skyflux = native_endian(fx['SKYFLUX'].data.astype('f8'))
if 'TRUTH' in fx:
truth = Table(fx['TRUTH'].data)
if 'FIBERMAP' in fx:
fibermap = Table(fx['FIBERMAP'].data)
if 'OBSCONDITIONS' in fx:
obsconditions = Table(fx['OBSCONDITIONS'].data)[0]
if comm is not None:
header = comm.bcast(header, root=0)
flavor = comm.bcast(flavor, root=0)
truth = comm.bcast(truth, root=0)
fibermap = comm.bcast(fibermap, root=0)
obsconditions = comm.bcast(obsconditions, root=0)
wave = comm.bcast(wave, root=0)
flux = comm.bcast(flux, root=0)
skyflux = comm.bcast(skyflux, root=0)
#- Trim arrays to match camera
#- Note: we do this after the MPI broadcast because rank 0 doesn't know
#- which ranks need which cameras. Although inefficient, in practice
#- this doesn't matter (yet?) because the place we use parallelism is
#- pixsim which uses readflux=False and thus doesn't broadcast flux anyway
ii = np.zeros(len(fibermap), dtype=bool)
for camera in cameras:
spectrograph = int(camera[1]) #- e.g. b0
fibers = np.arange(500, dtype=int) + spectrograph * 500
ii |= np.in1d(fibermap['FIBER'], fibers)
assert np.any(ii), "input simspec doesn't cover cameras {}".format(cameras)
full_fibermap = fibermap
fibermap = fibermap[ii]
if flux is not None:
flux = flux[ii]
if skyflux is not None:
skyflux = skyflux[ii]
if truth is not None:
truth = truth[ii]
simspec = SimSpec(flavor, wave, flux, skyflux, fibermap, truth,
obsconditions, header)
#- Now read individual camera photons
#- NOTE: this is somewhat inefficient since the same PHOT_B/R/Z HDUs
#- are read multiple times by different nodes for different cameras,
#- though at least only one reader per camera (node) not per rank.
#- If this is slow, this would be an area for optimization.
if readphot:
for camera in cameras:
channel = camera[0].upper()
spectrograph = int(camera[1])
fiber = full_fibermap['FIBER']
ii = (spectrograph * 500 <= fiber) & (fiber <
(spectrograph + 1) * 500)
assert np.any(ii), 'Camera {} is not in fibers {}-{}'.format(
camera, np.min(fiber), np.max(fiber))
#- Split MPI communicator by camera
#- read and broadcast each camera
if comm is not None:
tmp = 'b0 r0 z0 b1 r1 z1 b2 r2 z2 b3 r3 z3 b4 r4 z4 b5 r5 z5 b6 r6 z6 b7 r7 z7 b8 r8 z8 b9 r9 z9'.split(
)
camcomm = comm.Split(color=tmp.index(camera))
camrank = camcomm.rank
else:
camcomm = None
camrank = 0
wave = phot = skyphot = None
if camrank == 0:
with fits.open(filename, memmap=False) as fx:
wave = native_endian(fx['WAVE_' + channel].data.copy())
phot = native_endian(fx['PHOT_' +
channel].data[ii].astype('f8'))
if 'SKYPHOT_' + channel in fx:
skyphot = native_endian(
fx['SKYPHOT_' + channel].data[ii].astype('f8'))
else:
skyphot = None
if camcomm is not None:
wave = camcomm.bcast(wave, root=0)
phot = camcomm.bcast(phot, root=0)
skyphot = camcomm.bcast(skyphot, root=0)
simspec.add_camera(camera, wave, phot, skyphot)
return simspec
def read_qframe(filename, nspec=None, skip_resolution=False):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file
skip_resolution: bool, option
Speed up read time (>5x) by avoiding the Resolution matrix
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
log = get_logger()
if not os.path.isfile(filename):
raise IOError("cannot open" + filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if wave.shape != flux.shape:
log.error(
"{} is not a valid QFrame file because wave.shape != flux.shape".
format(filename))
return None
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
if 'SIGMA' in fx:
sigma = native_endian(fx['SIGMA'].data.astype('f8'))
else:
sigma = None
if 'FIBERMAP' in fx:
fibermap = fx['FIBERMAP'].data
fibers = fibermap['FIBER']
else:
fibermap = None
fibers = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
if mask is not None:
mask = mask[0:nspec]
if fibermap is not None:
fibermap = fibermap[:][0:nspec]
if fibers is not None:
fibers = fibers[:][0:nspec]
# return flux,ivar,wave,resolution_data, hdr
qframe = QFrame(wave,
flux,
ivar,
mask=mask,
sigma=sigma,
meta=hdr,
fibermap=fibermap,
fibers=fibers)
# Return
return qframe
def read_simspec_mpi(filename, comm, channel, spectrographs=None):
"""
Read simspec data from filename and return SimSpec object.
"""
import astropy.table
from mpi4py import MPI #need to reimport this so MPI.DOUBLE can be used
# rank 0 opens file and gets the metadata and wavelength
# grids, which will be kept on all processes.
hdr = None
flavor = None
fibermap = None
obs = None
wave = dict()
totalspec = None
if comm.rank == 0:
fx = fits.open(filename, memmap=True)
hdr = fx[0].header.copy()
flavor = hdr['FLAVOR']
if 'WAVE' in fx:
wave['brz'] = native_endian(fx['WAVE'].data.copy())
#for channel in ('b', 'r', 'z'):
hname = 'WAVE_' + channel.upper()
wave[channel] = native_endian(fx[hname].data.copy())
if 'FIBERMAP' in fx:
fibermap = astropy.table.Table(fx['FIBERMAP'].data.copy())
totalspec = len(fibermap)
else:
# Get the number of spectra from one of the photon HDUs
totalspec = fx['PHOT_B'].header['NAXIS2']
if 'OBSCONDITIONS' in fx:
obs = astropy.table.Table(fx['OBSCONDITIONS'].data.copy())[0]
fx.close()
# Memmap file handle should close here when fx goes out of scope...
hdr = comm.bcast(hdr, root=0)
flavor = comm.bcast(flavor, root=0)
obs = comm.bcast(obs, root=0)
totalspec = comm.bcast(totalspec, root=0)
wave = comm.bcast(wave, root=0)
fibermap = comm.bcast(fibermap, root=0)
# Based on the spectrographs for this process, compute the range of
# spectra we need to store. The number of spectra per spectrograph (500)
# is hard-coded several places in desisim. This should be put in desimodel
# somewhere...
fibers = None
if fibermap is not None:
fibers = np.array(fibermap['FIBER'], dtype=np.int32)
else:
fibers = np.arange(totalspec, dtype=np.int32)
#this comes in as an input, spectrographs=group_spectra[i]
#just simulate and write one spectrograph at a time
if spectrographs is None:
spectrographs = np.arange(10, dtype=np.int32)
specslice = np.in1d(fibers // 500, spectrographs)
if fibermap is not None:
fibermap = fibermap[specslice]
# Now read one HDU at a time, broadcast, and every process grabs its slice.
# Note: this is global scope within the function, so memmap file handle
# will not close until the function returns (which is fine).
# only want to open file for rank 0 to cut down on I/O
if comm.rank == 0:
hdus = None
hdus = fits.open(filename, memmap=True)
#in these next blocks, we fix the endian-ness of the data for each type
#we also extract the data for later broadcast if comm.rank == 0
#and we store the shape of the data to allocate numpy arrays if comm_rank !=0
#it looks a little silly but it's better than trying to convert from a python
#dictionary back to a numpy array later
#preallocate shape_dict
shape_dict = dict()
#have to check in a different way than read_simspec since we don't
#have fx availble on all ranks
#we will sort according to flavor in the mpi version
#flavor = flat has phot and flux
#flavor = arc has phot
#flavor = science has phot, flux, skyphot, and skyflux
#all flavors have photons
hname = 'PHOT_' + channel.upper()
if comm.rank == 0:
phot_hdata = np.empty(1, dtype=np.float64)
phot_hdata = native_endian(hdus[hname].data.copy().astype('f8'))
shape_dict[hname] = hdus[hname].shape
del hname
#flavors=science and flat have flux
hname = 'FLUX'
if comm.rank == 0:
flux_hdata = np.empty(1, dtype=np.float64)
if (flavor == 'science') or (flavor == 'flat'):
flux_hdata = native_endian(hdus[hname].data.copy().astype('f8'))
shape_dict[hname] = hdus[hname].shape
del hname
#only flavor=science has skyphot
hname = 'SKYPHOT_' + channel.upper()
if comm.rank == 0:
sky_hdata = np.empty(1, dtype=np.float64)
if flavor == 'science':
sky_hdata = native_endian(hdus[hname].data.copy().astype('f8'))
shape_dict[hname] = hdus[hname].shape
del hname
#only flavor=science has skyflux
hname = 'SKYFLUX'
if comm.rank == 0:
skyflux_hdata = np.empty(1, dtype=np.float64)
if flavor == 'science':
skyflux_hdata = native_endian(hdus[hname].data.copy().astype('f8'))
shape_dict[hname] = hdus[hname].shape
del hname
#now put shape tuples into a dict so we can broadcast them
#one broadcast is more efficienct than many broadcasts
#this allows the ranks to know what size to preallocate
shape_dict = comm.bcast(shape_dict, root=0)
#add a barrier so we can make sure these data have been broadcast
#to all ranks before we start the next process
comm.Barrier()
#we will sort according to flavor in the mpi version
#flavor = flat has phot and flux
#flavor = arc has phot
#flavor = science has phot, flux, skyphot, and skyflux
# Read photons
# all flavors have photons
phot = dict()
if comm.rank == 0:
#hdata=phot_hdata
phot[channel] = phot_hdata[specslice]
phot = comm.bcast(phot, root=0)
# Read flux
#flavors science and flat have flux
flux = dict()
if (flavor == 'science') or (flavor == 'flat'):
if comm.rank == 0:
flux[channel] = flux_hdata[specslice]
flux = comm.bcast(flux, root=0)
# Read sky photons
#only flavor=science has skyphot
skyphot = dict()
if flavor == 'science':
if comm.rank == 0:
skyphot[channel] = sky_hdata[specslice]
skyphot = comm.bcast(skyphot, root=0)
else:
skyphot[channel] = np.zeros_like(phot[channel])
assert phot[channel].shape == skyphot[channel].shape
# Read skyflux
#only flavor science has skyflux
skyflux = np.empty(1, dtype=np.float64)
if flavor == 'science':
if comm.rank == 0:
skyflux = skyflux_hdata[specslice]
elif comm.rank != 0:
skyflux = np.empty(shape_dict['SKYFLUX'],
dtype=np.float64)[specslice]
comm.Bcast([skyflux, MPI.DOUBLE], root=0)
# metadata / truth
metadata = None
hname = 'TRUTH'
found = False
if comm.rank == 0:
if hname in hdus:
found = True
found = comm.bcast(found, root=0)
if not found:
hname = 'METADATA'
if comm.rank == 0:
if hname in hdus:
found = True
found = comm.bcast(found, root=0)
if found:
hdata = None
if comm.rank == 0:
hdata = astropy.table.Table(hdus[hname].data.copy())
hdata = comm.bcast(
hdata, root=0
) #leaving this call alone because hdata is not a numpy array
metadata = hdata[specslice].copy()
del hdata
if comm.rank == 0:
hdus.close()
return SimSpec(flavor,
wave,
phot,
flux=flux,
skyflux=skyflux,
skyphot=skyphot,
metadata=metadata,
fibermap=fibermap,
obs=obs,
header=hdr)
def read_simspec(filename, nspec=None, firstspec=0):
"""Read simspec data from filename and return SimSpec object.
"""
import astropy.table
with fits.open(filename, memmap=False) as fx:
hdr = fx[0].header
flavor = hdr['FLAVOR']
#- All flavors have photons
wave = dict()
phot = dict()
skyphot = dict()
for channel in ('b', 'r', 'z'):
wave[channel] = native_endian(fx['WAVE_' + channel.upper()].data)
phot[channel] = native_endian(
fx['PHOT_' + channel.upper()].data.astype('f8'))
skyext = 'SKYPHOT_' + channel.upper()
if skyext in fx:
skyphot[channel] = native_endian(fx[skyext].data.astype('f8'))
else:
skyphot[channel] = np.zeros_like(phot[channel])
assert phot[channel].shape == skyphot[channel].shape
#- Check for flux, skyflux, and metadata
flux = None
skyflux = None
if 'WAVE' in fx:
wave['brz'] = native_endian(fx['WAVE'].data)
if 'FLUX' in fx:
flux = native_endian(fx['FLUX'].data.astype('f8'))
if 'SKYFLUX' in fx:
skyflux = native_endian(fx['SKYFLUX'].data.astype('f8'))
if 'TRUTH' in fx:
metadata = astropy.table.Table(fx['TRUTH'].data)
#- For backwards compatibility
elif 'METADATA' in fx:
metadata = astropy.table.Table(fx['METADATA'].data)
else:
metadata = None
if 'FIBERMAP' in fx:
fibermap = astropy.table.Table(fx['FIBERMAP'].data)
else:
fibermap = None
if 'OBSCONDITIONS' in fx:
obs = astropy.table.Table(fx['OBSCONDITIONS'].data)[0]
else:
obs = None
#- Trim down if requested
if nspec is None:
nspec = phot['b'].shape[0]
if firstspec > 0 or firstspec + nspec < phot['b'].shape[0]:
for channel in ('b', 'r', 'z'):
phot[channel] = phot[channel][firstspec:firstspec + nspec]
skyphot[channel] = skyphot[channel][firstspec:firstspec + nspec]
if flux is not None:
flux = flux[firstspec:firstspec + nspec]
if skyflux is not None:
skyflux = skyflux[firstspec:firstspec + nspec]
if metadata is not None:
metadata = metadata[firstspec:firstspec + nspec]
return SimSpec(flavor,
wave,
phot,
flux=flux,
skyflux=skyflux,
skyphot=skyphot,
metadata=metadata,
fibermap=fibermap,
obs=obs,
header=hdr)
