def write_fibermap(outfile, fibermap, header=None):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: ndarray with named columns of fibermap data
header: header data to include in same HDU as fibermap
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
hdr = fitsheader(header)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile, fibermap, hdr, comments=fibermap_comments,
extname="FIBERMAP", clobber=True)
return outfile
def write_sky(outfile, skymodel, header=None):
"""Write sky model.
Args:
outfile : filename or (night, expid, camera) tuple
skymodel : SkyModel object, with the following attributes
wave : 1D wavelength in vacuum Angstroms
flux : 2D[nspec, nwave] sky flux
ivar : 2D inverse variance of sky flux
mask : 2D mask for sky flux
header : optional fits header data (fits.Header, dict, or list)
"""
outfile = makepath(outfile, 'sky')
#- Convert header to fits.Header if needed
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(skymodel.header)
add_dependencies(hdr)
hx = fits.HDUList()
hdr['EXTNAME'] = ('SKY', 'no dimension')
hx.append( fits.PrimaryHDU(skymodel.flux.astype('f4'), header=hdr) )
hx.append( fits.ImageHDU(skymodel.ivar.astype('f4'), name='IVAR') )
hx.append( fits.CompImageHDU(skymodel.mask, name='MASK') )
hx.append( fits.ImageHDU(skymodel.wave.astype('f4'), name='WAVELENGTH') )
hx.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_fibermap(outfile, fibermap, header=None):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile,
fibermap,
hdr,
comments=fibermap_comments,
extname="FIBERMAP",
clobber=True)
return outfile
def write_fibermap(outfile, fibermap, header=None, clobber=True, extname='FIBERMAP'):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
clobber (bool, optional): overwrite outfile if it exists
extname (str, optional): set the extension name.
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
write_bintable(outfile, fibermap, hdr, comments=fibermap_comments,
extname=extname, clobber=clobber)
return outfile
def write_flux_calibration(outfile, fluxcalib, header=None):
"""Writes flux calibration.
Args:
outfile : output file name
fluxcalib : FluxCalib object
Options:
header : dict-like object of key/value pairs to include in header
"""
hx = fits.HDUList()
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = 'FLUXCALIB'
hdr['BUNIT'] = ('10**+17 cm2 count s / erg',
'i.e. (elec/A) / (1e-17 erg/s/cm2/A)')
hx.append(fits.PrimaryHDU(fluxcalib.calib.astype('f4'), header=hdr))
hx.append(fits.ImageHDU(fluxcalib.ivar.astype('f4'), name='IVAR'))
# hx.append( fits.CompImageHDU(fluxcalib.mask, name='MASK') )
hx.append(fits.ImageHDU(fluxcalib.mask, name='MASK'))
hx.append(fits.ImageHDU(fluxcalib.wave.astype('f4'), name='WAVELENGTH'))
hx[-1].header['BUNIT'] = 'Angstrom'
hx.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_flux_calibration(outfile, fluxcalib, header=None):
"""Writes flux calibration.
Args:
outfile : output file name
fluxcalib : FluxCalib object
Options:
header : dict-like object of key/value pairs to include in header
"""
hx = fits.HDUList()
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = 'FLUXCALIB'
hdr['BUNIT'] = ('(electrons/A) / (erg/s/cm2/A)', 'electrons per flux unit')
hx.append( fits.PrimaryHDU(fluxcalib.calib.astype('f4'), header=hdr) )
hx.append( fits.ImageHDU(fluxcalib.ivar.astype('f4'), name='IVAR') )
hx.append( fits.CompImageHDU(fluxcalib.mask, name='MASK') )
hx.append( fits.ImageHDU(fluxcalib.wave, name='WAVELENGTH') )
hx.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_fiberflat(outfile,fiberflat,header=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
fiberflat: FiberFlat object
header: (optional) dict or fits.Header object to use as HDU 0 header
Returns:
filepath of file that was written
"""
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
if fiberflat.chi2pdf is not None:
hdr['chi2pdf'] = float(fiberflat.chi2pdf)
add_dependencies(hdr)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr))
hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH'))
hdus.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_fiberflat(outfile, fiberflat, header=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
fiberflat: FiberFlat object
header: (optional) dict or fits.Header object to use as HDU 0 header
Returns:
filepath of file that was written
"""
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
if fiberflat.chi2pdf is not None:
hdr['chi2pdf'] = float(fiberflat.chi2pdf)
add_dependencies(hdr)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr))
hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH'))
hdus.writeto(outfile + '.tmp', clobber=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_stdstar_models(norm_modelfile, normalizedFlux, wave, fibers, data,
fibermap, input_frames, header=None):
"""Writes the normalized flux for the best models.
Args:
norm_modelfile : output file path
normalizedFlux : 2D array of flux[nstdstars, nwave]
wave : 1D array of wavelengths[nwave] in Angstroms
fibers : 1D array of fiberids for these spectra
data : meta data table about which templates best fit
fibermap : fibermaps rows for the input standard stars
input_frames : Table with NIGHT, EXPID, CAMERA of input frames used
"""
log = get_logger()
hdr = fitsheader(header)
add_dependencies(hdr)
#- support input Table, np.array, and dict
data = Table(data)
hdr['EXTNAME'] = ('FLUX', '[10**-17 erg/(s cm2 Angstrom)]')
hdr['BUNIT'] = ('10**-17 erg/(s cm2 Angstrom)', 'Flux units')
hdu1=fits.PrimaryHDU(normalizedFlux.astype('f4'), header=hdr)
hdu2 = fits.ImageHDU(wave.astype('f4'))
hdu2.header['EXTNAME'] = ('WAVELENGTH', '[Angstrom]')
hdu2.header['BUNIT'] = ('Angstrom', 'Wavelength units')
hdu3 = fits.ImageHDU(fibers, name='FIBERS')
# metadata
from astropy.io.fits import Column
cols=[]
for k in data.colnames:
if len(data[k].shape)==1 :
cols.append(Column(name=k,format='D',array=data[k]))
tbhdu=fits.BinTableHDU.from_columns(fits.ColDefs(cols), name='METADATA')
hdulist=fits.HDUList([hdu1,hdu2,hdu3,tbhdu])
# add coefficients
if "COEFF" in data.colnames:
hdulist.append(fits.ImageHDU(data["COEFF"],name="COEFF"))
fmhdu = table_to_hdu(Table(fibermap))
fmhdu.name = 'FIBERMAP'
hdulist.append(fmhdu)
inhdu = table_to_hdu(Table(input_frames))
inhdu.name = 'INPUT_FRAMES'
hdulist.append(inhdu)
t0 = time.time()
tmpfile = norm_modelfile+".tmp"
hdulist.writeto(tmpfile, overwrite=True, checksum=True)
os.rename(tmpfile, norm_modelfile)
duration = time.time() - t0
log.info(iotime.format('write', norm_modelfile, duration))
def write_fiberflat(outfile,fiberflat,header=None, fibermap=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
fiberflat: FiberFlat object
Optional:
header: dict or fits.Header object to use as HDU 0 header
fibermap: table to store as FIBERMAP HDU
Returns:
filepath of file that was written
"""
log = get_logger()
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
if fiberflat.chi2pdf is not None:
hdr['chi2pdf'] = float(fiberflat.chi2pdf)
hdr['EXTNAME'] = 'FIBERFLAT'
if 'BUNIT' in hdr:
del hdr['BUNIT']
add_dependencies(hdr)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr))
hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.ImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH'))
if fibermap is None :
fibermap=ff.fibermap
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
hdus[0].header['BUNIT'] = ("","adimensional quantity to divide to flatfield a frame")
hdus["IVAR"].header['BUNIT'] = ("","inverse variance, adimensional")
hdus["MEANSPEC"].header['BUNIT'] = ("electron/Angstrom")
hdus["WAVELENGTH"].header['BUNIT'] = 'Angstrom'
t0 = time.time()
hdus.writeto(outfile+'.tmp', overwrite=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
duration = time.time() - t0
log.info(iotime.format('write', outfile, duration))
return outfile
def write_frame(outfile, frame, header=None, fibermap=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
hdus.append(x)
hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(frame.mask, name='MASK'))
hdus.append(fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH'))
hdus.append(
fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION'))
if fibermap is not None:
hdus.append(fits.BinTableHDU(np.asarray(fibermap), name='FIBERMAP'))
elif frame.fibermap is not None:
hdus.append(
fits.BinTableHDU(np.asarray(frame.fibermap), name='FIBERMAP'))
elif frame.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error(
"You are likely writing a frame without sufficient fiber info")
hdus.writeto(outfile + '.tmp', clobber=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_frame(outfile, frame, header=None, fibermap=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
hdus.append(x)
hdus.append( fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR') )
hdus.append( fits.CompImageHDU(frame.mask, name='MASK') )
hdus.append( fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH') )
hdus.append( fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION' ) )
if fibermap is not None:
hdus.append( fits.BinTableHDU(np.asarray(fibermap), name='FIBERMAP' ) )
elif frame.fibermap is not None:
hdus.append( fits.BinTableHDU(np.asarray(frame.fibermap), name='FIBERMAP' ) )
elif frame.spectrograph is not None:
x.header['FIBERMIN'] = 500*frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error("You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append( fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX' ) )
hdus.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_zbest(filename, brickname, targetids, zfind, zspec=False):
"""Writes zfinder output to ``filename``.
Args:
filename : full path to output file
brickname : brick name, e.g. '1234p5678'
targetids[nspec] : 1D array of target IDs
zfind : subclass of desispec.zfind.ZFindBase with member variables
z, zerr, zwarn, type, subtype
zspec : if True, also write the following member variables from zfind
wave[nwave], flux[nspec, nwave], ivar[nspec, nwave],
model[nspec, nwave]
The first set of variables are each 1D arrays and are written into a
binary table in an HDU with EXTNAME=ZBEST. The zspec=True outputs are
written to 3 additional image HDUs with names WAVELENGTH, FLUX, IVAR,
and MODEL.
"""
dtype = [
('BRICKNAME', (str, 8)),
('TARGETID', np.int64),
('Z', zfind.z.dtype),
('ZERR', zfind.zerr.dtype),
('ZWARN', zfind.zwarn.dtype),
('SPECTYPE', zfind.spectype.dtype),
('SUBTYPE', zfind.subtype.dtype),
]
data = np.empty(zfind.nspec, dtype=dtype)
data['BRICKNAME'] = brickname
data['TARGETID'] = targetids
data['Z'] = zfind.z
data['ZERR'] = zfind.zerr
data['ZWARN'] = zfind.zwarn
data['SPECTYPE'] = zfind.spectype
data['SUBTYPE'] = zfind.subtype
hdus = fits.HDUList()
phdr = fits.Header()
add_dependencies(phdr)
hdus.append(fits.PrimaryHDU(None, header=phdr))
data = desiutil.io.encode_table(data)
data.meta['EXTNAME'] = 'ZBEST'
hdus.append(fits.convenience.table_to_hdu(data))
if zspec:
hdus.append(fits.ImageHDU(zfind.wave.astype('f4'), name='WAVELENGTH'))
hdus.append(fits.ImageHDU(zfind.flux.astype('f4'), name='FLUX'))
hdus.append(fits.ImageHDU(zfind.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.ImageHDU(zfind.model.astype('f4'), name='MODEL'))
hdus.writeto(filename + '.tmp', clobber=True, checksum=True)
os.rename(filename + '.tmp', filename)
def write_zbest(filename, brickname, targetids, zfind, zspec=False):
"""Writes zfinder output to ``filename``.
Args:
filename : full path to output file
brickname : brick name, e.g. '1234p5678'
targetids[nspec] : 1D array of target IDs
zfind : subclass of desispec.zfind.ZFindBase with member variables
z, zerr, zwarn, type, subtype
zspec : if True, also write the following member variables from zfind
wave[nwave], flux[nspec, nwave], ivar[nspec, nwave],
model[nspec, nwave]
The first set of variables are each 1D arrays and are written into a
binary table in an HDU with EXTNAME=ZBEST. The zspec=True outputs are
written to 3 additional image HDUs with names WAVELENGTH, FLUX, IVAR,
and MODEL.
"""
dtype = [
('BRICKNAME', 'S8'),
('TARGETID', np.int64),
('Z', zfind.z.dtype),
('ZERR', zfind.zerr.dtype),
('ZWARN', zfind.zwarn.dtype),
('SPECTYPE', zfind.spectype.dtype),
('SUBTYPE', zfind.subtype.dtype),
]
data = np.empty(zfind.nspec, dtype=dtype)
data['BRICKNAME'] = brickname
data['TARGETID'] = targetids
data['Z'] = zfind.z
data['ZERR'] = zfind.zerr
data['ZWARN'] = zfind.zwarn
data['SPECTYPE'] = zfind.spectype
data['SUBTYPE'] = zfind.subtype
hdus = fits.HDUList()
phdr = fits.Header()
add_dependencies(phdr)
hdus.append(fits.PrimaryHDU(None, header=phdr))
hdus.append(fits.BinTableHDU(data, name='ZBEST', uint=True))
if zspec:
hdus.append(fits.ImageHDU(zfind.wave.astype('f4'), name='WAVELENGTH'))
hdus.append(fits.ImageHDU(zfind.flux.astype('f4'), name='FLUX'))
hdus.append(fits.ImageHDU(zfind.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.ImageHDU(zfind.model.astype('f4'), name='MODEL'))
hdus.writeto(filename+'.tmp', clobber=True, checksum=True)
os.rename(filename+'.tmp', filename)
def write_sky(outfile, skymodel, header=None):
"""Write sky model.
Args:
outfile : filename or (night, expid, camera) tuple
skymodel : SkyModel object, with the following attributes
wave : 1D wavelength in vacuum Angstroms
flux : 2D[nspec, nwave] sky flux
ivar : 2D inverse variance of sky flux
mask : 2D mask for sky flux
stat_ivar : 2D inverse variance of sky flux (statistical only)
header : optional fits header data (fits.Header, dict, or list)
"""
from desiutil.depend import add_dependencies
from .util import fitsheader, makepath
log = get_logger()
outfile = makepath(outfile, 'sky')
#- Convert header to fits.Header if needed
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(skymodel.header)
add_dependencies(hdr)
hx = fits.HDUList()
hdr['EXTNAME'] = ('SKY', 'no dimension')
hx.append( fits.PrimaryHDU(skymodel.flux.astype('f4'), header=hdr) )
hx.append( fits.ImageHDU(skymodel.ivar.astype('f4'), name='IVAR') )
# hx.append( fits.CompImageHDU(skymodel.mask, name='MASK') )
hx.append( fits.ImageHDU(skymodel.mask, name='MASK') )
hx.append( fits.ImageHDU(skymodel.wave.astype('f4'), name='WAVELENGTH') )
if skymodel.stat_ivar is not None :
hx.append( fits.ImageHDU(skymodel.stat_ivar.astype('f4'), name='STATIVAR') )
if skymodel.throughput_corrections is not None:
hx.append( fits.ImageHDU(skymodel.throughput_corrections.astype('f4'), name='THRPUTCORR') )
hx[-1].header['BUNIT'] = 'Angstrom'
t0 = time.time()
hx.writeto(outfile+'.tmp', overwrite=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
duration = time.time() - t0
log.info(iotime.format('write', outfile, duration))
return outfile
def write_fiberflat(outfile, fiberflat, header=None, fibermap=None):
"""Write fiberflat object to outfile
Args:
outfile: filepath string or (night, expid, camera) tuple
fiberflat: FiberFlat object
Optional:
header: dict or fits.Header object to use as HDU 0 header
fibermap: table to store as FIBERMAP HDU
Returns:
filepath of file that was written
"""
outfile = makepath(outfile, 'fiberflat')
if header is None:
hdr = fitsheader(fiberflat.header)
else:
hdr = fitsheader(header)
if fiberflat.chi2pdf is not None:
hdr['chi2pdf'] = float(fiberflat.chi2pdf)
hdr['EXTNAME'] = 'FIBERFLAT'
if 'BUNIT' in hdr:
del hdr['BUNIT']
add_dependencies(hdr)
ff = fiberflat #- shorthand
hdus = fits.HDUList()
hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr))
hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.ImageHDU(ff.mask, name='MASK'))
hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC'))
hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH'))
if fibermap is None:
fibermap = ff.fibermap
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
hdus[-1].header['BUNIT'] = 'Angstrom'
hdus.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_stdstar_models(norm_modelfile,
normalizedFlux,
wave,
fibers,
data,
header=None):
"""Writes the normalized flux for the best models.
Args:
norm_modelfile : output file path
normalizedFlux : 2D array of flux[nstdstars, nwave]
wave : 1D array of wavelengths[nwave] in Angstroms
fibers : 1D array of fiberids for these spectra
data : meta data table about which templates best fit
"""
hdr = fitsheader(header)
add_dependencies(hdr)
#- support input Table, np.array, and dict
data = Table(data)
hdr['EXTNAME'] = ('FLUX', '[10**-17 erg/(s cm2 Angstrom)]')
hdr['BUNIT'] = ('10**-17 erg/(s cm2 Angstrom)', 'Flux units')
hdu1 = fits.PrimaryHDU(normalizedFlux.astype('f4'), header=hdr)
hdu2 = fits.ImageHDU(wave.astype('f4'))
hdu2.header['EXTNAME'] = ('WAVELENGTH', '[Angstrom]')
hdu2.header['BUNIT'] = ('Angstrom', 'Wavelength units')
hdu3 = fits.ImageHDU(fibers, name='FIBERS')
# metadata
from astropy.io.fits import Column
cols = []
for k in data.colnames:
if len(data[k].shape) == 1:
cols.append(Column(name=k, format='D', array=data[k]))
tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs(cols), name='METADATA')
hdulist = fits.HDUList([hdu1, hdu2, hdu3, tbhdu])
# add coefficients
if "COEFF" in data.colnames:
hdulist.append(fits.ImageHDU(data["COEFF"], name="COEFF"))
tmpfile = norm_modelfile + ".tmp"
hdulist.writeto(tmpfile, overwrite=True, checksum=True)
os.rename(tmpfile, norm_modelfile)
def write_image(outfile, image, meta=None):
"""Writes image object to outfile
Args:
outfile : output file string
image : desispec.image.Image object
(or any object with 2D array attributes image, ivar, mask)
Optional:
meta : dict-like object with metadata key/values (e.g. FITS header)
"""
if meta is not None:
hdr = fitsheader(meta)
else:
hdr = fitsheader(image.meta)
add_dependencies(hdr)
outdir = os.path.dirname(os.path.abspath(outfile))
if not os.path.isdir(outdir):
os.makedirs(outdir)
hx = fits.HDUList()
hdu = fits.ImageHDU(image.pix.astype(np.float32), name='IMAGE', header=hdr)
if 'CAMERA' not in hdu.header:
hdu.header.append(
('CAMERA', image.camera.lower(), 'Spectrograph Camera'))
if 'RDNOISE' not in hdu.header and np.isscalar(image.readnoise):
hdu.header.append(
('RDNOISE', image.readnoise, 'Read noise [RMS electrons/pixel]'))
hx.append(hdu)
hx.append(fits.ImageHDU(image.ivar.astype(np.float32), name='IVAR'))
hx.append(fits.CompImageHDU(image.mask.astype(np.int16), name='MASK'))
if not np.isscalar(image.readnoise):
hx.append(
fits.ImageHDU(image.readnoise.astype(np.float32),
name='READNOISE'))
hx.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_stdstar_models(norm_modelfile,
normalizedFlux,
wave,
fibers,
data,
header=None):
"""Writes the normalized flux for the best models.
Args:
norm_modelfile : output file path
normalizedFlux : 2D array of flux[nstdstars, nwave]
wave : 1D array of wavelengths[nwave] in Angstroms
fibers : 1D array of fiberids for these spectra
data : meta data table about which templates best fit; should include
BESTMODEL, TEMPLATEID, CHI2DOF, REDSHIFT
"""
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = ('FLUX', 'erg/s/cm2/A')
hdr['BUNIT'] = ('erg/s/cm2/A', 'Flux units')
hdu1 = fits.PrimaryHDU(normalizedFlux.astype('f4'), header=hdr.copy())
hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]')
hdr['BUNIT'] = ('Angstrom', 'Wavelength units')
hdu2 = fits.ImageHDU(wave.astype('f4'), header=hdr.copy())
hdr['EXTNAME'] = ('FIBERS', 'no dimension')
hdu3 = fits.ImageHDU(fibers, header=hdr.copy())
hdr['EXTNAME'] = ('METADATA', 'no dimension')
from astropy.io.fits import Column
BESTMODEL = Column(name='BESTMODEL', format='K', array=data['BESTMODEL'])
TEMPLATEID = Column(name='TEMPLATEID',
format='K',
array=data['TEMPLATEID'])
CHI2DOF = Column(name='CHI2DOF', format='D', array=data['CHI2DOF'])
REDSHIFT = Column(name='REDSHIFT', format='D', array=data['REDSHIFT'])
cols = fits.ColDefs([BESTMODEL, TEMPLATEID, CHI2DOF, REDSHIFT])
tbhdu = fits.BinTableHDU.from_columns(cols, header=hdr)
hdulist = fits.HDUList([hdu1, hdu2, hdu3, tbhdu])
tmpfile = norm_modelfile + ".tmp"
hdulist.writeto(tmpfile, clobber=True, checksum=True)
os.rename(tmpfile, norm_modelfile)
def write_flux_calibration(outfile, fluxcalib, header=None):
"""Writes flux calibration.
Args:
outfile : output file name
fluxcalib : FluxCalib object
Options:
header : dict-like object of key/value pairs to include in header
"""
log = get_logger()
hx = fits.HDUList()
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = 'FLUXCALIB'
hdr['BUNIT'] = ('10**+17 cm2 count s / erg', 'i.e. (elec/A) / (1e-17 erg/s/cm2/A)')
hx.append( fits.PrimaryHDU(fluxcalib.calib.astype('f4'), header=hdr) )
hx.append( fits.ImageHDU(fluxcalib.ivar.astype('f4'), name='IVAR') )
# hx.append( fits.CompImageHDU(fluxcalib.mask, name='MASK') )
hx.append( fits.ImageHDU(fluxcalib.mask, name='MASK') )
hx.append( fits.ImageHDU(fluxcalib.wave.astype('f4'), name='WAVELENGTH') )
hx[-1].header['BUNIT'] = 'Angstrom'
if fluxcalib.fibercorr is not None :
tbl = encode_table(fluxcalib.fibercorr) #- unicode -> bytes
tbl.meta['EXTNAME'] = 'FIBERCORR'
hx.append( fits.convenience.table_to_hdu(tbl) )
if fluxcalib.fibercorr_comments is not None : # add comments in header
hdu=hx['FIBERCORR']
for i in range(1,999):
key = 'TTYPE'+str(i)
if key in hdu.header:
value = hdu.header[key]
if value in fluxcalib.fibercorr_comments.keys() :
hdu.header[key] = (value, fluxcalib.fibercorr_comments[value])
t0 = time.time()
hx.writeto(outfile+'.tmp', overwrite=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
duration = time.time() - t0
log.info(iotime.format('write', outfile, duration))
return outfile
def write_fibermap(outfile,
fibermap,
header=None,
clobber=True,
extname='FIBERMAP'):
"""Write fibermap binary table to outfile.
Args:
outfile (str): output filename
fibermap: astropy Table of fibermap data
header: header data to include in same HDU as fibermap
clobber (bool, optional): overwrite outfile if it exists
extname (str, optional): set the extension name.
Returns:
write_fibermap (str): full path to filename of fibermap file written.
"""
log = get_logger()
outfile = makepath(outfile)
#- astropy.io.fits incorrectly generates warning about 2D arrays of strings
#- Temporarily turn off warnings to avoid this; desispec.test.test_io will
#- catch it if the arrays actually are written incorrectly.
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(fibermap.meta)
add_dependencies(hdr)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
t0 = time.time()
write_bintable(outfile,
fibermap,
hdr,
comments=fibermap_comments,
extname=extname,
clobber=clobber)
duration = time.time() - t0
log.info(iotime.format('write', outfile, duration))
return outfile
def write_image(outfile, image, meta=None):
"""Writes image object to outfile
Args:
outfile : output file string
image : desispec.image.Image object
(or any object with 2D array attributes image, ivar, mask)
Optional:
meta : dict-like object with metadata key/values (e.g. FITS header)
"""
if meta is not None:
hdr = fitsheader(meta)
else:
hdr = fitsheader(image.meta)
add_dependencies(hdr)
outdir = os.path.dirname(os.path.abspath(outfile))
if not os.path.isdir(outdir):
os.makedirs(outdir)
hx = fits.HDUList()
hdu = fits.ImageHDU(image.pix.astype(np.float32), name='IMAGE', header=hdr)
if 'CAMERA' not in hdu.header:
hdu.header.append( ('CAMERA', image.camera.lower(), 'Spectrograph Camera') )
if 'RDNOISE' not in hdu.header and np.isscalar(image.readnoise):
hdu.header.append( ('RDNOISE', image.readnoise, 'Read noise [RMS electrons/pixel]'))
hx.append(hdu)
hx.append(fits.ImageHDU(image.ivar.astype(np.float32), name='IVAR'))
hx.append(fits.CompImageHDU(image.mask.astype(np.int16), name='MASK'))
if not np.isscalar(image.readnoise):
hx.append(fits.ImageHDU(image.readnoise.astype(np.float32), name='READNOISE'))
hx.writeto(outfile+'.tmp', overwrite=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_stdstar_models(norm_modelfile,normalizedFlux,wave,fibers,data,header=None):
"""Writes the normalized flux for the best models.
Args:
norm_modelfile : output file path
normalizedFlux : 2D array of flux[nstdstars, nwave]
wave : 1D array of wavelengths[nwave] in Angstroms
fibers : 1D array of fiberids for these spectra
data : meta data table about which templates best fit
"""
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = ('FLUX', '[10**-17 erg/(s cm2 Angstrom)]')
hdr['BUNIT'] = ('10**-17 erg/(s cm2 Angstrom)', 'Flux units')
hdu1=fits.PrimaryHDU(normalizedFlux.astype('f4'), header=hdr)
hdu2 = fits.ImageHDU(wave.astype('f4'))
hdu2.header['EXTNAME'] = ('WAVELENGTH', '[Angstrom]')
hdu2.header['BUNIT'] = ('Angstrom', 'Wavelength units')
hdu3 = fits.ImageHDU(fibers, name='FIBERS')
# metadata
from astropy.io.fits import Column
cols=[]
for k in data.keys() :
if len(data[k].shape)==1 :
cols.append(Column(name=k,format='D',array=data[k]))
tbhdu=fits.BinTableHDU.from_columns(fits.ColDefs(cols), name='METADATA')
hdulist=fits.HDUList([hdu1,hdu2,hdu3,tbhdu])
# add coefficients
if "COEFF" in data :
hdulist.append(fits.ImageHDU(data["COEFF"],name="COEFF"))
tmpfile = norm_modelfile+".tmp"
hdulist.writeto(tmpfile, overwrite=True, checksum=True)
os.rename(tmpfile, norm_modelfile)
def write_stdstar_models(norm_modelfile,normalizedFlux,wave,fibers,data,header=None):
"""Writes the normalized flux for the best models.
Args:
norm_modelfile : output file path
normalizedFlux : 2D array of flux[nstdstars, nwave]
wave : 1D array of wavelengths[nwave] in Angstroms
fibers : 1D array of fiberids for these spectra
data : meta data table about which templates best fit; should include
BESTMODEL, TEMPLATEID, CHI2DOF, REDSHIFT
"""
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = ('FLUX', 'erg/s/cm2/A')
hdr['BUNIT'] = ('erg/s/cm2/A', 'Flux units')
hdu1=fits.PrimaryHDU(normalizedFlux.astype('f4'), header=hdr.copy())
hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]')
hdr['BUNIT'] = ('Angstrom', 'Wavelength units')
hdu2 = fits.ImageHDU(wave.astype('f4'), header=hdr.copy())
hdr['EXTNAME'] = ('FIBERS', 'no dimension')
hdu3 = fits.ImageHDU(fibers, header=hdr.copy())
hdr['EXTNAME'] = ('METADATA', 'no dimension')
from astropy.io.fits import Column
BESTMODEL=Column(name='BESTMODEL',format='K',array=data['BESTMODEL'])
TEMPLATEID=Column(name='TEMPLATEID',format='K',array=data['TEMPLATEID'])
CHI2DOF=Column(name='CHI2DOF',format='D',array=data['CHI2DOF'])
REDSHIFT=Column(name='REDSHIFT',format='D',array=data['REDSHIFT'])
cols=fits.ColDefs([BESTMODEL,TEMPLATEID,CHI2DOF,REDSHIFT])
tbhdu=fits.BinTableHDU.from_columns(cols,header=hdr)
hdulist=fits.HDUList([hdu1,hdu2,hdu3,tbhdu])
tmpfile = norm_modelfile+".tmp"
hdulist.writeto(tmpfile, clobber=True, checksum=True)
os.rename(tmpfile, norm_modelfile)
def write_qframe(outfile, qframe, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
qframe: desispec.qproc.QFrame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a QFrame object to pass into write_qframe,
qframe = QFrame(wave, flux, ivar)
"""
log = get_logger()
outfile = makepath(outfile, 'qframe')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(qframe.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(qframe.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append( fits.ImageHDU(qframe.ivar.astype('f4'), name='IVAR') )
if qframe.mask is None :
qframe.mask=np.zeros(qframe.flux.shape,dtype=np.uint32)
# hdus.append( fits.CompImageHDU(qframe.mask, name='MASK') )
hdus.append( fits.ImageHDU(qframe.mask, name='MASK') )
if qframe.sigma is None :
qframe.sigma=np.zeros(qframe.flux.shape,dtype=np.float)
hdus.append( fits.ImageHDU(qframe.sigma.astype('f4'), name='YSIGMA') )
hdus.append( fits.ImageHDU(qframe.wave.astype('f8'), name='WAVELENGTH') )
hdus[-1].header['BUNIT'] = 'Angstrom'
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
elif qframe.fibermap is not None:
fibermap = encode_table(qframe.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
elif qframe.spectrograph is not None:
x.header['FIBERMIN'] = 500*qframe.spectrograph # Hard-coded (as in desispec.qproc.qframe)
else:
log.error("You are likely writing a qframe without sufficient fiber info")
hdus.writeto(outfile+'.tmp', clobber=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def write_frame(outfile, frame, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
log = get_logger()
outfile = makepath(outfile, 'frame')
#- Ignore some known and harmless units warnings
import warnings
warnings.filterwarnings(
'ignore', message="'.*nanomaggies.* did not parse as fits unit.*")
warnings.filterwarnings(
'ignore', message=".*'10\*\*6 arcsec.* did not parse as fits unit.*")
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
# Vette
diagnosis = frame.vet()
if diagnosis != 0:
raise IOError(
"Frame did not pass simple vetting test. diagnosis={:d}".format(
diagnosis))
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR'))
# hdus.append( fits.CompImageHDU(frame.mask, name='MASK') )
hdus.append(fits.ImageHDU(frame.mask, name='MASK'))
hdus.append(fits.ImageHDU(frame.wave.astype('f8'), name='WAVELENGTH'))
hdus[-1].header['BUNIT'] = 'Angstrom'
if frame.resolution_data is not None:
hdus.append(
fits.ImageHDU(frame.resolution_data.astype('f4'),
name='RESOLUTION'))
elif frame.wsigma is not None:
log.debug("Using ysigma from qproc")
qrimg = fits.ImageHDU(frame.wsigma.astype('f4'), name='YSIGMA')
qrimg.header["NDIAG"] = frame.ndiag
hdus.append(qrimg)
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.fibermap is not None:
fibermap = encode_table(frame.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error(
"You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append(fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX'))
if frame.scores is not None:
scores_tbl = encode_table(frame.scores) #- unicode -> bytes
scores_tbl.meta['EXTNAME'] = 'SCORES'
hdus.append(fits.convenience.table_to_hdu(scores_tbl))
if frame.scores_comments is not None: # add comments in header
hdu = hdus['SCORES']
for i in range(1, 999):
key = 'TTYPE' + str(i)
if key in hdu.header:
value = hdu.header[key]
if value in frame.scores_comments.keys():
hdu.header[key] = (value, frame.scores_comments[value])
hdus.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def __init__(self, path, mode='readonly', header=None):
from .util import fitsheader
from .fibermap import fibermap_columns, fibermap_comments
if mode not in ('readonly', 'update'):
raise RuntimeError('Invalid mode %r' % mode)
self.path = path
self.mode = mode
# Create a new file if necessary.
if self.mode == 'update' and not os.path.exists(self.path):
# BRICKNAM must be in header if creating the file for the first time
if header is None or 'BRICKNAM' not in header:
raise ValueError(
'header must have BRICKNAM when creating new brick file')
self.brickname = header['BRICKNAM']
if 'CHANNEL' in header:
self.channel = header['CHANNEL']
else:
self.channel = 'brz' #- could be any spectrograph channel
# Create the parent directory, if necessary.
head, tail = os.path.split(os.path.abspath(self.path))
if not os.path.exists(head):
os.makedirs(head)
# Create empty HDUs. It would be good to refactor io.frame to avoid any duplication here.
hdr = fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = ('FLUX', '1e-17 erg/(s cm2 Angstrom)')
hdr['BUNIT'] = '1e-17 erg/(s cm2 Angstrom)'
hdu0 = astropy.io.fits.PrimaryHDU(header=hdr)
hdu1 = astropy.io.fits.ImageHDU(name='IVAR')
hdu2 = astropy.io.fits.ImageHDU(name='WAVELENGTH')
hdu2.header['BUNIT'] = 'Angstrom'
hdu3 = astropy.io.fits.ImageHDU(name='RESOLUTION')
# Create an HDU4 using the columns from fibermap with a few extras added.
columns = fibermap_columns[:]
columns.extend([
('NIGHT', 'i4'),
('EXPID', 'i4'),
('INDEX', 'i4'),
])
data = np.empty(shape=(0, ), dtype=columns)
data = encode_table(data) #- unicode -> bytes
data.meta['EXTNAME'] = 'FIBERMAP'
for key, value in header.items():
data.meta[key] = value
hdu4 = astropy.io.fits.convenience.table_to_hdu(data)
# Add comments for fibermap columns.
num_fibermap_columns = len(fibermap_comments)
for i in range(1, 1 + num_fibermap_columns):
key = 'TTYPE%d' % i
name = hdu4.header[key]
comment = fibermap_comments[name]
hdu4.header[key] = (name, comment)
# Add comments for our additional columns.
hdu4.header['TTYPE%d' % (1 + num_fibermap_columns)] = (
'NIGHT', 'Night of exposure YYYYMMDD')
hdu4.header['TTYPE%d' %
(2 + num_fibermap_columns)] = ('EXPID', 'Exposure ID')
hdu4.header['TTYPE%d' % (3 + num_fibermap_columns)] = (
'INDEX', 'Index of this object in other HDUs')
self.hdu_list = astropy.io.fits.HDUList(
[hdu0, hdu1, hdu2, hdu3, hdu4])
else:
self.hdu_list = astropy.io.fits.open(path, mode=self.mode)
try:
self.brickname = self.hdu_list[0].header['BRICKNAM']
self.channel = self.hdu_list[0].header['CHANNEL']
except KeyError:
self.channel, self.brickname = _parse_brick_filename(path)
def main_mpi(args, comm=None, timing=None):
freeze_iers()
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
mark_start = time.time()
log = get_logger()
psf_file = args.psf
input_file = args.input
# these parameters are interpreted as the *global* spec range,
# to be divided among processes.
specmin = args.specmin
nspec = args.nspec
#- Load input files and broadcast
# FIXME: after we have fixed the serialization
# of the PSF, read and broadcast here, to reduce
# disk contention.
img = None
if rank == 0:
img = io.read_image(input_file)
if comm is not None:
img = comm.bcast(img, root=0)
psf = load_psf(psf_file)
mark_read_input = time.time()
# get spectral range
if nspec is None:
nspec = psf.nspec
if args.fibermap is not None:
fibermap = io.read_fibermap(args.fibermap)
else:
try:
fibermap = io.read_fibermap(args.input)
except (AttributeError, IOError, KeyError):
fibermap = None
if fibermap is not None:
fibermap = fibermap[specmin:specmin + nspec]
if nspec > len(fibermap):
log.warning(
"nspec {} > len(fibermap) {}; reducing nspec to {}".format(
nspec, len(fibermap), len(fibermap)))
nspec = len(fibermap)
fibers = fibermap['FIBER']
else:
fibers = np.arange(specmin, specmin + nspec)
specmax = specmin + nspec
#- Get wavelength grid from options
if args.wavelength is not None:
raw_wstart, raw_wstop, raw_dw = [
float(tmp) for tmp in args.wavelength.split(',')
]
else:
raw_wstart = np.ceil(psf.wmin_all)
raw_wstop = np.floor(psf.wmax_all)
raw_dw = 0.7
raw_wave = np.arange(raw_wstart, raw_wstop + raw_dw / 2.0, raw_dw)
nwave = len(raw_wave)
bundlesize = args.bundlesize
if args.barycentric_correction:
if ('RA' in img.meta) or ('TARGTRA' in img.meta):
barycentric_correction_factor = \
barycentric_correction_multiplicative_factor(img.meta)
#- Early commissioning has RA/TARGTRA in fibermap but not HDU 0
elif fibermap is not None and \
(('RA' in fibermap.meta) or ('TARGTRA' in fibermap.meta)):
barycentric_correction_factor = \
barycentric_correction_multiplicative_factor(fibermap.meta)
else:
msg = 'Barycentric corr requires (TARGT)RA in HDU 0 or fibermap'
log.critical(msg)
raise KeyError(msg)
else:
barycentric_correction_factor = 1.
# Explictly define the correct wavelength values to avoid confusion of reference frame
# If correction applied, otherwise divide by 1 and use the same raw values
wstart = raw_wstart / barycentric_correction_factor
wstop = raw_wstop / barycentric_correction_factor
dw = raw_dw / barycentric_correction_factor
wave = raw_wave / barycentric_correction_factor
#- Confirm that this PSF covers these wavelengths for these spectra
psf_wavemin = np.max(psf.wavelength(list(range(specmin, specmax)), y=-0.5))
psf_wavemax = np.min(
psf.wavelength(list(range(specmin, specmax)), y=psf.npix_y - 0.5))
if psf_wavemin - 5 > wstart:
raise ValueError(
'Start wavelength {:.2f} < min wavelength {:.2f} for these fibers'.
format(wstart, psf_wavemin))
if psf_wavemax + 5 < wstop:
raise ValueError(
'Stop wavelength {:.2f} > max wavelength {:.2f} for these fibers'.
format(wstop, psf_wavemax))
if rank == 0:
#- Print parameters
log.info("extract: input = {}".format(input_file))
log.info("extract: psf = {}".format(psf_file))
log.info("extract: specmin = {}".format(specmin))
log.info("extract: nspec = {}".format(nspec))
log.info("extract: wavelength = {},{},{}".format(wstart, wstop, dw))
log.info("extract: nwavestep = {}".format(args.nwavestep))
log.info("extract: regularize = {}".format(args.regularize))
if barycentric_correction_factor != 1.:
img.meta['HELIOCOR'] = barycentric_correction_factor
#- Augment input image header for output
img.meta['NSPEC'] = (nspec, 'Number of spectra')
img.meta['WAVEMIN'] = (raw_wstart, 'First wavelength [Angstroms]')
img.meta['WAVEMAX'] = (raw_wstop, 'Last wavelength [Angstroms]')
img.meta['WAVESTEP'] = (raw_dw, 'Wavelength step size [Angstroms]')
img.meta['SPECTER'] = (specter.__version__,
'https://github.com/desihub/specter')
img.meta['IN_PSF'] = (io.shorten_filename(psf_file), 'Input spectral PSF')
img.meta['IN_IMG'] = io.shorten_filename(input_file)
depend.add_dependencies(img.meta)
#- Check if input PSF was itself a traceshifted version of another PSF
orig_psf = None
if rank == 0:
try:
psfhdr = fits.getheader(psf_file, 'PSF')
orig_psf = psfhdr['IN_PSF']
except KeyError:
#- could happen due to PSF format not having "PSF" extension,
#- or due to PSF header not having 'IN_PSF' keyword. Either is OK
pass
if comm is not None:
orig_psf = comm.bcast(orig_psf, root=0)
if orig_psf is not None:
img.meta['ORIG_PSF'] = orig_psf
#- If not using MPI, use a single call to each of these and then end this function call
# Otherwise, continue on to splitting things up for the different ranks
if comm is None:
_extract_and_save(img, psf, specmin, nspec, specmin, wave, raw_wave,
fibers, fibermap, args.output, args.model,
bundlesize, args, log)
#- This is it if we aren't running MPI, so return
return
#else:
# # Continue to the MPI section, which could go under this else statment
# # But to save on indentation we'll just pass on to the rest of the function
# # since the alternative has already returned
# pass
# Now we divide our spectra into bundles
checkbundles = set()
checkbundles.update(
np.floor_divide(np.arange(specmin, specmax),
bundlesize * np.ones(nspec)).astype(int))
bundles = sorted(checkbundles)
nbundle = len(bundles)
bspecmin = {}
bnspec = {}
for b in bundles:
if specmin > b * bundlesize:
bspecmin[b] = specmin
else:
bspecmin[b] = b * bundlesize
if (b + 1) * bundlesize > specmax:
bnspec[b] = specmax - bspecmin[b]
else:
bnspec[b] = bundlesize
# Now we assign bundles to processes
mynbundle = int(nbundle // nproc)
myfirstbundle = 0
leftover = nbundle % nproc
if rank < leftover:
mynbundle += 1
myfirstbundle = rank * mynbundle
else:
myfirstbundle = ((mynbundle + 1) * leftover) + (mynbundle *
(rank - leftover))
# get the root output file
outpat = re.compile(r'(.*)\.fits')
outmat = outpat.match(args.output)
if outmat is None:
raise RuntimeError(
"extraction output file should have .fits extension")
outroot = outmat.group(1)
outdir = os.path.normpath(os.path.dirname(outroot))
if rank == 0:
if not os.path.isdir(outdir):
os.makedirs(outdir)
if comm is not None:
comm.barrier()
mark_preparation = time.time()
time_total_extraction = 0.0
time_total_write_output = 0.0
failcount = 0
for b in range(myfirstbundle, myfirstbundle + mynbundle):
mark_iteration_start = time.time()
outbundle = "{}_{:02d}.fits".format(outroot, b)
outmodel = "{}_model_{:02d}.fits".format(outroot, b)
log.info('extract: Rank {} extracting {} spectra {}:{} at {}'.format(
rank,
os.path.basename(input_file),
bspecmin[b],
bspecmin[b] + bnspec[b],
time.asctime(),
))
sys.stdout.flush()
#- The actual extraction
try:
mark_extraction = _extract_and_save(img, psf, bspecmin[b],
bnspec[b], specmin, wave,
raw_wave, fibers, fibermap,
outbundle, outmodel,
bundlesize, args, log)
mark_write_output = time.time()
time_total_extraction += mark_extraction - mark_iteration_start
time_total_write_output += mark_write_output - mark_extraction
except:
# Log the error and increment the number of failures
log.error(
"extract: FAILED bundle {}, spectrum range {}:{}".format(
b, bspecmin[b], bspecmin[b] + bnspec[b]))
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
log.error(''.join(lines))
failcount += 1
sys.stdout.flush()
if comm is not None:
failcount = comm.allreduce(failcount)
if failcount > 0:
# all processes throw
raise RuntimeError("some extraction bundles failed")
time_merge = None
if rank == 0:
mark_merge_start = time.time()
mergeopts = ['--output', args.output, '--force', '--delete']
mergeopts.extend(
["{}_{:02d}.fits".format(outroot, b) for b in bundles])
mergeargs = mergebundles.parse(mergeopts)
mergebundles.main(mergeargs)
if args.model is not None:
model = None
for b in bundles:
outmodel = "{}_model_{:02d}.fits".format(outroot, b)
if model is None:
model = fits.getdata(outmodel)
else:
#- TODO: test and warn if models overlap for pixels with
#- non-zero values
model += fits.getdata(outmodel)
os.remove(outmodel)
fits.writeto(args.model, model)
mark_merge_end = time.time()
time_merge = mark_merge_end - mark_merge_start
# Resolve difference timer data
if type(timing) is dict:
timing["read_input"] = mark_read_input - mark_start
timing["preparation"] = mark_preparation - mark_read_input
timing["total_extraction"] = time_total_extraction
timing["total_write_output"] = time_total_write_output
timing["merge"] = time_merge
def write_qframe(outfile, qframe, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
qframe: desispec.qproc.QFrame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a QFrame object to pass into write_qframe,
qframe = QFrame(wave, flux, ivar)
"""
log = get_logger()
outfile = makepath(outfile, 'qframe')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(qframe.meta)
add_dependencies(hdr)
hdus = fits.HDUList()
x = fits.PrimaryHDU(qframe.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append(fits.ImageHDU(qframe.ivar.astype('f4'), name='IVAR'))
if qframe.mask is None:
qframe.mask = np.zeros(qframe.flux.shape, dtype=np.uint32)
# hdus.append( fits.CompImageHDU(qframe.mask, name='MASK') )
hdus.append(fits.ImageHDU(qframe.mask, name='MASK'))
if qframe.sigma is None:
qframe.sigma = np.zeros(qframe.flux.shape, dtype=np.float)
hdus.append(fits.ImageHDU(qframe.sigma.astype('f4'), name='YSIGMA'))
hdus.append(fits.ImageHDU(qframe.wave.astype('f8'), name='WAVELENGTH'))
hdus[-1].header['BUNIT'] = 'Angstrom'
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif qframe.fibermap is not None:
fibermap = encode_table(qframe.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif qframe.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * qframe.spectrograph # Hard-coded (as in desispec.qproc.qframe)
else:
log.error(
"You are likely writing a qframe without sufficient fiber info")
raise ValueError('no fibermap')
hdus.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_spectra(outfile, spec, units=None):
"""
Write Spectra object to FITS file.
This places the metadata into the header of the (empty) primary HDU.
The first extension contains the fibermap, and then HDUs are created for
the different data arrays for each band.
Floating point data is converted to 32 bits before writing.
Args:
outfile (str): path to write
spec (Spectra): the object containing the data
units (str): optional string to use for the BUNIT key of the flux
HDUs for each band.
Returns:
The absolute path to the file that was written.
"""
outfile = os.path.abspath(outfile)
# Create the parent directory, if necessary.
dir, base = os.path.split(outfile)
if not os.path.exists(dir):
os.makedirs(dir)
# Create HDUs from the data
all_hdus = fits.HDUList()
# metadata goes in empty primary HDU
hdr = fitsheader(spec.meta)
add_dependencies(hdr)
all_hdus.append(fits.PrimaryHDU(header=hdr))
# Next is the fibermap
fmap = spec.fibermap.copy()
fmap.meta["EXTNAME"] = "FIBERMAP"
hdu = fits.convenience.table_to_hdu(fmap)
# Add comments for fibermap columns.
for i, colname in enumerate(fmap.dtype.names):
if colname in fibermap_comments:
key = "TTYPE{}".format(i+1)
name = hdu.header[key]
assert name == colname
comment = fibermap_comments[name]
hdu.header[key] = (name, comment)
else:
print('Unknown comment for {}'.format(colname))
all_hdus.append(hdu)
# Now append the data for all bands
for band in spec.bands:
hdu = fits.ImageHDU(name="{}_WAVELENGTH".format(band.upper()))
hdu.header["BUNIT"] = "Angstrom"
hdu.data = spec.wave[band].astype("f8")
all_hdus.append(hdu)
hdu = fits.ImageHDU(name="{}_FLUX".format(band.upper()))
if units is None:
hdu.header["BUNIT"] = "10**-17 erg/(s cm2 Angstrom)"
else:
hdu.header["BUNIT"] = units
hdu.data = spec.flux[band].astype("f4")
all_hdus.append(hdu)
hdu = fits.ImageHDU(name="{}_IVAR".format(band.upper()))
if units is None:
hdu.header["BUNIT"] = '10**+34 (s2 cm4 Angstrom2) / erg2'
else:
hdu.header["BUNIT"] = ((u.Unit(units, format='fits'))**-2).to_string('fits')
hdu.data = spec.ivar[band].astype("f4")
all_hdus.append(hdu)
if spec.mask is not None:
# hdu = fits.CompImageHDU(name="{}_MASK".format(band.upper()))
hdu = fits.ImageHDU(name="{}_MASK".format(band.upper()))
hdu.data = spec.mask[band].astype(np.uint32)
all_hdus.append(hdu)
if spec.resolution_data is not None:
hdu = fits.ImageHDU(name="{}_RESOLUTION".format(band.upper()))
hdu.data = spec.resolution_data[band].astype("f4")
all_hdus.append(hdu)
if spec.extra is not None:
for ex in spec.extra[band].items():
hdu = fits.ImageHDU(name="{}_{}".format(band.upper(), ex[0]))
hdu.data = ex[1].astype("f4")
all_hdus.append(hdu)
all_hdus.writeto("{}.tmp".format(outfile), overwrite=True, checksum=True)
os.rename("{}.tmp".format(outfile), outfile)
return outfile
def write_frame(outfile, frame, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
log = get_logger()
outfile = makepath(outfile, 'frame')
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
# Vette
diagnosis = frame.vet()
if diagnosis != 0:
raise IOError(
"Frame did not pass simple vetting test. diagnosis={:d}".format(
diagnosis))
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warn('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR'))
hdus.append(fits.CompImageHDU(frame.mask, name='MASK'))
hdus.append(fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH'))
hdus[-1].header['BUNIT'] = 'Angstrom'
hdus.append(
fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION'))
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.fibermap is not None:
fibermap = encode_table(frame.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error(
"You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append(fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX'))
hdus.writeto(outfile + '.tmp', clobber=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_raw(filename, rawdata, header, camera=None, primary_header=None):
'''
Write raw pixel data to a DESI raw data file
Args:
filename : file name to write data; if this exists, append a new HDU
rawdata : 2D ndarray of raw pixel data including overscans
header : dict-like object or fits.Header with keywords
CCDSECx, BIASSECx, DATASECx where x=1,2,3, or 4
Options:
camera : b0, r1 .. z9 - override value in header
primary_header : header to write in HDU0 if filename doesn't yet exist
The primary utility of this function over raw fits calls is to ensure
that all necessary keywords are present before writing the file.
CCDSECx, BIASSECx, DATASECx where x=1,2,3, or 4
DATE-OBS, GAINx and RDNOISEx will generate a non-fatal warning if missing
'''
log = get_logger()
header = desispec.io.util.fitsheader(header)
primary_header = desispec.io.util.fitsheader(primary_header)
if rawdata.dtype not in (np.int16, np.int32, np.int64):
message = 'dtype {} not supported for raw data'.format(rawdata.dtype)
log.fatal(message)
raise ValueError(message)
fail_message = ''
for required_key in ['DOSVER', 'FEEVER', 'DETECTOR']:
if required_key not in primary_header:
if required_key in header:
primary_header[required_key] = header[required_key]
else:
fail_message = fail_message + \
'Keyword {} must be in header or primary_header\n'.format(required_key)
if fail_message != '':
raise ValueError(fail_message)
#- Check required keywords before writing anything
missing_keywords = list()
if camera is None and 'CAMERA' not in header:
log.error("Must provide camera keyword or header['CAMERA']")
missing_keywords.append('CAMERA')
for amp in ['1', '2', '3', '4']:
for prefix in ['CCDSEC', 'BIASSEC', 'DATASEC']:
keyword = prefix + amp
if keyword not in header:
log.error('Missing keyword ' + keyword)
missing_keywords.append(keyword)
#- Missing DATE-OBS is warning but not error
if 'DATE-OBS' not in primary_header:
if 'DATE-OBS' in header:
primary_header['DATE-OBS'] = header['DATE-OBS']
else:
log.warning('missing keyword DATE-OBS')
#- Missing GAINx is warning but not error
for amp in ['1', '2', '3', '4']:
keyword = 'GAIN' + amp
if keyword not in header:
log.warning('Gain keyword {} missing; using 1.0'.format(keyword))
header[keyword] = 1.0
#- Missing RDNOISEx is warning but not error
for amp in ['1', '2', '3', '4']:
keyword = 'RDNOISE' + amp
if keyword not in header:
log.warning('Readnoise keyword {} missing'.format(keyword))
#- Stop if any keywords are missing
if len(missing_keywords) > 0:
raise KeyError('missing required keywords {}'.format(missing_keywords))
#- Set EXTNAME=camera
if camera is not None:
header['CAMERA'] = camera.lower()
extname = camera.upper()
else:
if header['CAMERA'] != header['CAMERA'].lower():
log.warning('Converting CAMERA {} to lowercase'.format(
header['CAMERA']))
header['CAMERA'] = header['CAMERA'].lower()
extname = header['CAMERA'].upper()
header['INHERIT'] = True
#- fits.CompImageHDU doesn't know how to fill in default keywords, so
#- temporarily generate an uncompressed HDU to get those keywords
header = fits.ImageHDU(rawdata, header=header, name=extname).header
#- Bizarrely, compression of 64-bit integers isn't supported.
#- downcast to 32-bit if that won't lose precision.
#- Real raw data should be 32-bit or 16-bit anyway
if rawdata.dtype == np.int64:
if np.max(np.abs(rawdata)) < 2**31:
rawdata = rawdata.astype(np.int32)
if rawdata.dtype in (np.int16, np.int32):
dataHDU = fits.CompImageHDU(rawdata, header=header, name=extname)
elif rawdata.dtype == np.int64:
log.warning(
'Image compression not supported for 64-bit; writing uncompressed')
dataHDU = fits.ImageHDU(rawdata, header=header, name=extname)
else:
log.error("How did we get this far with rawdata dtype {}?".format(
rawdata.dtype))
dataHDU = fits.ImageHDU(rawdata, header=header, name=extname)
#- Actually write or update the file
if os.path.exists(filename):
hdus = fits.open(filename, mode='append', memmap=False)
if extname in hdus:
hdus.close()
raise ValueError('Camera {} already in {}'.format(
camera, filename))
else:
hdus.append(dataHDU)
hdus.flush()
hdus.close()
else:
hdus = fits.HDUList()
add_dependencies(primary_header)
hdus.append(fits.PrimaryHDU(None, header=primary_header))
hdus.append(dataHDU)
hdus.writeto(filename)
def write_spectra(outfile, spec, units=None):
"""
Write Spectra object to FITS file.
This places the metadata into the header of the (empty) primary HDU.
The first extension contains the fibermap, and then HDUs are created for
the different data arrays for each band.
Floating point data is converted to 32 bits before writing.
Args:
outfile (str): path to write
spec (Spectra): the object containing the data
units (str): optional string to use for the BUNIT key of the flux
HDUs for each band.
Returns:
The absolute path to the file that was written.
"""
outfile = os.path.abspath(outfile)
# Create the parent directory, if necessary.
dir, base = os.path.split(outfile)
if not os.path.exists(dir):
os.makedirs(dir)
# Create HDUs from the data
all_hdus = fits.HDUList()
# metadata goes in empty primary HDU
hdr = fitsheader(spec.meta)
add_dependencies(hdr)
all_hdus.append(fits.PrimaryHDU(header=hdr))
# Next is the fibermap
fmap = spec.fibermap.copy()
fmap.meta["EXTNAME"] = "FIBERMAP"
hdu = fits.convenience.table_to_hdu(fmap)
# Add comments for fibermap columns.
comments = spectra_comments()
num_columns = len(comments)
for i in range(1, 1 + num_columns):
key = "TTYPE{}".format(i)
name = hdu.header[key]
comment = comments[name]
hdu.header[key] = (name, comment)
all_hdus.append(hdu)
# Now append the data for all bands
for band in spec.bands:
hdu = fits.ImageHDU(name="{}_WAVELENGTH".format(band.upper()))
hdu.header["BUNIT"] = "Angstrom"
hdu.data = spec.wave[band].astype("f4")
all_hdus.append(hdu)
hdu = fits.ImageHDU(name="{}_FLUX".format(band.upper()))
if units is None:
hdu.header["BUNIT"] = "1e-17 erg/(s cm2 Angstrom)"
else:
hdu.header["BUNIT"] = units
hdu.data = spec.flux[band].astype("f4")
all_hdus.append(hdu)
hdu = fits.ImageHDU(name="{}_IVAR".format(band.upper()))
hdu.data = spec.ivar[band].astype("f4")
all_hdus.append(hdu)
if spec.mask is not None:
hdu = fits.CompImageHDU(name="{}_MASK".format(band.upper()))
hdu.data = spec.mask[band].astype(np.uint32)
all_hdus.append(hdu)
if spec.resolution_data is not None:
hdu = fits.ImageHDU(name="{}_RESOLUTION".format(band.upper()))
hdu.data = spec.resolution_data[band].astype("f4")
all_hdus.append(hdu)
if spec.extra is not None:
for ex in spec.extra[band].items():
hdu = fits.ImageHDU(name="{}_{}".format(band.upper(), ex[0]))
hdu.data = ex[1].astype("f4")
all_hdus.append(hdu)
try:
all_hdus.writeto("{}.tmp".format(outfile),
overwrite=True,
checksum=True)
except TypeError:
all_hdus.writeto("{}.tmp".format(outfile), clobber=True, checksum=True)
os.rename("{}.tmp".format(outfile), outfile)
return outfile
def write_assignment_fits_tile(asgn, fulltarget, overwrite, params):
"""Write a single tile assignment to a FITS file.
Args:
outroot (str): full path of the output root file name.
asgn (Assignment): the assignment class instance.
fulltarget (bool): if True, dump the target information for all
available targets, not just the ones that are assigned.
overwrite (bool): overwrite output files or not
params (tuple): tuple containing the tile ID, RA, DEC, rotation,
output path, and GFA targets
Returns:
None
"""
tm = Timer()
tm.start()
tile_id, tile_ra, tile_dec, tile_obstheta, tile_obstime, tile_obsha, \
tile_file, gfa_targets = params
log = Logger.get()
# Hardware properties
hw = asgn.hardware()
# Targets available for all tile / locs
tgsavail = asgn.targets_avail()
# Target properties
tgs = asgn.targets()
# Data for this tile
tdata = asgn.tile_location_target(tile_id)
avail = tgsavail.tile_data(tile_id)
# The recarray dtypes
assign_dtype = np.dtype([(x, y)
for x, y in results_assign_columns.items()])
targets_dtype = np.dtype([(x, y)
for x, y in results_targets_columns.items()])
avail_dtype = np.dtype([(x, y) for x, y in results_avail_columns.items()])
# tm.stop()
# tm.report(" data pointers for tile {}".format(tile_id))
# tm.clear()
# tm.start()
locs = np.array(hw.locations)
nloc = len(locs)
# Compute the total list of targets
navail = np.sum([len(avail[x]) for x in avail.keys()])
tgids_avail = np.unique(
np.concatenate(
[np.array(avail[x], dtype=np.int64) for x in avail.keys()]))
# tm.stop()
# tm.report(" available targets tile {}".format(tile_id))
if len(tgids_avail) > 0:
# We have some available targets.
# tm.clear()
# tm.start()
if os.path.isfile(tile_file):
if overwrite:
log.warning("Overwriting {}".format(tile_file))
os.remove(tile_file)
else:
raise RuntimeError(
"output file {} already exists".format(tile_file))
# This tile has some available targets
log.info("Writing tile {}".format(tile_id))
tmp_file = tile_file + ".tmp"
fd = fitsio.FITS(tmp_file, "rw")
# tm.stop()
# tm.report(" opening file for tile {}".format(tile_id))
# tm.clear()
# tm.start()
# Unpack all our target properties from C++ objects into numpy arrays
tgids = None
if fulltarget:
# We are dumping all targets
tgids = tgids_avail
else:
# We are dumping only assigned targets
tgids = np.array(sorted([y for x, y in tdata.items()]),
dtype=np.int64)
ntarget = len(tgids)
tg_ra = np.empty(ntarget, dtype=np.float64)
tg_dec = np.empty(ntarget, dtype=np.float64)
tg_bits = np.zeros(ntarget, dtype=np.int64)
tg_x = np.empty(ntarget, dtype=np.float64)
tg_y = np.empty(ntarget, dtype=np.float64)
tg_type = np.empty(ntarget, dtype=np.uint8)
tg_priority = np.empty(ntarget, dtype=np.int32)
tg_subpriority = np.empty(ntarget, dtype=np.float64)
tg_obscond = np.empty(ntarget, dtype=np.int32)
tg_indx = dict()
for indx, tg in enumerate(tgids):
tg_indx[tg] = indx
props = tgs.get(tg)
tg_ra[indx] = props.ra
tg_dec[indx] = props.dec
tg_bits[indx] = props.bits
tg_type[indx] = props.type
tg_priority[indx] = props.priority
tg_subpriority[indx] = props.subpriority
tg_obscond[indx] = props.obscond
# We compute the X / Y focalplane coordinates for ALL available
# targets, not just the assigned ones. This allows us to write out
# the focalplane coordinates of all available targets as computed by
# the code at the time it was run.
#
# NOTE: The output format is explicitly CS5 coordinates, even though
# we use curved focal surface internally.
xy = hw.radec2xy_multi(tile_ra, tile_dec, tile_obstheta, tg_ra, tg_dec,
True, 0)
for indx, fxy in enumerate(xy):
tg_x[indx] = fxy[0]
tg_y[indx] = fxy[1]
# tm.stop()
# tm.report(" extract target props for {}".format(tile_id))
# tm.clear()
# tm.start()
# Write a header-only HDU 0 with some basic keywords
header = dict()
header["TILEID"] = tile_id
header["TILERA"] = tile_ra
header["TILEDEC"] = tile_dec
header["FIELDROT"] = tile_obstheta
header["FA_PLAN"] = tile_obstime
header["FA_HA"] = tile_obsha
header["FA_RUN"] = hw.time()
header["REQRA"] = tile_ra
header["REQDEC"] = tile_dec
header["FIELDNUM"] = 0
header["FA_VER"] = __version__
header["FA_SURV"] = tgs.survey()
add_dependencies(header,
module_names=[
"numpy", "matplotlib", "astropy", "fitsio",
"desiutil", "desimodel", "desitarget"
])
fd.write(None, header=header, extname="PRIMARY")
# FIXME: write "-1" for unassigned targets. Write all other fiber
# status and other properties to this table.
log.debug(
"Write: copying assignment data for tile {}".format(tile_id))
fdata = np.zeros(nloc, dtype=assign_dtype)
fdata["LOCATION"] = locs
# For unassigned fibers, we give each location a unique negative
# number based on the tile and loc.
unassign_offset = tile_id * nloc
assigned_tgids = np.array([
tdata[x] if x in tdata.keys() else -(unassign_offset + x)
for x in locs
],
dtype=np.int64)
fdata["TARGETID"] = assigned_tgids
# Rows containing assigned locations
assigned_valid = np.where(assigned_tgids >= 0)[0]
assigned_invalid = np.where(assigned_tgids < 0)[0]
# Buffers for X/Y/RA/DEC
assigned_tgx = np.full(nloc, 9999.9, dtype=np.float64)
assigned_tgy = np.full(nloc, 9999.9, dtype=np.float64)
assigned_tgra = np.full(nloc, 9999.9, dtype=np.float64)
assigned_tgdec = np.full(nloc, 9999.9, dtype=np.float64)
assigned_tgbits = np.zeros(nloc, dtype=np.int64)
assigned_tgtype = np.zeros(nloc, dtype=np.uint8)
if (len(assigned_invalid) > 0):
# Fill our unassigned location X/Y coordinates with the central
# positioner locations. Then convert these to RA/DEC.
# NOTE: Positioner locations are in curved focal surface coordinates.
empty_fibers = locs[assigned_invalid]
fpos_xy_mm = dict(hw.loc_pos_curved_mm)
empty_x = np.array([fpos_xy_mm[f][0] for f in empty_fibers],
dtype=np.float64)
empty_y = np.array([fpos_xy_mm[f][1] for f in empty_fibers],
dtype=np.float64)
radec = hw.xy2radec_multi(tile_ra, tile_dec, tile_obstheta,
empty_x, empty_y, False, 0)
assigned_tgra[assigned_invalid] = [x for x, y in radec]
assigned_tgdec[assigned_invalid] = [y for x, y in radec]
empty_xy = hw.radec2xy_multi(tile_ra, tile_dec, tile_obstheta,
assigned_tgra[assigned_invalid],
assigned_tgdec[assigned_invalid],
True, 0)
assigned_tgx[assigned_invalid] = [x for x, y in empty_xy]
assigned_tgy[assigned_invalid] = [y for x, y in empty_xy]
if (len(assigned_valid) > 0):
# The target IDs assigned to fibers (note- NOT sorted)
assigned_real = np.copy(assigned_tgids[assigned_valid])
# Mapping of our assigned target rows into the target properties.
target_rows = [tg_indx[x] for x in assigned_real]
# Copy values out of the full target list.
assigned_tgra[assigned_valid] = np.array(tg_ra[target_rows])
assigned_tgdec[assigned_valid] = np.array(tg_dec[target_rows])
assigned_tgx[assigned_valid] = np.array(tg_x[target_rows])
assigned_tgy[assigned_valid] = np.array(tg_y[target_rows])
assigned_tgtype[assigned_valid] = np.array(tg_type[target_rows])
assigned_tgbits[assigned_valid] = np.array(tg_bits[target_rows])
fdata["TARGET_RA"] = assigned_tgra
fdata["TARGET_DEC"] = assigned_tgdec
fdata["FIBERASSIGN_X"] = assigned_tgx
fdata["FIBERASSIGN_Y"] = assigned_tgy
fdata["FA_TARGET"] = assigned_tgbits
fdata["FA_TYPE"] = assigned_tgtype
fibers = dict(hw.loc_fiber)
etcloc = sorted([x for x, y in fibers.items() if y < 0])
fakefibers = {y: (5000 + x) for x, y in enumerate(etcloc)}
fullfibers = fibers
fullfibers.update(fakefibers)
device = dict(hw.loc_device)
petal = dict(hw.loc_petal)
device_type = dict(hw.loc_device_type)
fdata["FIBER"] = np.array([fullfibers[x]
for x in locs]).astype(np.int32)
fdata["DEVICE_LOC"] = np.array([device[x]
for x in locs]).astype(np.int32)
fdata["PETAL_LOC"] = np.array([petal[x]
for x in locs]).astype(np.int16)
fdata["DEVICE_TYPE"] = np.array([device_type[x]
for x in locs]).astype(np.dtype("a3"))
# This hard-coded value copied from the original code...
lambda_ref = np.ones(nloc, dtype=np.float32) * 5400.0
fdata["LAMBDA_REF"] = lambda_ref
# Fiber status
fstate = dict(hw.state)
fstatus = np.zeros(nloc, dtype=np.int32)
# Set unused bit
fstatus |= [
0 if x in tdata.keys() else FIBER_STATE_UNASSIGNED for x in locs
]
# Set stuck / broken bits
fstatus |= [2 if (fstate[x] & FIBER_STATE_STUCK) else 0 for x in locs]
fstatus |= [4 if (fstate[x] & FIBER_STATE_BROKEN) else 0 for x in locs]
fstatus[assigned_valid] |= \
[8 if (tg_type[x] & TARGET_TYPE_SAFE) else 0
for x in target_rows]
fdata["FIBERSTATUS"] = fstatus
# tm.stop()
# tm.report(" copy and compute assign data tile {}".format(tile_id))
# tm.clear()
# tm.start()
log.debug(
"Write: writing assignment data for tile {}".format(tile_id))
fd.write(fdata, header=header, extname="FASSIGN")
del fdata
# tm.stop()
# tm.report(" write assign data tile {}".format(tile_id))
# tm.clear()
# tm.start()
log.debug("Write: writing target data for tile {}".format(tile_id))
fdata = np.zeros(ntarget, dtype=targets_dtype)
fdata["TARGETID"] = tgids
fdata["TARGET_RA"] = tg_ra
fdata["TARGET_DEC"] = tg_dec
fdata["FA_TARGET"] = tg_bits
fdata["FA_TYPE"] = tg_type
fdata["PRIORITY"] = tg_priority
fdata["SUBPRIORITY"] = tg_subpriority
fdata["OBSCONDITIONS"] = tg_obscond
# tm.stop()
# tm.report(" copy targets data tile {}".format(tile_id))
# tm.clear()
# tm.start()
fd.write(fdata, header=header, extname="FTARGETS")
del fdata
# tm.stop()
# tm.report(" write targets data tile {}".format(tile_id))
# tm.clear()
# tm.start()
log.debug("Write: writing avail data for tile {}".format(tile_id))
fdata = np.zeros(navail, dtype=avail_dtype)
off = 0
for lid in sorted(avail.keys()):
for tg in avail[lid]:
fdata[off] = (lid, fibers[lid], tg)
off += 1
# tm.stop()
# tm.report(" copy avail data tile {}".format(tile_id))
# tm.clear()
# tm.start()
fd.write(fdata, header=header, extname="FAVAIL")
del fdata
if gfa_targets is not None:
try:
# Astropy Table
fd.write(gfa_targets.as_array(), extname="GFA_TARGETS")
except AttributeError:
# numpy structured array
fd.write(gfa_targets, extname="GFA_TARGETS")
fd.close()
os.rename(tmp_file, tile_file)
# tm.stop()
# tm.report(" write avail data tile {}".format(tile_id))
return
def write_frame(outfile, frame, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
log = get_logger()
outfile = makepath(outfile, 'frame')
#- Ignore some known and harmless units warnings
import warnings
warnings.filterwarnings('ignore', message="'.*nanomaggies.* did not parse as fits unit.*")
warnings.filterwarnings('ignore', message=".*'10\*\*6 arcsec.* did not parse as fits unit.*")
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
# Vette
diagnosis = frame.vet()
if diagnosis != 0:
raise IOError("Frame did not pass simple vetting test. diagnosis={:d}".format(diagnosis))
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append( fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR') )
# hdus.append( fits.CompImageHDU(frame.mask, name='MASK') )
hdus.append( fits.ImageHDU(frame.mask, name='MASK') )
hdus.append( fits.ImageHDU(frame.wave.astype('f8'), name='WAVELENGTH') )
hdus[-1].header['BUNIT'] = 'Angstrom'
if frame.resolution_data is not None:
hdus.append( fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION' ) )
elif frame.wsigma is not None:
log.debug("Using ysigma from qproc")
qrimg=fits.ImageHDU(frame.wsigma.astype('f4'), name='YSIGMA' )
qrimg.header["NDIAG"] =frame.ndiag
hdus.append(qrimg)
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
elif frame.fibermap is not None:
fibermap = encode_table(frame.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append( fits.convenience.table_to_hdu(fibermap) )
elif frame.spectrograph is not None:
x.header['FIBERMIN'] = 500*frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error("You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append( fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX' ) )
if frame.scores is not None :
scores_tbl = encode_table(frame.scores) #- unicode -> bytes
scores_tbl.meta['EXTNAME'] = 'SCORES'
hdus.append( fits.convenience.table_to_hdu(scores_tbl) )
if frame.scores_comments is not None : # add comments in header
hdu=hdus['SCORES']
for i in range(1,999):
key = 'TTYPE'+str(i)
if key in hdu.header:
value = hdu.header[key]
if value in frame.scores_comments.keys() :
hdu.header[key] = (value, frame.scores_comments[value])
hdus.writeto(outfile+'.tmp', overwrite=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
return outfile
def __init__(self,path,mode = 'readonly',header = None):
if mode not in ('readonly','update'):
raise RuntimeError('Invalid mode %r' % mode)
self.path = path
self.mode = mode
# Create a new file if necessary.
if self.mode == 'update' and not os.path.exists(self.path):
# BRICKNAM must be in header if creating the file for the first time
if header is None or 'BRICKNAM' not in header:
raise ValueError('header must have BRICKNAM when creating new brick file')
self.brickname = header['BRICKNAM']
if 'CHANNEL' in header:
self.channel = header['CHANNEL']
else:
self.channel = 'brz' #- could be any spectrograph channel
# Create the parent directory, if necessary.
head,tail = os.path.split(self.path)
if not os.path.exists(head):
os.makedirs(head)
# Create empty HDUs. It would be good to refactor io.frame to avoid any duplication here.
hdr = desispec.io.util.fitsheader(header)
add_dependencies(hdr)
hdr['EXTNAME'] = ('FLUX', 'no dimension')
hdu0 = astropy.io.fits.PrimaryHDU(header = hdr)
hdr['EXTNAME'] = ('IVAR', 'no dimension')
hdu1 = astropy.io.fits.ImageHDU(header = hdr)
hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]')
hdu2 = astropy.io.fits.ImageHDU(header = hdr)
hdr['EXTNAME'] = ('RESOLUTION', 'no dimension')
hdu3 = astropy.io.fits.ImageHDU(header = hdr)
# Create an HDU4 using the columns from fibermap with a few extras added.
columns = desispec.io.fibermap.fibermap_columns[:]
columns.extend([
('NIGHT','i4'),
('EXPID','i4'),
('INDEX','i4'),
])
data = np.empty(shape = (0,),dtype = columns)
hdr = desispec.io.util.fitsheader(header)
#- ignore incorrect and harmless fits TDIM7 warning for
#- FILTER column that is a 2D array of strings
with warnings.catch_warnings():
warnings.simplefilter('ignore')
hdu4 = astropy.io.fits.BinTableHDU(data=data, header=hdr, name='FIBERMAP')
# Add comments for fibermap columns.
num_fibermap_columns = len(desispec.io.fibermap.fibermap_comments)
for i in range(1,1+num_fibermap_columns):
key = 'TTYPE%d' % i
name = hdu4.header[key]
comment = desispec.io.fibermap.fibermap_comments[name]
hdu4.header[key] = (name,comment)
# Add comments for our additional columns.
hdu4.header['TTYPE%d' % (1+num_fibermap_columns)] = ('NIGHT','Night of exposure YYYYMMDD')
hdu4.header['TTYPE%d' % (2+num_fibermap_columns)] = ('EXPID','Exposure ID')
hdu4.header['TTYPE%d' % (3+num_fibermap_columns)] = ('INDEX','Index of this object in other HDUs')
self.hdu_list = astropy.io.fits.HDUList([hdu0,hdu1,hdu2,hdu3,hdu4])
else:
self.hdu_list = astropy.io.fits.open(path,mode = self.mode)
try:
self.brickname = self.hdu_list[0].header['BRICKNAM']
self.channel = self.hdu_list[0].header['CHANNEL']
except KeyError:
self.channel, self.brickname = _parse_brick_filename(path)
def write_image(outfile, image, meta=None):
"""Writes image object to outfile
Args:
outfile : output file string
image : desispec.image.Image object
(or any object with 2D array attributes image, ivar, mask)
Optional:
meta : dict-like object with metadata key/values (e.g. FITS header)
"""
log = get_logger()
if meta is not None:
hdr = fitsheader(meta)
else:
hdr = fitsheader(image.meta)
add_dependencies(hdr)
#- Work around fitsio>1.0 writing blank keywords, e.g. on 20191212
for key in hdr.keys():
if type(hdr[key]) == fits.card.Undefined:
log.warning('Setting blank keyword {} to None'.format(key))
hdr[key] = None
outdir = os.path.dirname(os.path.abspath(outfile))
if not os.path.isdir(outdir):
os.makedirs(outdir)
hx = fits.HDUList()
hdu = fits.ImageHDU(image.pix.astype(np.float32), name='IMAGE', header=hdr)
if 'CAMERA' not in hdu.header:
hdu.header.append(
('CAMERA', image.camera.lower(), 'Spectrograph Camera'))
if 'RDNOISE' not in hdu.header and np.isscalar(image.readnoise):
hdu.header.append(
('RDNOISE', image.readnoise, 'Read noise [RMS electrons/pixel]'))
hx.append(hdu)
hx.append(fits.ImageHDU(image.ivar.astype(np.float32), name='IVAR'))
hx.append(fits.CompImageHDU(image.mask.astype(np.int16), name='MASK'))
if not np.isscalar(image.readnoise):
hx.append(
fits.ImageHDU(image.readnoise.astype(np.float32),
name='READNOISE'))
if hasattr(image, 'fibermap'):
if isinstance(image.fibermap, Table):
fmhdu = fits.convenience.table_to_hdu(image.fibermap)
fmhdu.name = 'FIBERMAP'
else:
fmhdu = fits.BinTableHDU(image.fibermap, name='FIBERMAP')
hx.append(fmhdu)
hx.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def write_raw(filename, rawdata, header, camera=None, primary_header=None):
'''
Write raw pixel data to a DESI raw data file
Args:
filename : file name to write data; if this exists, append a new HDU
rawdata : 2D ndarray of raw pixel data including overscans
header : dict-like object or fits.Header with keywords
CCDSECx, BIASSECx, DATASECx where x=1,2,3, or 4
Options:
camera : b0, r1 .. z9 - override value in header
primary_header : header to write in HDU0 if filename doesn't yet exist
The primary utility of this function over raw fits calls is to ensure
that all necessary keywords are present before writing the file.
CCDSECx, BIASSECx, DATASECx where x=1,2,3, or 4
DATE-OBS, GAINx and RDNOISEx will generate a non-fatal warning if missing
'''
log = get_logger()
header = desispec.io.util.fitsheader(header)
primary_header = desispec.io.util.fitsheader(primary_header)
if rawdata.dtype not in (np.int16, np.int32, np.int64):
message = 'dtype {} not supported for raw data'.format(rawdata.dtype)
log.fatal(message)
raise ValueError(message)
fail_message = ''
for required_key in ['DOSVER', 'FEEVER', 'DETECTOR']:
if required_key not in primary_header:
if required_key in header:
primary_header[required_key] = header[required_key]
else:
fail_message = fail_message + \
'Keyword {} must be in header or primary_header\n'.format(required_key)
if fail_message != '':
raise ValueError(fail_message)
#- Check required keywords before writing anything
missing_keywords = list()
if camera is None and 'CAMERA' not in header:
log.error("Must provide camera keyword or header['CAMERA']")
missing_keywords.append('CAMERA')
for amp in ['1', '2', '3', '4']:
for prefix in ['CCDSEC', 'BIASSEC', 'DATASEC']:
keyword = prefix+amp
if keyword not in header:
log.error('Missing keyword '+keyword)
missing_keywords.append(keyword)
#- Missing DATE-OBS is warning but not error
if 'DATE-OBS' not in primary_header:
if 'DATE-OBS' in header:
primary_header['DATE-OBS'] = header['DATE-OBS']
else:
log.warning('missing keyword DATE-OBS')
#- Missing GAINx is warning but not error
for amp in ['1', '2', '3', '4']:
keyword = 'GAIN'+amp
if keyword not in header:
log.warning('Gain keyword {} missing; using 1.0'.format(keyword))
header[keyword] = 1.0
#- Missing RDNOISEx is warning but not error
for amp in ['1', '2', '3', '4']:
keyword = 'RDNOISE'+amp
if keyword not in header:
log.warning('Readnoise keyword {} missing'.format(keyword))
#- Stop if any keywords are missing
if len(missing_keywords) > 0:
raise KeyError('missing required keywords {}'.format(missing_keywords))
#- Set EXTNAME=camera
if camera is not None:
header['CAMERA'] = camera.lower()
extname = camera.upper()
else:
if header['CAMERA'] != header['CAMERA'].lower():
log.warning('Converting CAMERA {} to lowercase'.format(header['CAMERA']))
header['CAMERA'] = header['CAMERA'].lower()
extname = header['CAMERA'].upper()
header['INHERIT'] = True
#- fits.CompImageHDU doesn't know how to fill in default keywords, so
#- temporarily generate an uncompressed HDU to get those keywords
header = fits.ImageHDU(rawdata, header=header, name=extname).header
#- Bizarrely, compression of 64-bit integers isn't supported.
#- downcast to 32-bit if that won't lose precision.
#- Real raw data should be 32-bit or 16-bit anyway
if rawdata.dtype == np.int64:
if np.max(np.abs(rawdata)) < 2**31:
rawdata = rawdata.astype(np.int32)
if rawdata.dtype in (np.int16, np.int32):
dataHDU = fits.CompImageHDU(rawdata, header=header, name=extname)
elif rawdata.dtype == np.int64:
log.warning('Image compression not supported for 64-bit; writing uncompressed')
dataHDU = fits.ImageHDU(rawdata, header=header, name=extname)
else:
log.error("How did we get this far with rawdata dtype {}?".format(rawdata.dtype))
dataHDU = fits.ImageHDU(rawdata, header=header, name=extname)
#- Actually write or update the file
if os.path.exists(filename):
hdus = fits.open(filename, mode='append', memmap=False)
if extname in hdus:
hdus.close()
raise ValueError('Camera {} already in {}'.format(camera, filename))
else:
hdus.append(dataHDU)
hdus.flush()
hdus.close()
else:
hdus = fits.HDUList()
add_dependencies(primary_header)
hdus.append(fits.PrimaryHDU(None, header=primary_header))
hdus.append(dataHDU)
hdus.writeto(filename)
def write_spectra(outfile, spec, units=None):
"""
Write Spectra object to FITS file.
This places the metadata into the header of the (empty) primary HDU.
The first extension contains the fibermap, and then HDUs are created for
the different data arrays for each band.
Floating point data is converted to 32 bits before writing.
Args:
outfile (str): path to write
spec (Spectra): the object containing the data
units (str): optional string to use for the BUNIT key of the flux
HDUs for each band.
Returns:
The absolute path to the file that was written.
"""
outfile = os.path.abspath(outfile)
# Create the parent directory, if necessary.
dir, base = os.path.split(outfile)
if not os.path.exists(dir):
os.makedirs(dir)
# Create HDUs from the data
all_hdus = fits.HDUList()
# metadata goes in empty primary HDU
hdr = fitsheader(spec.meta)
add_dependencies(hdr)
all_hdus.append(fits.PrimaryHDU(header=hdr))
# Next is the fibermap
fmap = spec.fibermap.copy()
fmap.meta["EXTNAME"] = "FIBERMAP"
with warnings.catch_warnings():
#- nanomaggies aren't an official IAU unit but don't complain
warnings.filterwarnings('ignore', '.*nanomaggies.*')
hdu = fits.convenience.table_to_hdu(fmap)
# Add comments for fibermap columns.
for i, colname in enumerate(fmap.dtype.names):
if colname in fibermap_comments:
key = "TTYPE{}".format(i + 1)
name = hdu.header[key]
assert name == colname
comment = fibermap_comments[name]
hdu.header[key] = (name, comment)
else:
pass
#print('Unknown comment for {}'.format(colname))
all_hdus.append(hdu)
# Now append the data for all bands
for band in spec.bands:
hdu = fits.ImageHDU(name="{}_WAVELENGTH".format(band.upper()))
hdu.header["BUNIT"] = "Angstrom"
hdu.data = spec.wave[band].astype("f8")
all_hdus.append(hdu)
hdu = fits.ImageHDU(name="{}_FLUX".format(band.upper()))
if units is None:
hdu.header["BUNIT"] = "10**-17 erg/(s cm2 Angstrom)"
else:
hdu.header["BUNIT"] = units
hdu.data = spec.flux[band].astype("f4")
all_hdus.append(hdu)
hdu = fits.ImageHDU(name="{}_IVAR".format(band.upper()))
if units is None:
hdu.header["BUNIT"] = '10**+34 (s2 cm4 Angstrom2) / erg2'
else:
hdu.header["BUNIT"] = ((u.Unit(
units, format='fits'))**-2).to_string('fits')
hdu.data = spec.ivar[band].astype("f4")
all_hdus.append(hdu)
if spec.mask is not None:
# hdu = fits.CompImageHDU(name="{}_MASK".format(band.upper()))
hdu = fits.ImageHDU(name="{}_MASK".format(band.upper()))
hdu.data = spec.mask[band].astype(np.uint32)
all_hdus.append(hdu)
if spec.resolution_data is not None:
hdu = fits.ImageHDU(name="{}_RESOLUTION".format(band.upper()))
hdu.data = spec.resolution_data[band].astype("f4")
all_hdus.append(hdu)
if spec.extra is not None:
for ex in spec.extra[band].items():
hdu = fits.ImageHDU(name="{}_{}".format(band.upper(), ex[0]))
hdu.data = ex[1].astype("f4")
all_hdus.append(hdu)
if spec.scores is not None:
scores_tbl = encode_table(spec.scores) #- unicode -> bytes
scores_tbl.meta['EXTNAME'] = 'SCORES'
all_hdus.append(fits.convenience.table_to_hdu(scores_tbl))
if spec.scores_comments is not None: # add comments in header
hdu = all_hdus['SCORES']
for i in range(1, 999):
key = 'TTYPE' + str(i)
if key in hdu.header:
value = hdu.header[key]
if value in spec.scores_comments.keys():
hdu.header[key] = (value, spec.scores_comments[value])
all_hdus.writeto("{}.tmp".format(outfile), overwrite=True, checksum=True)
os.rename("{}.tmp".format(outfile), outfile)
return outfile
#- found a tile for this timeslot; don't check other programs
break
#- Advance by 10 min; approximates long slews and observing inefficiencies
now += 10*u.min
#- TODO: write outputs (exposures list, subset of tiles observed)
x = exposures = Table(obslist)
if not os.path.exists(args.outdir):
os.makedirs(args.outdir, exist_ok=True)
exposures.meta['EXTNAME'] = 'EXPOSURES'
exposures.meta['RANDSEED'] = args.randseed
add_dependencies(exposures.meta)
outfile = os.path.join(args.outdir, 'sv_exposures.fits')
exposures.write(outfile, overwrite=True)
print('Wrote {}'.format(outfile))
for tiles in program_tiles:
filename = os.path.join(args.outdir, tiles.meta['PROGRAM'] + '-tiles.fits')
ii = (tiles['NUMOBS_DARK'] > 0) | (tiles['NUMOBS_GRAY'] > 0) | (tiles['NUMOBS_BRIGHT'] > 0)
tiles.meta['EXTNAME'] = 'TILES'
tiles.meta['RANDSEED'] = args.randseed
add_dependencies(tiles.meta)
tiles[ii].write(filename, overwrite=True)
print('Wrote {}'.format(filename))
#- Create concatentation of tiles
for tiles in program_tiles: