def write_gfas(filename, data, indir=None, nside=None, survey="?",
gaiaepoch=None):
"""Write a catalogue of Guide/Focus/Alignment targets.
Parameters
----------
filename : :class:`str`
Output file name.
data : :class:`~numpy.ndarray`
Array of GFAs to write to file.
indir : :class:`str`, optional, defaults to None.
Name of input Legacy Survey Data Release directory, write to header
of output file if passed (and if not None).
nside: :class:`int`, defaults to None.
If passed, add a column to the GFAs array popluated with HEALPixels
at resolution `nside`.
survey : :class:`str`, optional, defaults to "?"
Written to output file header as the keyword `SURVEY`.
gaiaepoch: :class:`float`, defaults to None
Gaia proper motion reference epoch. If not None, write to header of
output file. If None, default to an epoch of 2015.5.
"""
# ADM rename 'TYPE' to 'MORPHTYPE'.
data = rfn.rename_fields(data, {'TYPE': 'MORPHTYPE'})
# ADM create header to include versions, etc.
hdr = fitsio.FITSHDR()
depend.setdep(hdr, 'desitarget', desitarget_version)
depend.setdep(hdr, 'desitarget-git', gitversion())
if indir is not None:
depend.setdep(hdr, 'input-data-release', indir)
# ADM note that if 'dr' is not in the indir DR
# ADM directory structure, garbage will
# ADM be rewritten gracefully in the header.
drstring = 'dr'+indir.split('dr')[-1][0]
depend.setdep(hdr, 'photcat', drstring)
# ADM add HEALPix column, if requested by input.
if nside is not None:
theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"])
hppix = hp.ang2pix(nside, theta, phi, nest=True)
data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False)
hdr['HPXNSIDE'] = nside
hdr['HPXNEST'] = True
# ADM add the type of survey (main, or commissioning "cmx") to the header.
hdr["SURVEY"] = survey
# ADM add the Gaia reference epoch, or pass 2015.5 if not included.
hdr['REFEPOCH'] = {'name': 'REFEPOCH',
'value': 2015.5,
'comment': "Gaia Proper Motion Reference Epoch"}
if gaiaepoch is not None:
hdr['REFEPOCH'] = gaiaepoch
fitsio.write(filename, data, extname='GFA_TARGETS', header=hdr, clobber=True)
def pixweight(nside,
tiles=None,
radius=None,
precision=0.01,
outfile=None,
outplot=None):
'''
Create an array of the fraction of each pixel that overlaps the passed tiles
Optional Args:
nside: integer healpix nside, 2**k where 0 < k < 30
tiles:
Table-like with RA,DEC columns; or
None to use all DESI tiles from desimodel.io.load_tiles()
radius: tile radius in degrees;
if None use desimodel.focalplane.get_tile_radius_deg()
precision: approximate precision at which to calculate the area of pixels
that partially overlap the footprint in SQUARE DEGREES
(e.g. 0.01 means precise to 0.01 sq. deg., or 36 sq. arcmin.)
lower numbers mean better precision
outfile: if not None, then write the pixel->weight array to the file
passed as outfile (could be full directory path + file)
outplot: if a string is passed, create a plot named that string
(pass a *name* for a plot in the current directory, a *full path*
for a plot in a different directory). This is passed to
matplotlib.pyplot's savefig routine
Returns pixweight:
an array of the weight for each pixel at the passed nside. The
weight is the fracion of the pixel that overlaps the passed tiles:
`WEIGHT=1` for the pixel is entirely contained in the tiles
`WEIGHT=0` for the pixel is entirely outside of the tiles
`0 < WEIGHT < 1` for a pixel that overlaps the tiles
The index of the array is the HEALPixel integer
Notes:
it's sufficient to create the weights at a suitably high nside, say
nside=256 (0.052456 sq. deg. per pixel) as pixel numbers at
lower nsides can be obtained by integer division by powers of 4, e.g.
pix_@_nside_128 = pix@nside_256//4 and fractional weights at lower
nsides are the mean of the 4 pixels at the higher nside
desimodel.io.load_pixweight() can downsample the array to lower nsides
'''
t0 = time()
#ADM create an array that is zero for each integer pixel at this nside
import healpy as hp
npix = hp.nside2npix(nside)
weight = np.zeros(npix, float)
#ADM recover pixels that are likely to be in the DESI footprint and
#ADM set their weight to one (it's the case, then, that anything that
#ADM is *definitely outside of* the footprint has a weight of zero)
import desimodel.footprint
pix = desimodel.footprint.tiles2pix(nside,
tiles=tiles,
radius=radius,
fact=2**8)
weight[pix] = 1.
#ADM loop through to find the "edge" (fractional) pixels, until convergence
log.info('Start integration around partial pixels...')
setfracpix = set([-1])
#ADM only have a limited range, to prevent this running forever
for i in range(20):
log.info(
'Trying {} pixel boundary points (step={})...t={:.1f}s'.format(
4 * 2**i, 2**i,
time() - t0))
#ADM find the fractional pixels at this step
fracpix = desimodel.footprint.tiles2fracpix(nside,
step=2**i,
tiles=tiles,
radius=radius,
fact=2**8)
log.info('...found {} fractional pixels...t={:.1f}s'.format(
len(fracpix),
time() - t0))
if set(fracpix) == setfracpix:
break
#ADM if we didn't converge, loop through again with the new
#ADM set of fractional pixels
setfracpix = set(fracpix)
#ADM warn the user if the integration didn't converge at 4*2**20 boundary points
if i == 20:
log.warning('Integration around pixel boundaries did NOT converge!')
#ADM create a mask that is True for fractional pixels, false for all other pixels
mask = np.zeros(npix, bool)
mask[fracpix] = True
#ADM find the minimum and maximum dec of interest (there's no need to Monte Carlo
#ADM integrate over declinations that lie beyond the fractional pixels)
xyzverts = hp.boundaries(nside, fracpix, nest=True)
theta, phi = hp.vec2ang(np.hstack(xyzverts).T)
ra, dec = np.degrees(phi), 90 - np.degrees(theta)
decmin, decmax = np.min(dec), np.max(dec)
sindecmin, sindecmax = np.sin(np.radians(decmin)), np.sin(
np.radians(decmax))
area = 360. * np.degrees(sindecmax - sindecmin)
log.info(
'Populating randoms between {:.2f} and {:.2f} degrees, an area of {:.1f} sq. deg....t={:.1f}s'
.format(decmin, decmax, area,
time() - t0))
#ADM determine the required precision for the area of interest
nptpersqdeg = int((1. / precision)**2)
npt = int(nptpersqdeg * area)
log.info('Generating {} random points...t={:.1f}s'.format(
npt,
time() - t0))
#ADM loop over chunks (if npt > 1e7) to reach npt points while avoiding memory issues
nchunk = int(1e7)
pixinmask = []
rainmask = []
decinmask = []
cnt = 0
while cnt < npt:
#ADM if a chunk would pass too many points (> npt), revert to the remaining number
#ADM of points instead of creating a full chunk
if nchunk + cnt > npt:
nchunk = npt - cnt
#ADM populate the portion of the sphere of interest with random points
ra = np.random.uniform(0., 360., nchunk)
dec = np.degrees(
np.arcsin(1. -
np.random.uniform(1 - sindecmax, 1 - sindecmin, nchunk)))
#ADM convert the random points to pixel number
pix = desimodel.footprint.radec2pix(nside, ra, dec)
#ADM retain random points for which the mask is True (i.e. just the fractional pixels)
inmask = np.where(mask[pix])[0]
decinmask.append(dec[inmask])
rainmask.append(ra[inmask])
pixinmask.append(pix[inmask])
cnt += nchunk
log.info('...generated {} random points...t={:.1f}s'.format(
cnt,
time() - t0))
#ADM collapse the 2-D chunks into a 1-D array
from itertools import chain
rainmask = np.array(list(chain.from_iterable(rainmask)))
decinmask = np.array(list(chain.from_iterable(decinmask)))
pixinmask = np.array(list(chain.from_iterable(pixinmask)))
log.info(
'{} of the random points are in fractional pixels...t={:.1f}s'.format(
len(pixinmask),
time() - t0))
#ADM find which random points in the fractional pixels are in the DESI footprint
log.info(
'Start integration over fractional pixels at edges of DESI footprint...'
)
indesi = desimodel.footprint.is_point_in_desi(desimodel.io.load_tiles(),
rainmask, decinmask)
log.info(
'...{} of the random points in fractional pixels are in DESI...t={:.1f}s'
.format(np.sum(indesi),
time() - t0))
#ADM assign the weights of the fractional pixels as the fraction of random points
#ADM in the fractional pixels that are in the DESI footprint
allinfracpix = np.histogram(pixinmask, bins=np.arange(npix))[0][fracpix]
desiinfracpix = np.histogram(pixinmask[np.where(indesi)],
bins=np.arange(npix))[0][fracpix]
#ADM guard against integer division (for backwards-compatability with Python2)
#ADM and create the final array of weights
weight[fracpix] = desiinfracpix.astype('float64') / allinfracpix
if outfile is not None:
#ADM get path to DESIMODEL footprint directory, create output file name
import desimodel.io
#ADM write information indicating HEALPix setup to file header
#ADM include desimodel version as a check in case footprint changes
import fitsio
from desiutil import depend
hdr = fitsio.FITSHDR()
depend.setdep(hdr, 'desimodel', desimodel_version)
hdr['PRECISE'] = precision
hdr['HPXNSIDE'] = nside
hdr['HPXNEST'] = True
fitsio.write(outfile,
weight,
extname='PIXWEIGHTS',
header=hdr,
clobber=True)
#ADM if outplot was passed, make a plot of the final mask in Mollweide projection
if outplot is not None:
import matplotlib.pyplot as plt
hp.mollview(weight, nest=True)
plt.savefig(outplot)
log.info('Done...t={:.1f}s'.format(time() - t0))
return weight
def write_targets(filename, data, indir=None, qso_selection=None,
sandboxcuts=False, nside=None):
"""Write a target catalogue.
Parameters
----------
filename : output target selection file
data : numpy structured array of targets to save
nside: :class:`int`
If passed, add a column to the targets array popluated
with HEALPix pixels at resolution nside
"""
# FIXME: assert data and tsbits schema
#ADM use RELEASE to determine the release string for the input targets
if len(data) == 0:
#ADM if there are no targets, then we don't know the Data Release
drstring = 'unknowndr'
else:
drint = np.max(data['RELEASE']//1000)
drstring = 'dr'+str(drint)
#- Create header to include versions, etc.
hdr = fitsio.FITSHDR()
depend.setdep(hdr, 'desitarget', desitarget_version)
depend.setdep(hdr, 'desitarget-git', gitversion())
depend.setdep(hdr, 'sandboxcuts', sandboxcuts)
depend.setdep(hdr, 'photcat', drstring)
if indir is not None:
depend.setdep(hdr, 'tractor-files', indir)
if qso_selection is None:
print('WARNING: qso_selection method not specified for output file')
depend.setdep(hdr, 'qso-selection', 'unknown')
else:
depend.setdep(hdr, 'qso-selection', qso_selection)
#ADM add HEALPix column, if requested by input
if nside is not None:
theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"])
hppix = hp.ang2pix(nside, theta, phi, nest=True)
data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False)
depend.setdep(hdr, 'HPXNSIDE', nside)
depend.setdep(hdr, 'HPXNEST', True)
#ADM add PHOTSYS column, mapped from RELEASE
photsys = release_to_photsys(data["RELEASE"])
data = rfn.append_fields(data, 'PHOTSYS', photsys, usemask=False)
fitsio.write(filename, data, extname='TARGETS', header=hdr, clobber=True)
def preproc(rawimage,
header,
primary_header,
bias=True,
dark=True,
pixflat=True,
mask=True,
bkgsub=False,
nocosmic=False,
cosmics_nsig=6,
cosmics_cfudge=3.,
cosmics_c2fudge=0.5,
ccd_calibration_filename=None,
nocrosstalk=False,
nogain=False,
overscan_per_row=False,
use_overscan_row=False,
use_savgol=None,
nodarktrail=False,
remove_scattered_light=False,
psf_filename=None,
bias_img=None,
model_variance=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = A, B, C, D for each of the 4 amplifiers
(also supports old naming convention 1, 2, 3, 4).
primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
DATE-OBS is also required if bias, pixflat, or mask=True
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
Optional overscan features:
overscan_per_row : bool, Subtract the overscan_col values
row by row from the data.
use_overscan_row : bool, Subtract off the overscan_row
from the data (default: False). Requires ORSEC in
the Header
use_savgol : bool, Specify whether to use Savitsky-Golay filter for
the overscan. (default: False). Requires use_overscan_row=True
to have any effect.
Optional variance model if model_variance=True
Optional background subtraction with median filtering if bkgsub=True
Optional disabling of cosmic ray rejection if nocosmic=True
Optional disabling of dark trail correction if nodarktrail=True
Optional bias image (testing only) may be provided by bias_img=
Optional tuning of cosmic ray rejection parameters:
cosmics_nsig: number of sigma above background required
cosmics_cfudge: number of sigma inconsistent with PSF required
cosmics_c2fudge: fudge factor applied to PSF
Optional fit and subtraction of scattered light
Returns Image object with member variables:
pix : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
log = get_logger()
header = header.copy()
depend.setdep(header, 'DESI_SPECTRO_CALIB',
os.getenv('DESI_SPECTRO_CALIB'))
for key in ['DESI_SPECTRO_REDUX', 'SPECPROD']:
if key in os.environ:
depend.setdep(header, key, os.environ[key])
cfinder = None
if ccd_calibration_filename is not False:
cfinder = CalibFinder([header, primary_header],
yaml_file=ccd_calibration_filename)
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
#- convert rawimage to float64 : this is the output format of read_image
rawimage = rawimage.astype(np.float64)
# Savgol
if cfinder and cfinder.haskey("USE_ORSEC"):
use_overscan_row = cfinder.value("USE_ORSEC")
if cfinder and cfinder.haskey("SAVGOL"):
use_savgol = cfinder.value("SAVGOL")
# Set bias image, as desired
if bias_img is None:
bias = get_calibration_image(cfinder, "BIAS", bias, header)
else:
bias = bias_img
#- Check if this file uses amp names 1,2,3,4 (old) or A,B,C,D (new)
amp_ids = get_amp_ids(header)
#- Double check that we have the necessary keywords
missing_keywords = list()
for prefix in ['CCDSEC', 'BIASSEC']:
for amp in amp_ids:
key = prefix + amp
if not key in header:
log.error('No {} keyword in header'.format(key))
missing_keywords.append(key)
if len(missing_keywords) > 0:
raise KeyError("Missing keywords {}".format(
' '.join(missing_keywords)))
#- Output arrays
ny = 0
nx = 0
for amp in amp_ids:
yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp])
ny = max(ny, yy.stop)
nx = max(nx, xx.stop)
image = np.zeros((ny, nx))
readnoise = np.zeros_like(image)
#- Load dark
if cfinder and cfinder.haskey("DARK") and (dark is not False):
#- Exposure time
if cfinder and cfinder.haskey("EXPTIMEKEY"):
exptime_key = cfinder.value("EXPTIMEKEY")
log.info("Using exposure time keyword %s for dark normalization" %
exptime_key)
else:
exptime_key = "EXPTIME"
exptime = primary_header[exptime_key]
log.info(
"Use exptime = {} sec to compute the dark current".format(exptime))
dark_filename = cfinder.findfile("DARK")
depend.setdep(header, 'CCD_CALIB_DARK',
shorten_filename(dark_filename))
log.info(f'Using DARK model from {dark_filename}')
# dark is multipled by exptime, or we use the non-linear dark model in the routine
dark = read_dark(filename=dark_filename, exptime=exptime)
if dark.shape == image.shape:
log.info("dark is trimmed")
trimmed_dark_in_electrons = dark
dark_is_trimmed = True
elif dark.shape == rawimage.shape:
log.info("dark is not trimmed")
trimmed_dark_in_electrons = np.zeros_like(image)
dark_is_trimmed = False
else:
message = "incompatible dark shape={} when raw shape={} and preproc shape={}".format(
dark.shape, rawimage.shape, image.shape)
log.error(message)
raise ValueError(message)
else:
dark = False
if bias is not False: #- it's an array
if bias.shape == rawimage.shape:
log.info("subtracting bias")
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(
bias.shape, rawimage.shape))
#- Load mask
mask = get_calibration_image(cfinder, "MASK", mask, header)
if mask is False:
mask = np.zeros(image.shape, dtype=np.int32)
else:
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(
mask.shape, image.shape))
for amp in amp_ids:
# Grab the sections
ov_col = parse_sec_keyword(header['BIASSEC' + amp])
if 'ORSEC' + amp in header.keys():
ov_row = parse_sec_keyword(header['ORSEC' + amp])
elif use_overscan_row:
log.error('No ORSEC{} keyword; not using overscan_row'.format(amp))
use_overscan_row = False
if nogain:
gain = 1.
else:
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
if cfinder and cfinder.haskey('GAIN' + amp):
gain = float(cfinder.value('GAIN' + amp))
log.info('Using GAIN{}={} from calibration data'.format(
amp, gain))
else:
gain = 1.0
log.warning(
'Missing keyword GAIN{} in header and nothing in calib data; using {}'
.format(amp, gain))
#- Record what gain value was actually used
header['GAIN' + amp] = gain
#- Add saturation level
if 'SATURLEV' + amp in header:
saturlev_adu = header['SATURLEV' + amp] # in ADU
else:
if cfinder and cfinder.haskey('SATURLEV' + amp):
saturlev_adu = float(cfinder.value('SATURLEV' + amp))
log.info('Using SATURLEV{}={} from calibration data'.format(
amp, saturlev_adu))
else:
saturlev_adu = 2**16 - 1 # 65535 is the max value in the images
log.warning(
'Missing keyword SATURLEV{} in header and nothing in calib data; using {} ADU'
.format(amp, saturlev_adu))
header['SATULEV' +
amp] = (saturlev_adu,
"saturation or non lin. level, in ADU, inc. bias")
# Generate the overscan images
raw_overscan_col = rawimage[ov_col].copy()
if use_overscan_row:
raw_overscan_row = rawimage[ov_row].copy()
overscan_row = np.zeros_like(raw_overscan_row)
# Remove overscan_col from overscan_row
raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy()
for row in range(raw_overscan_row.shape[0]):
o, r = _overscan(raw_overscan_squared[row])
overscan_row[row] = raw_overscan_row[row] - o
# Now remove the overscan_col
nrows = raw_overscan_col.shape[0]
log.info("nrows in overscan=%d" % nrows)
overscan_col = np.zeros(nrows)
rdnoise = np.zeros(nrows)
if (cfinder and cfinder.haskey('OVERSCAN' + amp)
and cfinder.value("OVERSCAN" + amp).upper()
== "PER_ROW") or overscan_per_row:
log.info(
"Subtracting overscan per row for amplifier %s of camera %s" %
(amp, camera))
for j in range(nrows):
if np.isnan(np.sum(overscan_col[j])):
log.warning(
"NaN values in row %d of overscan of amplifier %s of camera %s"
% (j, amp, camera))
continue
o, r = _overscan(raw_overscan_col[j])
#log.info("%d %f %f"%(j,o,r))
overscan_col[j] = o
rdnoise[j] = r
else:
log.info(
"Subtracting average overscan for amplifier %s of camera %s" %
(amp, camera))
o, r = _overscan(raw_overscan_col)
overscan_col += o
rdnoise += r
if bias is not False:
jj = parse_sec_keyword(header['DATASEC' + amp])
o, biasnoise = _overscan(bias[jj])
new_rdnoise = np.sqrt(rdnoise**2 + biasnoise**2)
log.info(
"Master bias noise for AMP %s = %4.3f ADU, rdnoise %4.3f -> %4.3f ADU"
% (amp, biasnoise, np.mean(rdnoise), np.mean(new_rdnoise)))
rdnoise = new_rdnoise
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan_col)
log.info("Median rdnoise and overscan= %f %f" %
(median_rdnoise, median_overscan))
kk = parse_sec_keyword(header['CCDSEC' + amp])
for j in range(nrows):
readnoise[kk][j] = rdnoise[j]
header['OVERSCN' + amp] = (median_overscan, 'ADUs (gain not applied)')
if gain != 1:
rdnoise_message = 'electrons (gain is applied)'
gain_message = 'e/ADU (gain applied to image)'
else:
rdnoise_message = 'ADUs (gain not applied)'
gain_message = 'gain not applied to image'
header['OBSRDN' + amp] = (median_rdnoise, rdnoise_message)
header['GAIN' + amp] = (gain, gain_message)
#- Warn/error if measured readnoise is very different from expected if exists
if 'RDNOISE' + amp in header:
expected_readnoise = header['RDNOISE' + amp]
if median_rdnoise < 0.5 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise < 0.9 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 2.0 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 1.2 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
#else:
# log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise))
#- subtract overscan from data region and apply gain
jj = parse_sec_keyword(header['DATASEC' + amp])
data = rawimage[jj].copy()
# Subtract columns
for k in range(nrows):
data[k] -= overscan_col[k]
saturlev_elec = gain * (saturlev_adu - np.mean(overscan_col))
header['SATUELE' +
amp] = (saturlev_elec,
"saturation or non lin. level, in electrons")
# And now the rows
if use_overscan_row:
# Savgol?
if use_savgol:
log.info("Using savgol")
collapse_oscan_row = np.zeros(overscan_row.shape[1])
for col in range(overscan_row.shape[1]):
o, _ = _overscan(overscan_row[:, col])
collapse_oscan_row[col] = o
oscan_row = _savgol_clipped(collapse_oscan_row, niter=0)
oimg_row = np.outer(np.ones(data.shape[0]), oscan_row)
data -= oimg_row
else:
o, r = _overscan(overscan_row)
data -= o
#- apply saturlev (defined in ADU), prior to multiplication by gain
saturated = (rawimage[jj] >= saturlev_adu)
mask[kk][saturated] |= ccdmask.SATURATED
#- ADC to electrons
image[kk] = data * gain
if dark is not False:
if not dark_is_trimmed:
trimmed_dark_in_electrons[kk] = dark[jj] * gain
if not nocrosstalk:
#- apply cross-talk
# the ccd looks like :
# C D
# A B
# for cross talk, we need a symmetric 4x4 flip_matrix
# of coordinates ABCD giving flip of both axis
# when computing crosstalk of
# A B C D
#
# A AA AB AC AD
# B BA BB BC BD
# C CA CB CC CD
# D DA DB DC BB
# orientation_matrix_defines change of orientation
#
fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1],
[-1, -1, 1, 1]])
fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1],
[-1, 1, -1, 1]])
for a1 in range(len(amp_ids)):
amp1 = amp_ids[a1]
ii1 = parse_sec_keyword(header['CCDSEC' + amp1])
a1flux = image[ii1]
#a1mask=mask[ii1]
for a2 in range(len(amp_ids)):
if a1 == a2:
continue
amp2 = amp_ids[a2]
if cfinder is None: continue
if not cfinder.haskey("CROSSTALK%s%s" % (amp1, amp2)): continue
crosstalk = cfinder.value("CROSSTALK%s%s" % (amp1, amp2))
if crosstalk == 0.: continue
log.info("Correct for crosstalk=%f from AMP %s into %s" %
(crosstalk, amp1, amp2))
a12flux = crosstalk * a1flux.copy()
#a12mask=a1mask.copy()
if fip_axis_0[a1, a2] == -1:
a12flux = a12flux[::-1]
#a12mask=a12mask[::-1]
if fip_axis_1[a1, a2] == -1:
a12flux = a12flux[:, ::-1]
#a12mask=a12mask[:,::-1]
ii2 = parse_sec_keyword(header['CCDSEC' + amp2])
image[ii2] -= a12flux
# mask[ii2] |= a12mask (not sure we really need to propagate the mask)
#- Poisson noise variance (prior to dark subtraction and prior to pixel flat field)
#- This is biasing, but that's what we have for now
poisson_var = image.clip(0)
#- subtract dark after multiplication by gain
if dark is not False:
log.info("subtracting dark")
image -= trimmed_dark_in_electrons
# measure its noise
new_readnoise = np.zeros(readnoise.shape)
for amp in amp_ids:
kk = parse_sec_keyword(header['CCDSEC' + amp])
o, darknoise = _overscan(trimmed_dark_in_electrons[kk])
new_readnoise[kk] = np.sqrt(readnoise[kk]**2 + darknoise**2)
log.info(
"Master dark noise for AMP %s = %4.3f elec, rdnoise %4.3f -> %4.3f elec"
% (amp, darknoise, np.mean(
readnoise[kk]), np.mean(new_readnoise[kk])))
readnoise = new_readnoise
#- Correct for dark trails if any
if not nodarktrail and cfinder is not None:
for amp in amp_ids:
if cfinder.haskey("DARKTRAILAMP%s" % amp):
amplitude = cfinder.value("DARKTRAILAMP%s" % amp)
width = cfinder.value("DARKTRAILWIDTH%s" % amp)
ii = _parse_sec_keyword(header["CCDSEC" + amp])
log.info(
"Removing dark trails for amplifier %s with width=%3.1f and amplitude=%5.4f"
% (amp, width, amplitude))
correct_dark_trail(image,
ii,
left=((amp == "B") | (amp == "D")),
width=width,
amplitude=amplitude)
#- Divide by pixflat image
pixflat = get_calibration_image(cfinder, "PIXFLAT", pixflat, header)
if pixflat is not False:
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(
pixflat.shape, image.shape))
almost_zero = 0.001
if np.all(pixflat > almost_zero):
image /= pixflat
readnoise /= pixflat
poisson_var /= pixflat**2
else:
good = (pixflat > almost_zero)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
poisson_var[good] /= pixflat[good]**2
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = poisson_var + readnoise**2
ivar = np.zeros(var.shape)
ivar[var > 0] = 1.0 / var[var > 0]
#- Ridiculously high readnoise is bad
mask[readnoise > 100] |= ccdmask.BADREADNOISE
if bkgsub:
bkg = _background(image, header)
image -= bkg
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
#- update img.mask to mask cosmic rays
if not nocosmic:
cosmics.reject_cosmic_rays(img,
nsig=cosmics_nsig,
cfudge=cosmics_cfudge,
c2fudge=cosmics_c2fudge)
mask = img.mask
xyset = None
if model_variance:
psf = None
if psf_filename is None:
psf_filename = cfinder.findfile("PSF")
depend.setdep(header, 'CCD_CALIB_PSF', shorten_filename(psf_filename))
xyset = read_xytraceset(psf_filename)
fiberflat = None
with_spectral_smoothing = True
with_sky_model = True
if with_sky_model:
log.debug("Will use a sky model to model the spectra")
fiberflat_filename = cfinder.findfile("FIBERFLAT")
depend.setdep(header, 'CCD_CALIB_FIBERFLAT',
shorten_filename(fiberflat_filename))
if fiberflat_filename is not None:
fiberflat = read_fiberflat(fiberflat_filename)
log.info("compute an image model after dark correction and pixel flat")
nsig = 5.
mimage = compute_image_model(
img,
xyset,
fiberflat=fiberflat,
with_spectral_smoothing=with_spectral_smoothing,
with_sky_model=with_sky_model,
spectral_smoothing_nsig=nsig,
psf=psf)
# here we bring back original image for large outliers
# this allows to have a correct ivar for cosmic rays and bright sources
eps = 0.1
out = (((ivar > 0) * (image - mimage)**2 /
(1. / (ivar + (ivar == 0)) + (0.1 * mimage)**2)) > nsig**2)
# out &= (image>mimage) # could request this to be conservative on the variance ... but this could cause other issues
mimage[out] = image[out]
log.info("use image model to compute variance")
if bkgsub:
mimage += bkg
if pixflat is not False:
# undo pixflat
mimage *= pixflat
if dark is not False:
mimage += dark
poisson_var = mimage.clip(0)
if pixflat is not False:
if np.all(pixflat > almost_zero):
poisson_var /= pixflat**2
else:
poisson_var[good] /= pixflat[good]**2
var = poisson_var + readnoise**2
ivar[var > 0] = 1.0 / var[var > 0]
# regenerate img object
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
if remove_scattered_light:
if xyset is None:
if psf_filename is None:
psf_filename = cfinder.findfile("PSF")
depend.setdep(header, 'SCATTERED_LIGHT_PSF',
shorten_filename(psf_filename))
xyset = read_xytraceset(psf_filename)
img.pix -= model_scattered_light(img, xyset)
#- Extend header with primary header keywords too
addkeys(img.meta, primary_header)
return img
def get_calibration_image(cfinder, keyword, entry, header=None):
"""Reads a calibration file
Args:
cfinder : None or CalibFinder object
keyword : BIAS, MASK, or PIXFLAT
entry : boolean or filename or image
if entry==False return False
if entry==True use calibration filename from calib. config and read it
if entry==str use this for the filename
if entry==image return input
Options:
header : if not None, update header['CAL...'] = calib provenance
returns:
2D numpy array with calibration image
"""
log = get_logger()
#- set the header to something so that we don't have to keep checking it
if header is None:
header = dict()
calkey = 'CCD_CALIB_{}'.format(keyword.upper())
if entry is False:
depend.setdep(header, calkey, 'None')
return False # we don't want do anything
filename = None
if entry is True:
# we have to find the filename
if cfinder is None:
log.error("no calibration data was found")
raise ValueError("no calibration data was found")
if cfinder.haskey(keyword):
filename = cfinder.findfile(keyword)
depend.setdep(header, calkey, shorten_filename(filename))
else:
depend.setdep(header, calkey, 'None')
return False # we say in the calibration data we don't need this
elif isinstance(entry, str):
filename = entry
depend.setdep(header, calkey, shorten_filename(filename))
else:
depend.setdep(header, calkey, 'Unknown image')
return entry # it's expected to be an image array
log.info("Using %s %s" % (keyword, filename))
if keyword == "BIAS":
return read_bias(filename=filename)
elif keyword == "MASK":
return read_mask(filename=filename)
elif keyword == "PIXFLAT":
return read_pixflat(filename=filename)
elif keyword == "DARK":
raise ValueError("Dark are now treated separately.")
else:
log.error("Don't known how to read %s in %s" % (keyword, path))
raise ValueError("Don't known how to read %s in %s" % (keyword, path))
return False
def write_randoms(filename, data, indir=None, hdr=None, nside=None, density=None):
"""Write a catalogue of randoms and associated pixel-level information.
Parameters
----------
filename : :class:`str`
Output file name.
data : :class:`~numpy.ndarray`
Array of randoms to write to file.
indir : :class:`str`, optional, defaults to None
Name of input Legacy Survey Data Release directory, write to header
of output file if passed (and if not None).
hdr : :class:`str`, optional, defaults to `None`
If passed, use this header to start the header of the output `filename`.
nside: :class:`int`
If passed, add a column to the randoms array popluated with HEALPixels
at resolution `nside`.
density: :class:`int`
Number of points per sq. deg. at which the catalog was generated,
write to header of the output file if not None.
"""
# ADM create header to include versions, etc. If a `hdr` was
# ADM passed, then use it, if not then create a new header.
if hdr is None:
hdr = fitsio.FITSHDR()
depend.setdep(hdr, 'desitarget', desitarget_version)
depend.setdep(hdr, 'desitarget-git', gitversion())
if indir is not None:
depend.setdep(hdr, 'input-data-release', indir)
# ADM note that if 'dr' is not in the indir DR
# ADM directory structure, garbage will
# ADM be rewritten gracefully in the header.
drstring = 'dr'+indir.split('dr')[-1][0]
depend.setdep(hdr, 'photcat', drstring)
# ADM also write the mask bits header information
# ADM from a mask bits file in this DR.
from glob import iglob
files = iglob(indir+'/coadd/*/*/*maskbits*')
# ADM we built an iterator over mask bits files for speed
# ADM if there are no such files to iterate over, just pass.
try:
fn = next(files)
mbhdr = fitsio.read_header(fn)
# ADM extract the keys that include the string 'BITNM'.
bncols = [key for key in mbhdr.keys() if 'BITNM' in key]
for col in bncols:
hdr[col] = {'name': col,
'value': mbhdr[col],
'comment': mbhdr.get_comment(col)}
except StopIteration:
pass
# ADM add HEALPix column, if requested by input.
if nside is not None:
theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"])
hppix = hp.ang2pix(nside, theta, phi, nest=True)
data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False)
hdr['HPXNSIDE'] = nside
hdr['HPXNEST'] = True
# ADM add density of points if requested by input.
if density is not None:
hdr['DENSITY'] = density
fitsio.write(filename, data, extname='RANDOMS', header=hdr, clobber=True)
def write_skies(filename, data, indir=None, apertures_arcsec=None,
nskiespersqdeg=None, nside=None):
"""Write a target catalogue of sky locations.
Parameters
----------
filename : :class:`str`
Output target selection file name
data : :class:`~numpy.ndarray`
Array of skies to write to file.
indir : :class:`str`, optional
Name of input Legacy Survey Data Release directory, write to header
of output file if passed (and if not None).
apertures_arcsec : :class:`list` or `float`, optional
list of aperture radii in arcseconds to write each aperture as an
individual line in the header, if passed (and if not None).
nskiespersqdeg : :class:`float`, optional
Number of sky locations generated per sq. deg., write to header
of output file if passed (and if not None).
nside: :class:`int`, optional
If passed, add a column to the skies array popluated with HEALPixels
at resolution `nside`.
"""
nskies = len(data)
# ADM force OBSCONDITIONS to be 65535
# ADM (see https://github.com/desihub/desitarget/pull/313).
data["OBSCONDITIONS"] = 2**16-1
# - Create header to include versions, etc.
hdr = fitsio.FITSHDR()
depend.setdep(hdr, 'desitarget', desitarget_version)
depend.setdep(hdr, 'desitarget-git', gitversion())
if indir is not None:
depend.setdep(hdr, 'input-data-release', indir)
# ADM note that if 'dr' is not in the indir DR
# ADM directory structure, garbage will
# ADM be rewritten gracefully in the header.
drstring = 'dr'+indir.split('dr')[-1][0]
depend.setdep(hdr, 'photcat', drstring)
if apertures_arcsec is not None:
for i, ap in enumerate(apertures_arcsec):
apname = "AP{}".format(i)
apsize = ap
hdr[apname] = apsize
if nskiespersqdeg is not None:
hdr['NPERSDEG'] = nskiespersqdeg
# ADM add HEALPix column, if requested by input.
if nside is not None:
theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"])
hppix = hp.ang2pix(nside, theta, phi, nest=True)
data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False)
hdr['HPXNSIDE'] = nside
hdr['HPXNEST'] = True
# ADM populate SUBPRIORITY with a reproducible random float.
if "SUBPRIORITY" in data.dtype.names:
np.random.seed(616)
data["SUBPRIORITY"] = np.random.random(nskies)
fitsio.write(filename, data, extname='SKY_TARGETS', header=hdr, clobber=True)
def write_targets(filename, data, indir=None, qso_selection=None,
sandboxcuts=False, nside=None, survey="?",
nsidefile=None, hpxlist=None):
"""Write a target catalogue.
Parameters
----------
filename : :class:`str`
output target selection file.
data : :class:`~numpy.ndarray`
numpy structured array of targets to save.
indir, qso_selection : :class:`str`, optional, default to `None`
If passed, note these as the input directory and
quasar selection method in the output file header.
sandboxcuts : :class:`bool`, optional, defaults to ``False``
If passed, note this whether we ran target seletion
in the sandbox in the output file header.
nside : :class:`int`, optional, defaults to `None`
If passed, add a column to the targets array popluated
with HEALPixels at resolution `nside`.
survey : :class:`str`, optional, defaults to "?"
Written to output file header as the keyword `SURVEY`.
nsidefile : :class:`int`, optional, defaults to `None`
Passed to indicate in the output file header that the targets
have been limited to only certain HEALPixels at a given
nside. Used in conjunction with `hpxlist`.
hpxlist : :class:`list`, optional, defaults to `None`
Passed to indicate in the output file header that the targets
have been limited to only this list of HEALPixels. Used in
conjunction with `nsidefile`.
"""
# FIXME: assert data and tsbits schema
# ADM use RELEASE to determine the release string for the input targets.
ntargs = len(data)
if ntargs == 0:
# ADM if there are no targets, then we don't know the Data Release.
drstring = 'unknowndr'
else:
drint = np.max(data['RELEASE']//1000)
drstring = 'dr'+str(drint)
# - Create header to include versions, etc.
hdr = fitsio.FITSHDR()
depend.setdep(hdr, 'desitarget', desitarget_version)
depend.setdep(hdr, 'desitarget-git', gitversion())
depend.setdep(hdr, 'sandboxcuts', sandboxcuts)
depend.setdep(hdr, 'photcat', drstring)
if indir is not None:
depend.setdep(hdr, 'tractor-files', indir)
if qso_selection is None:
log.warning('qso_selection method not specified for output file')
depend.setdep(hdr, 'qso-selection', 'unknown')
else:
depend.setdep(hdr, 'qso-selection', qso_selection)
# ADM add HEALPix column, if requested by input.
if nside is not None:
theta, phi = np.radians(90-data["DEC"]), np.radians(data["RA"])
hppix = hp.ang2pix(nside, theta, phi, nest=True)
data = rfn.append_fields(data, 'HPXPIXEL', hppix, usemask=False)
hdr['HPXNSIDE'] = nside
hdr['HPXNEST'] = True
# ADM populate SUBPRIORITY with a reproducible random float.
if "SUBPRIORITY" in data.dtype.names:
np.random.seed(616)
data["SUBPRIORITY"] = np.random.random(ntargs)
# ADM add the type of survey (main, commissioning; or "cmx", sv) to the header.
hdr["SURVEY"] = survey
# ADM record whether this file has been limited to only certain HEALPixels.
if hpxlist is not None or nsidefile is not None:
# ADM hpxlist and nsidefile need to be passed together.
if hpxlist is None or nsidefile is None:
msg = 'Both hpxlist (={}) and nsidefile (={}) need to be set' \
.format(hpxlist, nsidefile)
log.critical(msg)
raise ValueError(msg)
hdr['FILENSID'] = nsidefile
hdr['FILENEST'] = True
hdr['FILEHPX'] = hpxlist
fitsio.write(filename, data, extname='TARGETS', header=hdr, clobber=True)