def test_init(self):
"""Cleanup test files if they exist.
"""
pheader={"DATE-OBS":'2018-11-30T12:42:10.442593-05:00',"DOSVER":'SIM'}
header={"DETECTOR":'SIM',"CAMERA":'b0 ',"FEEVER":'SIM'}
cfinder = CalibFinder([pheader,header])
print(cfinder.value("DETECTOR"))
if cfinder.haskey("BIAS") :
print(cfinder.findfile("BIAS"))
def test_init(self):
"""Cleanup test files if they exist.
"""
pheader = {
"DATE-OBS": '2018-11-30T12:42:10.442593-05:00',
"DOSVER": 'SIM'
}
header = {"DETECTOR": 'SIM', "CAMERA": 'b0 ', "FEEVER": 'SIM'}
cfinder = CalibFinder([pheader, header])
print(cfinder.value("DETECTOR"))
if cfinder.haskey("BIAS"):
print(cfinder.findfile("BIAS"))
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
results = list()
infiles = glob.glob(os.path.join(indir, 'qframe-*.fits'))
if len(infiles) == 0:
log.error("no qframe in {}".format(indir))
return None
for filename in infiles:
qframe = read_qframe(filename)
night = int(qframe.meta['NIGHT'])
expid = int(qframe.meta['EXPID'])
cam = qframe.meta['CAMERA'][0].upper()
spectro = int(qframe.meta['CAMERA'][1])
try:
cfinder = CalibFinder([qframe.meta])
except:
log.error(
"failed to find calib for qframe {}".format(filename))
continue
if not cfinder.haskey("FIBERFLAT"):
log.warning(
"no known fiberflat for qframe {}".format(filename))
continue
fflat = read_fiberflat(cfinder.findfile("FIBERFLAT"))
tmp = np.median(fflat.fiberflat, axis=1)
reference_fflat = tmp / np.median(tmp)
tmp = np.median(qframe.flux, axis=1)
this_fflat = tmp / np.median(tmp)
for f, fiber in enumerate(qframe.fibermap["FIBER"]):
results.append(
collections.OrderedDict(NIGHT=night,
EXPID=expid,
SPECTRO=spectro,
CAM=cam,
FIBER=fiber,
FIBERFLAT=this_fflat[f],
REF_FIBERFLAT=reference_fflat[f]))
if len(results) == 0:
return None
return Table(results, names=results[0].keys())
def nights_and_expid():
badflats = []
for night in nights:
expids = desispec.io.get_exposures(night)
for expid in expids:
for spectro in np.arange(10):
for camera in ('b', 'r', 'z'):
framefile = desispec.io.findfile('frame',
night=int(night),
expid=expid,
camera='{}{}'.format(
camera, spectro))
if not os.path.isfile(framefile):
print('Missing {}'.format(framefile))
continue
hdr = fitsio.read_header(framefile)
calib = CalibFinder([hdr])
#fr = desispec.io.read_frame(framefile)
#calib = CalibFinder([fr.meta])
fiberflatfile = os.path.join(
os.getenv('DESI_SPECTRO_CALIB'),
calib.data['FIBERFLAT'])
#print(fiberflatfile)
fiberflat = desispec.io.fiberflat.read_fiberflat(
fiberflatfile)
if np.sum(
np.sum(fiberflat.ivar == 0, axis=1) ==
hdr['NAXIS1']):
badflats.append(fiberflatfile)
pdb.set_trace()
def psf_fit(idx,outdir,cam):
if os.path.isfile('{}/psf-{}-{}.fits'.format(outdir,cam,idx)):
print('INFO: already done')
return
h = fitsio.FITS('{}/preproc-{}-{}.fits'.format(outdir,cam,idx))
head = h['IMAGE'].read_header()
cfinder = CalibFinder([head])
p = cfinder.findfile('PSF')
h.close()
cmd = 'desi_psf_fit'
cmd += ' -a {}/preproc-{}-{}.fits'.format(outdir,cam,idx)
cmd += ' --in-psf {}'.format(p)
cmd += ' --out-psf {}/psf-{}-{}.fits'.format(outdir,cam,idx)
if 'b' in cam:
cmd += ' --trace-deg-wave 4 --trace-deg-x 4 --legendre-deg-x 4 --legendre-deg-wave 2 --single-bundle'
elif ('r' in cam) or ('z' in cam):
cmd += ' --legendre-deg-x 4 --legendre-deg-wave 3 --single-bundle'
print(cmd)
subprocess.call(cmd,shell=True)
return
def preproc(rawimage,
header,
primary_header,
bias=True,
dark=True,
pixflat=True,
mask=True,
bkgsub=False,
nocosmic=False,
cosmics_nsig=6,
cosmics_cfudge=3.,
cosmics_c2fudge=0.5,
ccd_calibration_filename=None,
nocrosstalk=False,
nogain=False,
overscan_per_row=False,
use_overscan_row=False,
use_savgol=None,
nodarktrail=False,
remove_scattered_light=False,
psf_filename=None,
bias_img=None):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = A, B, C, D for each of the 4 amplifiers
(also supports old naming convention 1, 2, 3, 4).
primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
DATE-OBS is also required if bias, pixflat, or mask=True
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
Optional overscan features:
overscan_per_row : bool, Subtract the overscan_col values
row by row from the data.
use_overscan_row : bool, Subtract off the overscan_row
from the data (default: False). Requires ORSEC in
the Header
use_savgol : bool, Specify whether to use Savitsky-Golay filter for
the overscan. (default: False). Requires use_overscan_row=True
to have any effect.
Optional background subtraction with median filtering if bkgsub=True
Optional disabling of cosmic ray rejection if nocosmic=True
Optional disabling of dark trail correction if nodarktrail=True
Optional bias image (testing only) may be provided by bias_img=
Optional tuning of cosmic ray rejection parameters:
cosmics_nsig: number of sigma above background required
cosmics_cfudge: number of sigma inconsistent with PSF required
cosmics_c2fudge: fudge factor applied to PSF
Optional fit and subtraction of scattered light
Returns Image object with member variables:
pix : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
log = get_logger()
header = header.copy()
cfinder = None
if ccd_calibration_filename is not False:
cfinder = CalibFinder([header, primary_header],
yaml_file=ccd_calibration_filename)
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
#- convert rawimage to float64 : this is the output format of read_image
rawimage = rawimage.astype(np.float64)
# Savgol
if cfinder and cfinder.haskey("USE_ORSEC"):
use_overscan_row = cfinder.value("USE_ORSEC")
if cfinder and cfinder.haskey("SAVGOL"):
use_savgol = cfinder.value("SAVGOL")
# Set bias image, as desired
if bias_img is None:
bias = get_calibration_image(cfinder, "BIAS", bias)
else:
bias = bias_img
if bias is not False: #- it's an array
if bias.shape == rawimage.shape:
log.info("subtracting bias")
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(
bias.shape, rawimage.shape))
#- Check if this file uses amp names 1,2,3,4 (old) or A,B,C,D (new)
amp_ids = get_amp_ids(header)
#- Double check that we have the necessary keywords
missing_keywords = list()
for prefix in ['CCDSEC', 'BIASSEC']:
for amp in amp_ids:
key = prefix + amp
if not key in header:
log.error('No {} keyword in header'.format(key))
missing_keywords.append(key)
if len(missing_keywords) > 0:
raise KeyError("Missing keywords {}".format(
' '.join(missing_keywords)))
#- Output arrays
ny = 0
nx = 0
for amp in amp_ids:
yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp])
ny = max(ny, yy.stop)
nx = max(nx, xx.stop)
image = np.zeros((ny, nx))
readnoise = np.zeros_like(image)
#- Load mask
mask = get_calibration_image(cfinder, "MASK", mask)
if mask is False:
mask = np.zeros(image.shape, dtype=np.int32)
else:
if mask.shape != image.shape:
raise ValueError('shape mismatch mask {} != image {}'.format(
mask.shape, image.shape))
#- Load dark
dark = get_calibration_image(cfinder, "DARK", dark)
if dark is not False:
if dark.shape != image.shape:
log.error('shape mismatch dark {} != image {}'.format(
dark.shape, image.shape))
raise ValueError('shape mismatch dark {} != image {}'.format(
dark.shape, image.shape))
if cfinder and cfinder.haskey("EXPTIMEKEY"):
exptime_key = cfinder.value("EXPTIMEKEY")
log.info("Using exposure time keyword %s for dark normalization" %
exptime_key)
else:
exptime_key = "EXPTIME"
exptime = primary_header[exptime_key]
log.info("Multiplying dark by exptime %f" % (exptime))
dark *= exptime
for amp in amp_ids:
# Grab the sections
ov_col = parse_sec_keyword(header['BIASSEC' + amp])
if 'ORSEC' + amp in header.keys():
ov_row = parse_sec_keyword(header['ORSEC' + amp])
elif use_overscan_row:
log.error('No ORSEC{} keyword; not using overscan_row'.format(amp))
use_overscan_row = False
if nogain:
gain = 1.
else:
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
if cfinder and cfinder.haskey('GAIN' + amp):
gain = float(cfinder.value('GAIN' + amp))
log.info('Using GAIN{}={} from calibration data'.format(
amp, gain))
else:
gain = 1.0
log.warning(
'Missing keyword GAIN{} in header and nothing in calib data; using {}'
.format(amp, gain))
#- Add saturation level
if 'SATURLEV' + amp in header:
saturlev = header['SATURLEV' + amp] # in electrons
else:
if cfinder and cfinder.haskey('SATURLEV' + amp):
saturlev = float(cfinder.value('SATURLEV' + amp))
log.info('Using SATURLEV{}={} from calibration data'.format(
amp, saturlev))
else:
saturlev = 200000
log.warning(
'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'
.format(amp, saturlev))
# Generate the overscan images
raw_overscan_col = rawimage[ov_col].copy()
if use_overscan_row:
raw_overscan_row = rawimage[ov_row].copy()
overscan_row = np.zeros_like(raw_overscan_row)
# Remove overscan_col from overscan_row
raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy()
for row in range(raw_overscan_row.shape[0]):
o, r = _overscan(raw_overscan_squared[row])
overscan_row[row] = raw_overscan_row[row] - o
# Now remove the overscan_col
nrows = raw_overscan_col.shape[0]
log.info("nrows in overscan=%d" % nrows)
overscan_col = np.zeros(nrows)
rdnoise = np.zeros(nrows)
if (cfinder and cfinder.haskey('OVERSCAN' + amp)
and cfinder.value("OVERSCAN" + amp).upper()
== "PER_ROW") or overscan_per_row:
log.info(
"Subtracting overscan per row for amplifier %s of camera %s" %
(amp, camera))
for j in range(nrows):
if np.isnan(np.sum(overscan_col[j])):
log.warning(
"NaN values in row %d of overscan of amplifier %s of camera %s"
% (j, amp, camera))
continue
o, r = _overscan(raw_overscan_col[j])
#log.info("%d %f %f"%(j,o,r))
overscan_col[j] = o
rdnoise[j] = r
else:
log.info(
"Subtracting average overscan for amplifier %s of camera %s" %
(amp, camera))
o, r = _overscan(raw_overscan_col)
overscan_col += o
rdnoise += r
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan_col)
log.info("Median rdnoise and overscan= %f %f" %
(median_rdnoise, median_overscan))
kk = parse_sec_keyword(header['CCDSEC' + amp])
for j in range(nrows):
readnoise[kk][j] = rdnoise[j]
header['OVERSCN' + amp] = (median_overscan, 'ADUs (gain not applied)')
if gain != 1:
rdnoise_message = 'electrons (gain is applied)'
gain_message = 'e/ADU (gain applied to image)'
else:
rdnoise_message = 'ADUs (gain not applied)'
gain_message = 'gain not applied to image'
header['OBSRDN' + amp] = (median_rdnoise, rdnoise_message)
header['GAIN' + amp] = (gain, gain_message)
#- Warn/error if measured readnoise is very different from expected if exists
if 'RDNOISE' + amp in header:
expected_readnoise = header['RDNOISE' + amp]
if median_rdnoise < 0.5 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise < 0.9 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 2.0 * expected_readnoise:
log.error(
'Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 1.2 * expected_readnoise:
log.warning(
'Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'
.format(amp, median_rdnoise, expected_readnoise))
#else:
# log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
log.info("Measured readnoise for AMP %s = %f" % (amp, median_rdnoise))
#- subtract overscan from data region and apply gain
jj = parse_sec_keyword(header['DATASEC' + amp])
data = rawimage[jj].copy()
# Subtract columns
for k in range(nrows):
data[k] -= overscan_col[k]
# And now the rows
if use_overscan_row:
# Savgol?
if use_savgol:
log.info("Using savgol")
collapse_oscan_row = np.zeros(overscan_row.shape[1])
for col in range(overscan_row.shape[1]):
o, _ = _overscan(overscan_row[:, col])
collapse_oscan_row[col] = o
oscan_row = _savgol_clipped(collapse_oscan_row, niter=0)
oimg_row = np.outer(np.ones(data.shape[0]), oscan_row)
data -= oimg_row
else:
o, r = _overscan(overscan_row)
data -= o
#- apply saturlev (defined in ADU), prior to multiplication by gain
saturated = (rawimage[jj] >= saturlev)
mask[kk][saturated] |= ccdmask.SATURATED
#- ADC to electrons
image[kk] = data * gain
if not nocrosstalk:
#- apply cross-talk
# the ccd looks like :
# C D
# A B
# for cross talk, we need a symmetric 4x4 flip_matrix
# of coordinates ABCD giving flip of both axis
# when computing crosstalk of
# A B C D
#
# A AA AB AC AD
# B BA BB BC BD
# C CA CB CC CD
# D DA DB DC BB
# orientation_matrix_defines change of orientation
#
fip_axis_0 = np.array([[1, 1, -1, -1], [1, 1, -1, -1], [-1, -1, 1, 1],
[-1, -1, 1, 1]])
fip_axis_1 = np.array([[1, -1, 1, -1], [-1, 1, -1, 1], [1, -1, 1, -1],
[-1, 1, -1, 1]])
for a1 in range(len(amp_ids)):
amp1 = amp_ids[a1]
ii1 = parse_sec_keyword(header['CCDSEC' + amp1])
a1flux = image[ii1]
#a1mask=mask[ii1]
for a2 in range(len(amp_ids)):
if a1 == a2:
continue
amp2 = amp_ids[a2]
if cfinder is None: continue
if not cfinder.haskey("CROSSTALK%s%s" % (amp1, amp2)): continue
crosstalk = cfinder.value("CROSSTALK%s%s" % (amp1, amp2))
if crosstalk == 0.: continue
log.info("Correct for crosstalk=%f from AMP %s into %s" %
(crosstalk, amp1, amp2))
a12flux = crosstalk * a1flux.copy()
#a12mask=a1mask.copy()
if fip_axis_0[a1, a2] == -1:
a12flux = a12flux[::-1]
#a12mask=a12mask[::-1]
if fip_axis_1[a1, a2] == -1:
a12flux = a12flux[:, ::-1]
#a12mask=a12mask[:,::-1]
ii2 = parse_sec_keyword(header['CCDSEC' + amp2])
image[ii2] -= a12flux
# mask[ii2] |= a12mask (not sure we really need to propagate the mask)
#- Poisson noise variance (prior to dark subtraction and prior to pixel flat field)
#- This is biasing, but that's what we have for now
poisson_var = image.clip(0)
#- subtract dark after multiplication by gain
if dark is not False:
log.info("subtracting dark for amp %s" % amp)
image -= dark
#- Correct for dark trails if any
if not nodarktrail and cfinder is not None:
for amp in amp_ids:
if cfinder.haskey("DARKTRAILAMP%s" % amp):
amplitude = cfinder.value("DARKTRAILAMP%s" % amp)
width = cfinder.value("DARKTRAILWIDTH%s" % amp)
ii = _parse_sec_keyword(header["CCDSEC" + amp])
log.info(
"Removing dark trails for amplifier %s with width=%3.1f and amplitude=%5.4f"
% (amp, width, amplitude))
correct_dark_trail(image,
ii,
left=((amp == "B") | (amp == "D")),
width=width,
amplitude=amplitude)
#- Divide by pixflat image
pixflat = get_calibration_image(cfinder, "PIXFLAT", pixflat)
if pixflat is not False:
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(
pixflat.shape, image.shape))
almost_zero = 0.001
if np.all(pixflat > almost_zero):
image /= pixflat
readnoise /= pixflat
poisson_var /= pixflat**2
else:
good = (pixflat > almost_zero)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
poisson_var[good] /= pixflat[good]**2
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = poisson_var + readnoise**2
ivar = np.zeros(var.shape)
ivar[var > 0] = 1.0 / var[var > 0]
#- Ridiculously high readnoise is bad
mask[readnoise > 100] |= ccdmask.BADREADNOISE
if bkgsub:
bkg = _background(image, header)
image -= bkg
img = Image(image,
ivar=ivar,
mask=mask,
meta=header,
readnoise=readnoise,
camera=camera)
#- update img.mask to mask cosmic rays
if not nocosmic:
cosmics.reject_cosmic_rays(img,
nsig=cosmics_nsig,
cfudge=cosmics_cfudge,
c2fudge=cosmics_c2fudge)
if remove_scattered_light:
if psf_filename is None:
psf_filename = cfinder.findfile("PSF")
xyset = read_xytraceset(psf_filename)
img.pix -= model_scattered_light(img, xyset)
return img
def expand_config(self):
"""
config: desispec.quicklook.qlconfig.Config object
"""
self.log.debug("Building Full Configuration")
self.debuglevel = self.conf["Debuglevel"]
self.period = self.conf["Period"]
self.timeout = self.conf["Timeout"]
#- some global variables:
self.rawfile = findfile("raw",
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir)
self.fibermap = None
if self.flavor != 'bias' and self.flavor != 'dark':
self.fibermap = findfile("fibermap",
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir)
hdulist = pyfits.open(self.rawfile)
primary_header = hdulist[0].header
camera_header = hdulist[self.camera].header
self.program = primary_header['PROGRAM']
hdulist.close()
cfinder = CalibFinder([camera_header, primary_header])
if self.flavor == 'dark' or self.flavor == 'bias' or self.flavor == 'zero':
self.calibpsf = None
else:
self.calibpsf = cfinder.findfile("PSF")
if self.psfid is None:
self.psf_filename = findfile('psf',
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir)
else:
self.psf_filename = findfile('psf',
night=self.night,
expid=self.psfid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir)
if self.flavor == 'dark' or self.flavor == 'bias' or self.flavor == 'zero':
self.fiberflat = None
elif self.flatid is None and self.flavor != 'flat':
self.fiberflat = cfinder.findfile("FIBERFLAT")
elif self.flavor == 'flat':
self.fiberflat = findfile('fiberflat',
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir)
else:
self.fiberflat = findfile('fiberflat',
night=self.night,
expid=self.flatid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir)
#SE: QL no longer get references from a template or merged json
#- Get reference metrics from template json file
self.reference = None
outconfig = {}
outconfig['Night'] = self.night
outconfig['Program'] = self.program
outconfig['Flavor'] = self.flavor
outconfig['Camera'] = self.camera
outconfig['Expid'] = self.expid
outconfig['DumpIntermediates'] = self.dumpintermediates
outconfig['FiberMap'] = self.fibermap
outconfig['Period'] = self.period
pipeline = []
for ii, PA in enumerate(self.palist):
pipe = {}
pipe['PA'] = {
'ClassName': PA,
'ModuleName': self.pamodule,
'kwargs': self.paargs[PA]
}
pipe['QAs'] = []
for jj, QA in enumerate(self.qalist[PA]):
pipe_qa = {
'ClassName': QA,
'ModuleName': self.qamodule,
'kwargs': self.qaargs[QA]
}
pipe['QAs'].append(pipe_qa)
pipe['StepName'] = PA
pipeline.append(pipe)
outconfig['PipeLine'] = pipeline
outconfig['RawImage'] = self.rawfile
outconfig['singleqa'] = self.singqa
outconfig['Timeout'] = self.timeout
outconfig['FiberFlatFile'] = self.fiberflat
outconfig['PlotConfig'] = self.plotconf
#- Check if all the files exist for this QL configuraion
check_config(outconfig, self.singqa)
return outconfig
def get_sky(night,
expid,
exptime,
ftype="model",
redux="daily",
smoothing=100.0,
specsim_darksky=False,
nightly_darksky=False):
# AR ftype = "data" or "model"
# AR redux = "daily" or "blanc"
# AR if ftype = "data" : read the sky fibers from frame*fits + apply flat-field
# AR if ftype = "model": read the sky model from sky*.fits for the first fiber of each petal (see DJS email from 29Dec2020)
# AR those are in electron / angstrom
# AR to integrate over the decam-r-band, we need cameras b and r
if ftype not in ["data", "model"]:
sys.exit("ftype should be 'data' or 'model'")
sky = np.zeros(len(fullwave))
reduxdir = dailydir.replace("daily", redux)
# AR see AK email [desi-data 5218]
if redux == "blanc":
specs = ["0", "1", "3", "4", "5", "7", "8", "9"]
else:
specs = np.arange(10, dtype=int).astype(str)
for camera in ["b", "r", "z"]:
norm_cam = np.zeros(len(fullwave[cslice[camera]]))
sky_cam = np.zeros(len(fullwave[cslice[camera]]))
for spec in specs:
# AR data
if ftype == "data":
frfn = os.path.join(
reduxdir,
"exposures",
"{}".format(night),
expid,
"frame-{}{}-{}.fits".format(camera, spec, expid),
)
if not os.path.isfile(frfn):
print("Skipping non-existent {}".format(frfn))
continue
fr = read_frame(frfn, skip_resolution=True)
flfn = os.environ['DESI_SPECTRO_CALIB'] + '/' + CalibFinder(
[fr.meta]).data['FIBERFLAT']
# calib_flux [1e-17 erg/s/cm2/Angstrom] = uncalib_flux [electron/Angstrom] / (calibration_model * exptime [s])
# fl = read_fiberflat(flfn)
# No exptime correction.
# apply_fiberflat(fr, fl)
# AR cutting on sky fibers with at least one valid pixel.
ii = (fr.fibermap["OBJTYPE"]
== "SKY") & (fr.ivar.sum(axis=1) > 0)
# AR frame*fits are in e- / angstrom ; adding the N sky fibers
# sky_cam += fr.flux[ii, :].sum(axis=0)
# nspec += ii.sum()
# Ignores fiberflat corrected (fl), as includes e.g. fiberloss.
sky_cam += (fr.flux[ii, :] * fr.ivar[ii, :]).sum(axis=0)
norm_cam += fr.ivar[ii, :].sum(axis=0)
# AR model
if ftype == "model":
fn = os.path.join(
reduxdir,
"exposures",
"{}".format(night),
expid,
"sky-{}{}-{}.fits".format(camera, spec, expid),
)
if not os.path.isfile(fn):
print("Skipping non-existent {}".format(fn))
else:
print("Solving for {}".format(fn))
fd = fitsio.FITS(fn)
assert np.allclose(fullwave[cslice[camera]],
fd["WAVELENGTH"].read())
fd = fitsio.FITS(fn)
with fd as hdus:
exptime = hdus[0].read_header()['EXPTIME']
flux = hdus['SKY'].read()
ivar = hdus['IVAR'].read()
mask = hdus['MASK'].read()
# Verify that we have the expected wavelengths.
# assert np.allclose(detected[camera].wave, hdus['WAVELENGTH'].read())
# Verify that ivar is purely statistical.
# assert np.array_equal(ivar, hdus['STATIVAR'].read())
# Verify that the model has no masked pixels.
# assert np.all((mask == 0) & (ivar > 0))
# Verify that the sky model is constant.
# assert np.array_equal(np.max(ivar, axis=0), np.min(ivar, axis=0))
# assert np.allclose(np.max(flux, axis=0), np.min(flux, axis=0))
# There are actually small variations in flux!
# TODO: figure out where these variations come from.
# For now, take the median over fibers.
if fd["IVAR"][0, :][0].max() > 0:
sky_cam += fd["SKY"][0, :][
0] # AR reading the first fiber only
norm_cam += np.ones(len(fullwave[cslice[camera]]))
# sky_cam += np.median(flux, axis=0)
# norm_cam += np.ones(len(fullwave[cslice[camera]]))
fd.close()
# AR sky model flux in incident photon / angstrom / s
# if nspec > 0:
keep = norm_cam > 0
if keep.sum() > 0:
# [e/A/s] / throughput.
sky[cslice[camera]][keep] = (sky_cam[keep] / norm_cam[keep] /
exptime / spec_thru[camera][keep])
else:
print("{}-{}-{}: no spectra for {}".format(night, expid, camera,
ftype))
# AR sky model flux in erg / angstrom / s (using the photon energy in erg).
e_phot_erg = (constants.h.to(units.erg * units.s) *
constants.c.to(units.angstrom / units.s) /
(fullwave * units.angstrom))
sky *= e_phot_erg.value
# AR sky model flux in [erg / angstrom / s / cm**2 / arcsec**2].
sky /= (telap_cm2 * fiber_area_arcsec2)
if smoothing > 0.0:
sky = scipy.ndimage.gaussian_filter1d(sky, 100.)
# AR integrate over the DECam r-band
vfilter = filters.load_filters('bessell-V')
rfilter = filters.load_filters('decam2014-r')
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = vfilter.pad_spectrum(sky, fullwave, method="zero")
vmag = vfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = rfilter.pad_spectrum(sky, fullwave, method="zero")
rmag = rfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
if specsim_darksky:
fd = fitsio.FITS(fn)
# Dark Sky at given airmass (cnst. across spectrograph / camera).
simulator.atmosphere.airmass = fd['SKY'].read_header()['AIRMASS']
# Force below the horizon: dark sky.
simulator.atmosphere.moon.moon_zenith = 120. * u.deg
simulator.simulate()
# [1e-17 erg / (Angstrom arcsec2 cm2 s)].
sim_darksky = simulator.atmosphere.surface_brightness
sim_darksky *= 1.e-17
dsky_pad, dskywave_pad = rfilter.pad_spectrum(sim_darksky.value,
config.wavelength.value,
method="zero")
dsky_rmag = rfilter.get_ab_magnitudes(
dsky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
dskywave_pad * units.angstrom).as_array()[0][0]
sim_darksky = resample_flux(fullwave, config.wavelength.value,
sim_darksky.value)
sim_darksky *= u.erg / (u.cm**2 * u.s * u.angstrom * u.arcsec**2)
sky = np.clip(sky - sim_darksky.value, a_min=0.0, a_max=None)
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = vfilter.pad_spectrum(sky,
fullwave,
method="zero")
vmag_nodark = vfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = rfilter.pad_spectrum(sky,
fullwave,
method="zero")
rmag_nodark = rfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
return fullwave, sky, vmag, rmag, vmag_nodark, rmag_nodark
elif nightly_darksky:
from pkg_resources import resource_filename
if night in nightly_dsky_cache.keys():
return nightly_dsky_cache[night]
gfa_info = resource_filename('bgs-cmxsv', 'dat/sv1-exposures.fits')
gfa_info = Table.read(gfa_info)
bright_cut = 20.50
good_conds = (gfa_info['GFA_TRANSPARENCY_MED'] >
0.95) & (gfa_info['GFA_SKY_MAG_AB_MED'] >= bright_cut)
good_conds = gfa_info[good_conds]
expids = np.array([
np.int(x.split('/')[-1]) for x in glob.glob(
os.path.join(reduxdir, "exposures", "{}/00*".format(night)))
])
isgood = np.isin(good_conds['EXPID'], expids)
if np.any(isgood):
good_conds = good_conds[isgood]
best = good_conds['GFA_SKY_MAG_AB_MED'] == good_conds[
'GFA_SKY_MAG_AB_MED'].max()
print('Nightly Dark GFA r-mag for {}: {}'.format(
night, good_conds['GFA_SKY_MAG_AB_MED'].max()))
best_expid = good_conds[best]['EXPID'][0]
best_expid = '{:08d}'.format(best_expid)
darkwave, darksky, dark_vmag, dark_rmag = get_sky(
night,
best_expid,
exptime,
ftype="model",
redux="daily",
smoothing=0.0,
specsim_darksky=False,
nightly_darksky=False)
sky = np.clip(sky - darksky, a_min=0.0, a_max=None)
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = vfilter.pad_spectrum(sky,
fullwave,
method="zero")
vmag_nodark = vfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = rfilter.pad_spectrum(sky,
fullwave,
method="zero")
rmag_nodark = rfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
nightly_dsky_cache[
night] = fullwave, sky, vmag, rmag, vmag_nodark, rmag_nodark
else:
print(
'No nightly darksky available for night: {}. Defaulting to specsim.'
.format(night))
nightly_dsky_cache[night] = get_sky(night,
expid,
exptime,
ftype="model",
redux="daily",
smoothing=0.0,
specsim_darksky=True,
nightly_darksky=False)
return nightly_dsky_cache[night]
else:
pass
return fullwave, sky, vmag, rmag
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
t0 = time.time()
log = get_logger()
# guess if it is a preprocessed or a raw image
hdulist = fits.open(args.image)
is_input_preprocessed = ("IMAGE" in hdulist) & ("IVAR" in hdulist)
primary_header = hdulist[0].header
hdulist.close()
if is_input_preprocessed:
image = read_image(args.image)
else:
if args.camera is None:
print(
"ERROR: Need to specify camera to open a raw fits image (with all cameras in different fits HDUs)"
)
print(
"Try adding the option '--camera xx', with xx in {brz}{0-9}, like r7, or type 'desi_qproc --help' for more options"
)
sys.exit(12)
image = read_raw(args.image, args.camera, fill_header=[
1,
])
if args.auto:
log.debug("AUTOMATIC MODE")
try:
night = image.meta['NIGHT']
if not 'EXPID' in image.meta:
if 'EXPNUM' in image.meta:
log.warning('using EXPNUM {} for EXPID'.format(
image.meta['EXPNUM']))
image.meta['EXPID'] = image.meta['EXPNUM']
expid = image.meta['EXPID']
except KeyError as e:
log.error(
"Need at least NIGHT and EXPID (or EXPNUM) to run in auto mode. Retry without the --auto option."
)
log.error(str(e))
sys.exit(12)
indir = os.path.dirname(args.image)
if args.fibermap is None:
filename = '{}/fibermap-{:08d}.fits'.format(indir, expid)
if os.path.isfile(filename):
log.debug("auto-mode: found a fibermap, {}, using it!".format(
filename))
args.fibermap = filename
if args.output_preproc is None:
if not is_input_preprocessed:
args.output_preproc = '{}/preproc-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera.lower(), expid)
log.debug("auto-mode: will write preproc in " +
args.output_preproc)
else:
log.debug(
"auto-mode: will not write preproc because input is a preprocessed image"
)
if args.auto_output_dir != '.':
if not os.path.isdir(args.auto_output_dir):
log.debug("auto-mode: creating directory " +
args.auto_output_dir)
os.makedirs(args.auto_output_dir)
if args.output_preproc is not None:
write_image(args.output_preproc, image)
cfinder = None
if args.psf is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
args.psf = cfinder.findfile("PSF")
log.info(" Using PSF {}".format(args.psf))
tset = read_xytraceset(args.psf)
# add fibermap
if args.fibermap:
if os.path.isfile(args.fibermap):
fibermap = read_fibermap(args.fibermap)
else:
log.error("no fibermap file {}".format(args.fibermap))
fibermap = None
else:
fibermap = None
if "OBSTYPE" in image.meta:
obstype = image.meta["OBSTYPE"].upper()
image.meta["OBSTYPE"] = obstype # make sure it's upper case
qframe = None
else:
log.warning("No OBSTYPE keyword, trying to guess ...")
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
obstype = check_qframe_flavor(
qframe, input_flavor=image.meta["FLAVOR"]).upper()
image.meta["OBSTYPE"] = obstype
log.info("OBSTYPE = '{}'".format(obstype))
if args.auto:
# now set the things to do
if obstype == "SKY" or obstype == "TWILIGHT" or obstype == "SCIENCE":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.apply_fiberflat = True
args.skysub = True
args.output_skyframe = '{}/qsky-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.fluxcalib = True
args.outframe = '{}/qcframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
elif obstype == "ARC" or obstype == "TESTARC":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_lsf_sigma = True
elif obstype == "FLAT" or obstype == "TESTFLAT":
args.shift_psf = True
args.output_psf = '{}/psf-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.output_rawframe = '{}/qframe-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
args.compute_fiberflat = '{}/qfiberflat-{}-{:08d}.fits'.format(
args.auto_output_dir, args.camera, expid)
if args.shift_psf:
# using the trace shift script
if args.auto:
options = option_list({
"psf":
args.psf,
"image":
"dummy",
"outpsf":
"dummy",
"continuum": ((obstype == "FLAT") | (obstype == "TESTFLAT")),
"sky": ((obstype == "SCIENCE") | (obstype == "SKY"))
})
else:
options = option_list({
"psf": args.psf,
"image": "dummy",
"outpsf": "dummy"
})
tmp_args = trace_shifts_script.parse(options=options)
tset = trace_shifts_script.fit_trace_shifts(image=image, args=tmp_args)
qframe = qproc_boxcar_extraction(tset,
image,
width=args.width,
fibermap=fibermap)
if tset.meta is not None:
# add traceshift info in the qframe, this will be saved in the qframe header
if qframe.meta is None:
qframe.meta = dict()
for k in tset.meta.keys():
qframe.meta[k] = tset.meta[k]
if args.output_rawframe is not None:
write_qframe(args.output_rawframe, qframe)
log.info("wrote raw extracted frame in {}".format(
args.output_rawframe))
if args.compute_lsf_sigma:
tset = process_arc(qframe, tset, linelist=None, npoly=2, nbins=2)
if args.output_psf is not None:
for k in qframe.meta:
if k not in tset.meta:
tset.meta[k] = qframe.meta[k]
write_xytraceset(args.output_psf, tset)
if args.compute_fiberflat is not None:
fiberflat = qproc_compute_fiberflat(qframe)
#write_qframe(args.compute_fiberflat,qflat)
write_fiberflat(args.compute_fiberflat, fiberflat, header=qframe.meta)
log.info("wrote fiberflat in {}".format(args.compute_fiberflat))
if args.apply_fiberflat or args.input_fiberflat:
if args.input_fiberflat is None:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
try:
args.input_fiberflat = cfinder.findfile("FIBERFLAT")
except KeyError as e:
log.error("no FIBERFLAT for this spectro config")
sys.exit(12)
log.info("applying fiber flat {}".format(args.input_fiberflat))
flat = read_fiberflat(args.input_fiberflat)
qproc_apply_fiberflat(qframe, flat)
if args.skysub:
log.info("sky subtraction")
if args.output_skyframe is not None:
skyflux = qproc_sky_subtraction(qframe, return_skymodel=True)
sqframe = QFrame(qframe.wave, skyflux, np.ones(skyflux.shape))
write_qframe(args.output_skyframe, sqframe)
log.info("wrote sky model in {}".format(args.output_skyframe))
else:
qproc_sky_subtraction(qframe)
if args.fluxcalib:
if cfinder is None:
cfinder = CalibFinder([image.meta, primary_header])
# check for flux calib
if cfinder.haskey("FLUXCALIB"):
fluxcalib_filename = cfinder.findfile("FLUXCALIB")
fluxcalib = read_average_flux_calibration(fluxcalib_filename)
log.info("read average calib in {}".format(fluxcalib_filename))
seeing = qframe.meta["SEEING"]
airmass = qframe.meta["AIRMASS"]
exptime = qframe.meta["EXPTIME"]
exposure_calib = fluxcalib.value(seeing=seeing, airmass=airmass)
for q in range(qframe.nspec):
fiber_calib = np.interp(qframe.wave[q], fluxcalib.wave,
exposure_calib) * exptime
inv_calib = (fiber_calib > 0) / (fiber_calib +
(fiber_calib == 0))
qframe.flux[q] *= inv_calib
qframe.ivar[q] *= fiber_calib**2 * (fiber_calib > 0)
# add keyword in header giving the calibration factor applied at a reference wavelength
band = qframe.meta["CAMERA"].upper()[0]
if band == "B":
refwave = 4500
elif band == "R":
refwave = 6500
else:
refwave = 8500
calvalue = np.interp(refwave, fluxcalib.wave,
exposure_calib) * exptime
qframe.meta["CALWAVE"] = refwave
qframe.meta["CALVALUE"] = calvalue
else:
log.error(
"Cannot calibrate fluxes because no FLUXCALIB keywork in calibration files"
)
fibers = parse_fibers(args.fibers)
if fibers is None:
fibers = qframe.flux.shape[0]
else:
ii = np.arange(qframe.fibers.size)[np.in1d(qframe.fibers, fibers)]
if ii.size == 0:
log.error("no such fibers in frame,")
log.error("fibers are in range [{}:{}]".format(
qframe.fibers[0], qframe.fibers[-1] + 1))
sys.exit(12)
qframe = qframe[ii]
if args.outframe is not None:
write_qframe(args.outframe, qframe)
log.info("wrote {}".format(args.outframe))
t1 = time.time()
log.info("all done in {:3.1f} sec".format(t1 - t0))
if args.plot:
log.info("plotting {} spectra".format(qframe.wave.shape[0]))
import matplotlib.pyplot as plt
fig = plt.figure()
for i in range(qframe.wave.shape[0]):
j = (qframe.ivar[i] > 0)
plt.plot(qframe.wave[i, j], qframe.flux[i, j])
plt.grid()
plt.xlabel("wavelength")
plt.ylabel("flux")
plt.show()
def fluxcalib(night, verbose=False, overwrite=False):
"""Fit the standards on each petal and then perform flux-calibration.
desi_fit_stdstars --frames 00028833/frame-*.fits --skymodels 00028833/sky-*.fits \
--fiberflats $DESI_SPECTRO_CALIB/spec/sp3/fiberflat-sm4-*-20191108.fits \
--starmodels $DESI_BASIS_TEMPLATES/stdstar_templates_v2.2.fits --outfile stdstars-3-00028833.fits
desi_compute_fluxcalibration --infile 00028833/frame-b3-*.fits --sky 00028833/sky-b3-*.fits \
--fiberflat $DESI_SPECTRO_CALIB/spec/sp3/fiberflat-sm4-b-20191108.fits --models stdstars-3-00028833.fits \
--outfile fluxcalib-b3-00028833.fits --delta-color-cut 12
"""
starmodelfile = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
'stdstar_templates_v2.2.fits')
allexpiddir = glob(
os.path.join(outdir, 'exposures', str(night), '????????'))
for expiddir in allexpiddir:
expid = os.path.basename(expiddir)
# Process each spectrograph separately.
for spectro in np.arange(10):
framefiles = sorted(
glob(
desispec.io.findfile('frame',
night=night,
expid=int(expid),
spectrograph=spectro,
camera='*{}'.format(spectro),
specprod_dir=outdir)))
if len(framefiles) == 0:
print(
'No frame files found for spectrograph {}'.format(spectro))
continue
# Gather the calibration files.
havefiles = True if len(framefiles) == 3 else False
if havefiles:
fiberflatfiles, skymodelfiles = [], []
for framefile in framefiles:
hdr = fitsio.read_header(framefile)
calib = CalibFinder([hdr])
fiberflatfile = os.path.join(
os.getenv('DESI_SPECTRO_CALIB'),
calib.data['FIBERFLAT'])
skymodelfile = desispec.io.findfile(
'sky',
night=night,
expid=int(expid),
spectrograph=spectro,
camera=hdr['CAMERA'].strip(),
specprod_dir=outdir)
if not os.path.isfile(fiberflatfile) or not os.path.isfile(
skymodelfile):
havefiles = False
else:
fiberflatfiles.append(fiberflatfile)
skymodelfiles.append(skymodelfile)
# Fit the standard stars (per-spectrograph, all three cameras simultaneously).
stdstarsfile = desispec.io.findfile('stdstars',
night=night,
expid=int(expid),
spectrograph=spectro,
specprod_dir=outdir)
if os.path.isfile(stdstarsfile) and not overwrite:
print('File exists {}'.format(stdstarsfile))
else:
if havefiles:
cmd = 'desi_fit_stdstars --frames {framefiles} --skymodels {skymodelfiles} '
cmd += '--fiberflats {fiberflatfiles} --starmodels {starmodelfile} --ncpu 8 '
cmd += '--outfile {stdstarsfile}'
cmd = cmd.format(framefiles=' '.join(framefiles),
skymodelfiles=' '.join(skymodelfiles),
fiberflatfiles=' '.join(fiberflatfiles),
starmodelfile=starmodelfile,
stdstarsfile=stdstarsfile)
#print(cmd)
os.system(cmd)
# Perform flux-calibration (per-spectrograph).
if os.path.isfile(stdstarsfile) and havefiles:
for iframe in np.arange(len(framefiles)):
framefile = framefiles[iframe]
camera = fitsio.read_header(framefile)['CAMERA'].strip()
outcalibfile = desispec.io.findfile('fluxcalib',
night=night,
expid=int(expid),
spectrograph=spectro,
camera=camera,
specprod_dir=outdir)
if os.path.isfile(outcalibfile) and not overwrite:
print('File exists {}'.format(outcalibfile))
continue
## Gather the calibration files.
#skyfile = framefile.replace(outdir, reduxdir).replace('frame-', 'sky-')
#calib = CalibFinder([fitsio.read_header(framefile)])
#fiberflatfile = os.path.join(os.getenv('DESI_SPECTRO_CALIB'), calib.data['FIBERFLAT'])
cmd = 'desi_compute_fluxcalibration --infile {framefile} --sky {skyfile} '
cmd += '--fiberflat {fiberflatfile} --models {stdstarsfile} --outfile {outcalibfile} '
cmd += '--delta-color-cut 12'
cmd = cmd.format(framefile=framefiles[iframe],
skyfile=skymodelfiles[iframe],
fiberflatfile=fiberflatfiles[iframe],
stdstarsfile=stdstarsfile,
outcalibfile=outcalibfile)
#print(cmd)
os.system(cmd)
def main(args=None, comm=None):
if args is None:
args = parse()
# elif isinstance(args, (list, tuple)):
# args = parse(args)
log = get_logger()
start_mpi_connect = time.time()
if comm is not None:
#- Use the provided comm to determine rank and size
rank = comm.rank
size = comm.size
else:
#- Check MPI flags and determine the comm, rank, and size given the arguments
comm, rank, size = assign_mpi(do_mpi=args.mpi,
do_batch=args.batch,
log=log)
stop_mpi_connect = time.time()
#- Start timer; only print log messages from rank 0 (others are silent)
timer = desiutil.timer.Timer(silent=(rank > 0))
#- Fill in timing information for steps before we had the timer created
if args.starttime is not None:
timer.start('startup', starttime=args.starttime)
timer.stop('startup', stoptime=start_imports)
timer.start('imports', starttime=start_imports)
timer.stop('imports', stoptime=stop_imports)
timer.start('mpi_connect', starttime=start_mpi_connect)
timer.stop('mpi_connect', stoptime=stop_mpi_connect)
#- Freeze IERS after parsing args so that it doesn't bother if only --help
timer.start('freeze_iers')
desiutil.iers.freeze_iers()
timer.stop('freeze_iers')
#- Preflight checks
timer.start('preflight')
if rank > 0:
#- Let rank 0 fetch these, and then broadcast
args, hdr, camhdr = None, None, None
else:
args, hdr, camhdr = update_args_with_headers(args)
## Make sure badamps is formatted properly
if comm is not None and rank == 0 and args.badamps is not None:
args.badamps = validate_badamps(args.badamps)
if comm is not None:
args = comm.bcast(args, root=0)
hdr = comm.bcast(hdr, root=0)
camhdr = comm.bcast(camhdr, root=0)
known_obstype = [
'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT',
'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER'
]
if args.obstype not in known_obstype:
raise RuntimeError('obstype {} not in {}'.format(
args.obstype, known_obstype))
timer.stop('preflight')
#-------------------------------------------------------------------------
#- Create and submit a batch job if requested
if args.batch:
#exp_str = '{:08d}'.format(args.expid)
jobdesc = args.obstype.lower()
if args.obstype == 'SCIENCE':
# if not doing pre-stdstar fitting or stdstar fitting and if there is
# no flag stopping flux calibration, set job to poststdstar
if args.noprestdstarfit and args.nostdstarfit and (
not args.nofluxcalib):
jobdesc = 'poststdstar'
# elif told not to do std or post stdstar but the flag for prestdstar isn't set,
# then perform prestdstar
elif (not args.noprestdstarfit
) and args.nostdstarfit and args.nofluxcalib:
jobdesc = 'prestdstar'
#elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib):
# jobdesc = 'science'
scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\
jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\
batch_opts=args.batch_opts, timingfile=args.timingfile,
system_name=args.system_name)
err = 0
if not args.nosubmit:
err = subprocess.call(['sbatch', scriptfile])
sys.exit(err)
#-------------------------------------------------------------------------
#- Proceeding with running
#- What are we going to do?
if rank == 0:
log.info('----------')
log.info('Input {}'.format(args.input))
log.info('Night {} expid {}'.format(args.night, args.expid))
log.info('Obstype {}'.format(args.obstype))
log.info('Cameras {}'.format(args.cameras))
log.info('Output root {}'.format(desispec.io.specprod_root()))
log.info('----------')
#- Create output directories if needed
if rank == 0:
preprocdir = os.path.dirname(
findfile('preproc', args.night, args.expid, 'b0'))
expdir = os.path.dirname(
findfile('frame', args.night, args.expid, 'b0'))
os.makedirs(preprocdir, exist_ok=True)
os.makedirs(expdir, exist_ok=True)
#- Wait for rank 0 to make directories before proceeding
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Preproc
#- All obstypes get preprocessed
timer.start('fibermap')
#- Assemble fibermap for science exposures
fibermap = None
fibermap_ok = None
if rank == 0 and args.obstype == 'SCIENCE':
fibermap = findfile('fibermap', args.night, args.expid)
if not os.path.exists(fibermap):
tmp = findfile('preproc', args.night, args.expid, 'b0')
preprocdir = os.path.dirname(tmp)
fibermap = os.path.join(preprocdir, os.path.basename(fibermap))
log.info('Creating fibermap {}'.format(fibermap))
cmd = 'assemble_fibermap -n {} -e {} -o {}'.format(
args.night, args.expid, fibermap)
if args.badamps is not None:
cmd += ' --badamps={}'.format(args.badamps)
runcmd(cmd, inputs=[], outputs=[fibermap])
fibermap_ok = os.path.exists(fibermap)
#- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail
#- these are well known failures with no other solution, so for those, just force creation
#- of a fibermap with null coordinate information
if not fibermap_ok and int(args.night) < 20200310:
log.info(
"Since night is before 20200310, trying to force fibermap creation without coords file"
)
cmd += ' --force'
runcmd(cmd, inputs=[], outputs=[fibermap])
fibermap_ok = os.path.exists(fibermap)
#- If assemble_fibermap failed and obstype is SCIENCE, exit now
if comm is not None:
fibermap_ok = comm.bcast(fibermap_ok, root=0)
if args.obstype == 'SCIENCE' and not fibermap_ok:
sys.stdout.flush()
if rank == 0:
log.critical(
'assemble_fibermap failed for science exposure; exiting now')
sys.exit(13)
#- Wait for rank 0 to make fibermap if needed
if comm is not None:
fibermap = comm.bcast(fibermap, root=0)
timer.stop('fibermap')
if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit):
timer.start('preproc')
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
outfile = findfile('preproc', args.night, args.expid, camera)
outdir = os.path.dirname(outfile)
cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format(
args.input, outfile, outdir, camera)
if args.scattered_light:
cmd += " --scattered-light"
if fibermap is not None:
cmd += " --fibermap {}".format(fibermap)
if not args.obstype in ['ARC']: # never model variance for arcs
if not args.no_model_pixel_variance:
cmd += " --model-variance"
runcmd(cmd, inputs=[args.input], outputs=[outfile])
timer.stop('preproc')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Get input PSFs
timer.start('findpsf')
input_psf = dict()
if rank == 0:
for camera in args.cameras:
if args.psf is not None:
input_psf[camera] = args.psf
elif args.calibnight is not None:
# look for a psfnight psf for this calib night
psfnightfile = findfile('psfnight', args.calibnight,
args.expid, camera)
if not os.path.isfile(psfnightfile):
log.error("no {}".format(psfnightfile))
raise IOError("no {}".format(psfnightfile))
input_psf[camera] = psfnightfile
else:
# look for a psfnight psf
psfnightfile = findfile('psfnight', args.night, args.expid,
camera)
if os.path.isfile(psfnightfile):
input_psf[camera] = psfnightfile
elif args.most_recent_calib:
nightfile = find_most_recent(args.night,
file_type='psfnight')
if nightfile is None:
input_psf[camera] = findcalibfile(
[hdr, camhdr[camera]], 'PSF')
else:
input_psf[camera] = nightfile
else:
input_psf[camera] = findcalibfile([hdr, camhdr[camera]],
'PSF')
log.info("Will use input PSF : {}".format(input_psf[camera]))
if comm is not None:
input_psf = comm.bcast(input_psf, root=0)
timer.stop('findpsf')
#-------------------------------------------------------------------------
#- Traceshift
if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \
( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):
timer.start('traceshift')
if rank == 0 and args.traceshift:
log.info('Starting traceshift at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
preprocfile = findfile('preproc', args.night, args.expid, camera)
inpsf = input_psf[camera]
outpsf = findfile('psf', args.night, args.expid, camera)
if not os.path.isfile(outpsf):
if args.traceshift:
cmd = "desi_compute_trace_shifts"
cmd += " -i {}".format(preprocfile)
cmd += " --psf {}".format(inpsf)
cmd += " --outpsf {}".format(outpsf)
cmd += " --degxx 2 --degxy 0"
if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']:
cmd += " --continuum"
else:
cmd += " --degyx 2 --degyy 0"
if args.obstype in ['SCIENCE', 'SKY']:
cmd += ' --sky'
else:
cmd = "ln -s {} {}".format(inpsf, outpsf)
runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
else:
log.info("PSF {} exists".format(outpsf))
timer.stop('traceshift')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- PSF
#- MPI parallelize this step
if args.obstype in ['ARC', 'TESTARC']:
timer.start('arc_traceshift')
if rank == 0:
log.info('Starting traceshift before specex PSF fit at {}'.format(
time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
preprocfile = findfile('preproc', args.night, args.expid, camera)
inpsf = input_psf[camera]
outpsf = findfile('psf', args.night, args.expid, camera)
outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf")
if not os.path.isfile(outpsf):
cmd = "desi_compute_trace_shifts"
cmd += " -i {}".format(preprocfile)
cmd += " --psf {}".format(inpsf)
cmd += " --outpsf {}".format(outpsf)
cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0"
cmd += ' --arc-lamps'
runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
else:
log.info("PSF {} exists".format(outpsf))
timer.stop('arc_traceshift')
if comm is not None:
comm.barrier()
timer.start('psf')
if rank == 0:
log.info('Starting specex PSF fitting at {}'.format(
time.asctime()))
if rank > 0:
cmds = inputs = outputs = None
else:
cmds = dict()
inputs = dict()
outputs = dict()
for camera in args.cameras:
preprocfile = findfile('preproc', args.night, args.expid,
camera)
tmpname = findfile('psf', args.night, args.expid, camera)
inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf")
outpsf = replace_prefix(tmpname, "psf", "fit-psf")
log.info("now run specex psf fit")
cmd = 'desi_compute_psf'
cmd += ' --input-image {}'.format(preprocfile)
cmd += ' --input-psf {}'.format(inpsf)
cmd += ' --output-psf {}'.format(outpsf)
# look for fiber blacklist
cfinder = CalibFinder([hdr, camhdr[camera]])
blacklistkey = "FIBERBLACKLIST"
if not cfinder.haskey(blacklistkey) and cfinder.haskey(
"BROKENFIBERS"):
log.warning(
"BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
)
blacklistkey = "BROKENFIBERS"
if cfinder.haskey(blacklistkey):
blacklist = cfinder.value(blacklistkey)
cmd += ' --broken-fibers {}'.format(blacklist)
if rank == 0:
log.warning('broken fibers: {}'.format(blacklist))
if not os.path.exists(outpsf):
cmds[camera] = cmd
inputs[camera] = [preprocfile, inpsf]
outputs[camera] = [
outpsf,
]
if comm is not None:
cmds = comm.bcast(cmds, root=0)
inputs = comm.bcast(inputs, root=0)
outputs = comm.bcast(outputs, root=0)
#- split communicator by 20 (number of bundles)
group_size = 20
if (rank == 0) and (size % group_size != 0):
log.warning(
'MPI size={} should be evenly divisible by {}'.format(
size, group_size))
group = rank // group_size
num_groups = (size + group_size - 1) // group_size
comm_group = comm.Split(color=group)
if rank == 0:
log.info(
f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}'
)
for i in range(group, len(args.cameras), num_groups):
camera = args.cameras[i]
if camera in cmds:
cmdargs = cmds[camera].split()[1:]
cmdargs = desispec.scripts.specex.parse(cmdargs)
if comm_group.rank == 0:
print('RUNNING: {}'.format(cmds[camera]))
t0 = time.time()
timestamp = time.asctime()
log.info(
f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}'
)
try:
desispec.scripts.specex.main(cmdargs, comm=comm_group)
except Exception as e:
if comm_group.rank == 0:
log.error(
f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}'
)
log.error('FAILED: {}'.format(cmds[camera]))
log.error(e)
if comm_group.rank == 0:
specex_time = time.time() - t0
log.info(
f'specex fit for {camera} took {specex_time:.1f} seconds'
)
comm.barrier()
else:
log.warning(
'fitting PSFs without MPI parallelism; this will be SLOW')
for camera in args.cameras:
if camera in cmds:
runcmd(cmds[camera],
inputs=inputs[camera],
outputs=outputs[camera])
if comm is not None:
comm.barrier()
# loop on all cameras and interpolate bad fibers
for camera in args.cameras[rank::size]:
t0 = time.time()
log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers')
# look for fiber blacklist
cfinder = CalibFinder([hdr, camhdr[camera]])
blacklistkey = "FIBERBLACKLIST"
if not cfinder.haskey(blacklistkey) and cfinder.haskey(
"BROKENFIBERS"):
log.warning(
"BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
)
blacklistkey = "BROKENFIBERS"
if cfinder.haskey(blacklistkey):
fiberblacklist = cfinder.value(blacklistkey)
tmpname = findfile('psf', args.night, args.expid, camera)
inpsf = replace_prefix(tmpname, "psf", "fit-psf")
outpsf = replace_prefix(tmpname, "psf",
"fit-psf-fixed-blacklisted")
if os.path.isfile(inpsf) and not os.path.isfile(outpsf):
cmd = 'desi_interpolate_fiber_psf'
cmd += ' --infile {}'.format(inpsf)
cmd += ' --outfile {}'.format(outpsf)
cmd += ' --fibers {}'.format(fiberblacklist)
log.info(
'For camera {} interpolating PSF for broken fibers: {}'
.format(camera, fiberblacklist))
runcmd(cmd, inputs=[inpsf], outputs=[outpsf])
if os.path.isfile(outpsf):
os.rename(
inpsf,
inpsf.replace("fit-psf",
"fit-psf-before-blacklisted-fix"))
subprocess.call('cp {} {}'.format(outpsf, inpsf),
shell=True)
dt = time.time() - t0
log.info(
f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec')
timer.stop('psf')
#-------------------------------------------------------------------------
#- Merge PSF of night if applicable
#if args.obstype in ['ARC']:
if False:
if rank == 0:
for camera in args.cameras:
psfnightfile = findfile('psfnight', args.night, args.expid,
camera)
if not os.path.isfile(
psfnightfile
): # we still don't have a psf night, see if we can compute it ...
psfs = glob.glob(
findfile('psf', args.night, args.expid,
camera).replace("psf", "fit-psf").replace(
str(args.expid), "*"))
log.info(
"Number of PSF for night={} camera={} = {}".format(
args.night, camera, len(psfs)))
if len(psfs) > 4: # lets do it!
log.info("Computing psfnight ...")
dirname = os.path.dirname(psfnightfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
desispec.scripts.specex.mean_psf(psfs, psfnightfile)
if os.path.isfile(psfnightfile): # now use this one
input_psf[camera] = psfnightfile
#-------------------------------------------------------------------------
#- Extract
#- This is MPI parallel so handle a bit differently
# maybe add ARC and TESTARC too
if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \
( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):
timer.start('extract')
if rank == 0:
log.info('Starting extractions at {}'.format(time.asctime()))
if rank > 0:
cmds = inputs = outputs = None
else:
cmds = dict()
inputs = dict()
outputs = dict()
for camera in args.cameras:
cmd = 'desi_extract_spectra'
#- Based on data from SM1-SM8, looking at central and edge fibers
#- with in mind overlapping arc lamps lines
if camera.startswith('b'):
cmd += ' -w 3600.0,5800.0,0.8'
elif camera.startswith('r'):
cmd += ' -w 5760.0,7620.0,0.8'
elif camera.startswith('z'):
cmd += ' -w 7520.0,9824.0,0.8'
preprocfile = findfile('preproc', args.night, args.expid,
camera)
psffile = findfile('psf', args.night, args.expid, camera)
framefile = findfile('frame', args.night, args.expid, camera)
cmd += ' -i {}'.format(preprocfile)
cmd += ' -p {}'.format(psffile)
cmd += ' -o {}'.format(framefile)
cmd += ' --psferr 0.1'
if args.obstype == 'SCIENCE' or args.obstype == 'SKY':
if rank == 0:
log.info('Include barycentric correction')
cmd += ' --barycentric-correction'
if not os.path.exists(framefile):
cmds[camera] = cmd
inputs[camera] = [preprocfile, psffile]
outputs[camera] = [
framefile,
]
#- TODO: refactor/combine this with PSF comm splitting logic
if comm is not None:
cmds = comm.bcast(cmds, root=0)
inputs = comm.bcast(inputs, root=0)
outputs = comm.bcast(outputs, root=0)
#- split communicator by 20 (number of bundles)
extract_size = 20
if (rank == 0) and (size % extract_size != 0):
log.warning(
'MPI size={} should be evenly divisible by {}'.format(
size, extract_size))
extract_group = rank // extract_size
num_extract_groups = (size + extract_size - 1) // extract_size
comm_extract = comm.Split(color=extract_group)
for i in range(extract_group, len(args.cameras),
num_extract_groups):
camera = args.cameras[i]
if camera in cmds:
cmdargs = cmds[camera].split()[1:]
extract_args = desispec.scripts.extract.parse(cmdargs)
if comm_extract.rank == 0:
print('RUNNING: {}'.format(cmds[camera]))
desispec.scripts.extract.main_mpi(extract_args,
comm=comm_extract)
comm.barrier()
else:
log.warning(
'running extractions without MPI parallelism; this will be SLOW'
)
for camera in args.cameras:
if camera in cmds:
runcmd(cmds[camera],
inputs=inputs[camera],
outputs=outputs[camera])
timer.stop('extract')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Fiberflat
if args.obstype in ['FLAT', 'TESTFLAT']:
timer.start('fiberflat')
if rank == 0:
log.info('Starting fiberflats at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
fiberflatfile = findfile('fiberflat', args.night, args.expid,
camera)
cmd = "desi_compute_fiberflat"
cmd += " -i {}".format(framefile)
cmd += " -o {}".format(fiberflatfile)
runcmd(cmd, inputs=[
framefile,
], outputs=[
fiberflatfile,
])
timer.stop('fiberflat')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Average and auto-calib fiberflats of night if applicable
#if args.obstype in ['FLAT']:
if False:
if rank == 0:
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expid, args.cameras[0])
fiberflatdirname = os.path.dirname(fiberflatnightfile)
if not os.path.isfile(fiberflatnightfile) and len(
args.cameras
) >= 6: # we still don't have them, see if we can compute them, but need at least 2 spectros ...
flats = glob.glob(
findfile('fiberflat', args.night, args.expid,
"b0").replace(str(args.expid),
"*").replace("b0", "*"))
log.info("Number of fiberflat for night {} = {}".format(
args.night, len(flats)))
if len(flats) >= 3 * 4 * len(
args.cameras
): # lets do it! (3 exposures x 4 lamps x N cameras)
log.info(
"Computing fiberflatnight per lamp and camera ...")
tmpdir = os.path.join(fiberflatdirname, "tmp")
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
log.info(
"First average measurements per camera and per lamp")
average_flats = dict()
for camera in args.cameras:
# list of flats for this camera
flats_for_this_camera = []
for flat in flats:
if flat.find(camera) >= 0:
flats_for_this_camera.append(flat)
#log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera))
#sys.exit(12)
# average per lamp (and camera)
average_flats[camera] = list()
for lampbox in range(4):
ofile = os.path.join(
tmpdir,
"fiberflatnight-camera-{}-lamp-{}.fits".format(
camera, lampbox))
if not os.path.isfile(ofile):
log.info(
"Average flat for camera {} and lamp box #{}"
.format(camera, lampbox))
pg = "CALIB DESI-CALIB-0{} LEDs only".format(
lampbox)
cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format(
pg, ofile)
for flat in flats_for_this_camera:
cmd += " {} ".format(flat)
runcmd(cmd,
inputs=flats_for_this_camera,
outputs=[
ofile,
])
if os.path.isfile(ofile):
average_flats[camera].append(ofile)
else:
log.info("Will use existing {}".format(ofile))
average_flats[camera].append(ofile)
log.info(
"Auto-calibration across lamps and spectro per camera arm (b,r,z)"
)
for camera_arm in ["b", "r", "z"]:
cameras_for_this_arm = []
flats_for_this_arm = []
for camera in args.cameras:
if camera[0].lower() == camera_arm:
cameras_for_this_arm.append(camera)
if camera in average_flats:
for flat in average_flats[camera]:
flats_for_this_arm.append(flat)
cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format(
args.night, camera_arm)
for flat in flats_for_this_arm:
cmd += " {} ".format(flat)
runcmd(cmd, inputs=flats_for_this_arm, outputs=[])
log.info("Done with fiber flats per night")
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Get input fiberflat
if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat):
timer.start('find_fiberflat')
input_fiberflat = dict()
if rank == 0:
for camera in args.cameras:
if args.fiberflat is not None:
input_fiberflat[camera] = args.fiberflat
elif args.calibnight is not None:
# look for a fiberflatnight for this calib night
fiberflatnightfile = findfile('fiberflatnight',
args.calibnight, args.expid,
camera)
if not os.path.isfile(fiberflatnightfile):
log.error("no {}".format(fiberflatnightfile))
raise IOError("no {}".format(fiberflatnightfile))
input_fiberflat[camera] = fiberflatnightfile
else:
# look for a fiberflatnight fiberflat
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expid, camera)
if os.path.isfile(fiberflatnightfile):
input_fiberflat[camera] = fiberflatnightfile
elif args.most_recent_calib:
nightfile = find_most_recent(
args.night, file_type='fiberflatnight')
if nightfile is None:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
else:
input_fiberflat[camera] = nightfile
else:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
log.info("Will use input FIBERFLAT: {}".format(
input_fiberflat[camera]))
if comm is not None:
input_fiberflat = comm.bcast(input_fiberflat, root=0)
timer.stop('find_fiberflat')
#-------------------------------------------------------------------------
#- Apply fiberflat and write fframe file
if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \
( not args.nofiberflat ) and (not args.noprestdstarfit):
timer.start('apply_fiberflat')
if rank == 0:
log.info('Applying fiberflat at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
fframefile = findfile('fframe', args.night, args.expid, camera)
if not os.path.exists(fframefile):
framefile = findfile('frame', args.night, args.expid, camera)
fr = desispec.io.read_frame(framefile)
flatfilename = input_fiberflat[camera]
if flatfilename is not None:
ff = desispec.io.read_fiberflat(flatfilename)
fr.meta['FIBERFLT'] = desispec.io.shorten_filename(
flatfilename)
apply_fiberflat(fr, ff)
fframefile = findfile('fframe', args.night, args.expid,
camera)
desispec.io.write_frame(fframefile, fr)
else:
log.warning(
"Missing fiberflat for camera {}".format(camera))
timer.stop('apply_fiberflat')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Select random sky fibers (inplace update of frame file)
#- TODO: move this to a function somewhere
#- TODO: this assigns different sky fibers to each frame of same spectrograph
if (args.obstype in [
'SKY', 'SCIENCE'
]) and (not args.noskysub) and (not args.noprestdstarfit):
timer.start('picksky')
if rank == 0:
log.info('Picking sky fibers at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
orig_frame = desispec.io.read_frame(framefile)
#- Make a copy so that we can apply fiberflat
fr = deepcopy(orig_frame)
if np.any(fr.fibermap['OBJTYPE'] == 'SKY'):
log.info('{} sky fibers already set; skipping'.format(
os.path.basename(framefile)))
continue
#- Apply fiberflat then select random fibers below a flux cut
flatfilename = input_fiberflat[camera]
if flatfilename is None:
log.error("No fiberflat for {}".format(camera))
continue
ff = desispec.io.read_fiberflat(flatfilename)
apply_fiberflat(fr, ff)
sumflux = np.sum(fr.flux, axis=1)
fluxcut = np.percentile(sumflux, 30)
iisky = np.where(sumflux < fluxcut)[0]
iisky = np.random.choice(iisky, size=100, replace=False)
#- Update fibermap or original frame and write out
orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY'
orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY
desispec.io.write_frame(framefile, orig_frame)
timer.stop('picksky')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Sky subtraction
if args.obstype in [
'SCIENCE', 'SKY'
] and (not args.noskysub) and (not args.noprestdstarfit):
timer.start('skysub')
if rank == 0:
log.info('Starting sky subtraction at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
hdr = fitsio.read_header(framefile, 'FLUX')
fiberflatfile = input_fiberflat[camera]
if fiberflatfile is None:
log.error("No fiberflat for {}".format(camera))
continue
skyfile = findfile('sky', args.night, args.expid, camera)
cmd = "desi_compute_sky"
cmd += " -i {}".format(framefile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --o {}".format(skyfile)
if args.no_extra_variance:
cmd += " --no-extra-variance"
if not args.no_sky_wavelength_adjustment:
cmd += " --adjust-wavelength"
if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf"
runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[
skyfile,
])
#- sframe = flatfielded sky-subtracted but not flux calibrated frame
#- Note: this re-reads and re-does steps previously done for picking
#- sky fibers; desi_proc is about human efficiency,
#- not I/O or CPU efficiency...
sframefile = desispec.io.findfile('sframe', args.night, args.expid,
camera)
if not os.path.exists(sframefile):
frame = desispec.io.read_frame(framefile)
fiberflat = desispec.io.read_fiberflat(fiberflatfile)
sky = desispec.io.read_sky(skyfile)
apply_fiberflat(frame, fiberflat)
subtract_sky(frame, sky, apply_throughput_correction=True)
frame.meta['IN_SKY'] = shorten_filename(skyfile)
frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile)
desispec.io.write_frame(sframefile, frame)
timer.stop('skysub')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Standard Star Fitting
if args.obstype in ['SCIENCE',] and \
(not args.noskysub ) and \
(not args.nostdstarfit) :
timer.start('stdstarfit')
if rank == 0:
log.info('Starting flux calibration at {}'.format(time.asctime()))
#- Group inputs by spectrograph
framefiles = dict()
skyfiles = dict()
fiberflatfiles = dict()
night, expid = args.night, args.expid #- shorter
for camera in args.cameras:
sp = int(camera[1])
if sp not in framefiles:
framefiles[sp] = list()
skyfiles[sp] = list()
fiberflatfiles[sp] = list()
framefiles[sp].append(findfile('frame', night, expid, camera))
skyfiles[sp].append(findfile('sky', night, expid, camera))
fiberflatfiles[sp].append(input_fiberflat[camera])
#- Hardcoded stdstar model version
starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
'stdstar_templates_v2.2.fits')
#- Fit stdstars per spectrograph (not per-camera)
spectro_nums = sorted(framefiles.keys())
## for sp in spectro_nums[rank::size]:
for i in range(rank, len(spectro_nums), size):
sp = spectro_nums[i]
stdfile = findfile('stdstars', night, expid, spectrograph=sp)
cmd = "desi_fit_stdstars"
cmd += " --frames {}".format(' '.join(framefiles[sp]))
cmd += " --skymodels {}".format(' '.join(skyfiles[sp]))
cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp]))
cmd += " --starmodels {}".format(starmodels)
cmd += " --outfile {}".format(stdfile)
cmd += " --delta-color 0.1"
if args.maxstdstars is not None:
cmd += " --maxstdstars {}".format(args.maxstdstars)
inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp]
runcmd(cmd, inputs=inputs, outputs=[stdfile])
timer.stop('stdstarfit')
if comm is not None:
comm.barrier()
# -------------------------------------------------------------------------
# - Flux calibration
if args.obstype in ['SCIENCE'] and \
(not args.noskysub) and \
(not args.nofluxcalib):
timer.start('fluxcalib')
night, expid = args.night, args.expid #- shorter
#- Compute flux calibration vectors per camera
for camera in args.cameras[rank::size]:
framefile = findfile('frame', night, expid, camera)
skyfile = findfile('sky', night, expid, camera)
spectrograph = int(camera[1])
stdfile = findfile('stdstars',
night,
expid,
spectrograph=spectrograph)
calibfile = findfile('fluxcalib', night, expid, camera)
fiberflatfile = input_fiberflat[camera]
cmd = "desi_compute_fluxcalibration"
cmd += " --infile {}".format(framefile)
cmd += " --sky {}".format(skyfile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --models {}".format(stdfile)
cmd += " --outfile {}".format(calibfile)
cmd += " --delta-color-cut 0.1"
inputs = [framefile, skyfile, fiberflatfile, stdfile]
runcmd(cmd, inputs=inputs, outputs=[
calibfile,
])
timer.stop('fluxcalib')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Applying flux calibration
if args.obstype in [
'SCIENCE',
] and (not args.noskysub) and (not args.nofluxcalib):
night, expid = args.night, args.expid #- shorter
timer.start('applycalib')
if rank == 0:
log.info('Starting cframe file creation at {}'.format(
time.asctime()))
for camera in args.cameras[rank::size]:
framefile = findfile('frame', night, expid, camera)
skyfile = findfile('sky', night, expid, camera)
spectrograph = int(camera[1])
stdfile = findfile('stdstars',
night,
expid,
spectrograph=spectrograph)
calibfile = findfile('fluxcalib', night, expid, camera)
cframefile = findfile('cframe', night, expid, camera)
fiberflatfile = input_fiberflat[camera]
cmd = "desi_process_exposure"
cmd += " --infile {}".format(framefile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --sky {}".format(skyfile)
cmd += " --calib {}".format(calibfile)
cmd += " --outfile {}".format(cframefile)
cmd += " --cosmics-nsig 6"
if args.no_xtalk:
cmd += " --no-xtalk"
inputs = [framefile, fiberflatfile, skyfile, calibfile]
runcmd(cmd, inputs=inputs, outputs=[
cframefile,
])
if comm is not None:
comm.barrier()
timer.stop('applycalib')
#-------------------------------------------------------------------------
#- Wrap up
# if rank == 0:
# report = timer.report()
# log.info('Rank 0 timing report:\n' + report)
if comm is not None:
timers = comm.gather(timer, root=0)
else:
timers = [
timer,
]
if rank == 0:
stats = desiutil.timer.compute_stats(timers)
log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2))
if args.timingfile:
if os.path.exists(args.timingfile):
with open(args.timingfile) as fx:
previous_stats = json.load(fx)
#- augment previous_stats with new entries, but don't overwrite old
for name in stats:
if name not in previous_stats:
previous_stats[name] = stats[name]
stats = previous_stats
tmpfile = args.timingfile + '.tmp'
with open(tmpfile, 'w') as fx:
json.dump(stats, fx, indent=2)
os.rename(tmpfile, args.timingfile)
if rank == 0:
log.info('All done at {}'.format(time.asctime()))
def cal_rdnoise(hdul, camera, ccd_calibration_filename, use_overscan_row,
overscan_per_row, use_active):
if True:
hdul_this = hdul[camera]
header = hdul_this.header
rawimage = hdul_this.data
# works! preproc.preproc(hdul['B0'].data,hdul['B0'].header,h1)
amp_ids = preproc.get_amp_ids(header)
#- Output arrays
ny = 0
nx = 0
for amp in amp_ids:
yy, xx = parse_sec_keyword(header['CCDSEC%s' % amp])
ny = max(ny, yy.stop)
nx = max(nx, xx.stop)
image = np.zeros((ny, nx))
readnoise = np.zeros_like(image)
nogain = False
cfinder = None
if ccd_calibration_filename is not False:
cfinder = CalibFinder([header, primary_header],
yaml_file=ccd_calibration_filename)
for amp in amp_ids:
# Grab the sections
ov_col = parse_sec_keyword(header['BIASSEC' + amp])
ov_row = parse_sec_keyword(header['ORSEC' + amp])
if use_active:
if amp == 'A' or amp == 'D':
ov_col2 = np.s_[ov_col[1].start:ov_col[1].stop, 0:100]
else:
ov_col2 = np.s_[ov_col[1].start:ov_col[1].stop, 0:100]
import pdb
pdb.set_trace()
if 'ORSEC' + amp in header.keys():
ov_row = parse_sec_keyword(header['ORSEC' + amp])
elif use_overscan_row:
log.error(
'No ORSEC{} keyword; not using overscan_row'.format(amp))
use_overscan_row = False
if nogain:
gain = 1.
else:
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN' + amp in header:
gain = header['GAIN' + amp] #- gain = electrons / ADU
else:
if cfinder and cfinder.haskey('GAIN' + amp):
gain = float(cfinder.value('GAIN' + amp))
log.info(
'Using GAIN{}={} from calibration data'.format(
amp, gain))
else:
gain = 1.0
log.warning(
'Missing keyword GAIN{} in header and nothing in calib data; using {}'
.format(amp, gain))
#- Add saturation level
if 'SATURLEV' + amp in header:
saturlev = header['SATURLEV' + amp] # in electrons
else:
if cfinder and cfinder.haskey('SATURLEV' + amp):
saturlev = float(cfinder.value('SATURLEV' + amp))
log.info(
'Using SATURLEV{}={} from calibration data'.format(
amp, saturlev))
else:
saturlev = 200000
log.warning(
'Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'
.format(amp, saturlev))
# Generate the overscan images
raw_overscan_col = rawimage[ov_col].copy() # 2064*64
if use_overscan_row:
raw_overscan_row = rawimage[ov_row].copy() # 32*2057
overscan_row = np.zeros_like(raw_overscan_row)
# Remove overscan_col from overscan_row
raw_overscan_squared = rawimage[ov_row[0],
ov_col[1]].copy() # 32*64
for row in range(raw_overscan_row.shape[0]):
o, r = _overscan(raw_overscan_squared[row])
overscan_row[row] = raw_overscan_row[row] - o
# Now remove the overscan_col
nrows = raw_overscan_col.shape[0]
log.info("nrows in overscan=%d" % nrows)
overscan_col = np.zeros(nrows)
rdnoise = np.zeros(nrows)
if (cfinder and cfinder.haskey('OVERSCAN' + amp)
and cfinder.value("OVERSCAN" + amp).upper()
== "PER_ROW") or overscan_per_row:
log.info(
"Subtracting overscan per row for amplifier %s of camera %s"
% (amp, camera))
for j in range(nrows):
if np.isnan(np.sum(overscan_col[j])):
log.warning(
"NaN values in row %d of overscan of amplifier %s of camera %s"
% (j, amp, camera))
continue
o, r = _overscan(raw_overscan_col[j])
#log.info("%d %f %f"%(j,o,r))
overscan_col[j] = o
rdnoise[j] = r
else:
log.info(
"Subtracting average overscan for amplifier %s of camera %s"
% (amp, camera))
o, r = _overscan(raw_overscan_col)
overscan_col += o
rdnoise += r
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan_col)
return rdnoise, median_rdnoise
def compute_bias_file(rawfiles, outfile, camera, explistfile=None):
"""
Compute a bias file from input ZERO rawfiles
Args:
rawfiles: list of input raw file names
outfile (str): output filename
camera (str): camera, e.g. b0, r1, z9
Options:
explistfile: filename with text list of NIGHT EXPID to use
Notes: explistfile is only used if rawfiles=None; it should have
one NIGHT EXPID entry per line.
"""
log = get_logger()
if explistfile is not None:
if rawfiles is not None:
msg = "specify rawfiles or explistfile, but not both"
log.error(msg)
raise ValueError(msg)
rawfiles = list()
with open(explistfile, 'r') as fx:
for line in fx:
line = line.strip()
if line.startswith('#') or len(line) < 2:
continue
night, expid = map(int, line.split())
filename = io.findfile('raw', night, expid)
if not os.path.exists(filename):
msg = f'Missing {filename}'
log.critical(msg)
raise RuntimeError(msg)
rawfiles.append(filename)
log.info("read images ...")
images = []
shape = None
first_image_header = None
for filename in rawfiles:
log.info("reading %s" % filename)
fitsfile = pyfits.open(filename)
primary_header = fitsfile[0].header
image_header = fitsfile[camera].header
if first_image_header is None:
first_image_header = image_header
flavor = image_header['FLAVOR'].upper()
if flavor != 'ZERO':
message = f'Input {filename} flavor {flavor} != ZERO'
log.error(message)
raise ValueError(message)
# subtract overscan region
cfinder = CalibFinder([image_header, primary_header])
image = fitsfile[camera].data.astype("float64")
if cfinder and cfinder.haskey("AMPLIFIERS"):
amp_ids = list(cfinder.value("AMPLIFIERS"))
else:
amp_ids = ['A', 'B', 'C', 'D']
n0 = image.shape[0] // 2
n1 = image.shape[1] // 2
for a, amp in enumerate(amp_ids):
ii = parse_sec_keyword(image_header['BIASSEC' + amp])
overscan_image = image[ii].copy()
overscan, rdnoise = _overscan(overscan_image)
log.info("amp {} overscan = {}".format(amp, overscan))
if ii[0].start < n0 and ii[1].start < n1:
image[:n0, :n1] -= overscan
elif ii[0].start < n0 and ii[1].start >= n1:
image[:n0, n1:] -= overscan
elif ii[0].start >= n0 and ii[1].start < n1:
image[n0:, :n1] -= overscan
elif ii[0].start >= n0 and ii[1].start >= n1:
image[n0:, n1:] -= overscan
if shape is None:
shape = image.shape
images.append(image.ravel())
fitsfile.close()
images = np.array(images)
print(images.shape)
# compute a mask
log.info("compute median image ...")
medimage = np.median(images, axis=0) #.reshape(shape)
log.info("compute mask ...")
ares = np.abs(images - medimage)
nsig = 4.
mask = (ares < nsig * 1.4826 * np.median(ares, axis=0))
# average (not median)
log.info("compute average ...")
meanimage = np.sum(images * mask, axis=0) / np.sum(mask, axis=0)
meanimage = meanimage.reshape(shape)
log.info("write result in %s ..." % outfile)
hdus = pyfits.HDUList([pyfits.PrimaryHDU(meanimage.astype('float32'))])
# copy some keywords
for key in [
"TELESCOP", "INSTRUME", "SPECGRPH", "SPECID", "DETECTOR", "CAMERA",
"CCDNAME", "CCDPREP", "CCDSIZE", "CCDTEMP", "CPUTEMP", "CASETEMP",
"CCDTMING", "CCDCFG", "SETTINGS", "VESSEL", "FEEVER", "FEEBOX",
"PRESECA", "PRRSECA", "DATASECA", "TRIMSECA", "BIASSECA", "ORSECA",
"CCDSECA", "DETSECA", "AMPSECA", "PRESECB", "PRRSECB", "DATASECB",
"TRIMSECB", "BIASSECB", "ORSECB", "CCDSECB", "DETSECB", "AMPSECB",
"PRESECC", "PRRSECC", "DATASECC", "TRIMSECC", "BIASSECC", "ORSECC",
"CCDSECC", "DETSECC", "AMPSECC", "PRESECD", "PRRSECD", "DATASECD",
"TRIMSECD", "BIASSECD", "ORSECD", "CCDSECD", "DETSECD", "AMPSECD",
"DAC0", "DAC1", "DAC2", "DAC3", "DAC4", "DAC5", "DAC6", "DAC7",
"DAC8", "DAC9", "DAC10", "DAC11", "DAC12", "DAC13", "DAC14",
"DAC15", "DAC16", "DAC17", "CLOCK0", "CLOCK1", "CLOCK2", "CLOCK3",
"CLOCK4", "CLOCK5", "CLOCK6", "CLOCK7", "CLOCK8", "CLOCK9",
"CLOCK10", "CLOCK11", "CLOCK12", "CLOCK13", "CLOCK14", "CLOCK15",
"CLOCK16", "CLOCK17", "CLOCK18", "OFFSET0", "OFFSET1", "OFFSET2",
"OFFSET3", "OFFSET4", "OFFSET5", "OFFSET6", "OFFSET7", "DELAYS",
"CDSPARMS", "PGAGAIN", "OCSVER", "DOSVER", "CONSTVER"
]:
if key in first_image_header:
hdus[0].header[key] = (first_image_header[key],
first_image_header.comments[key])
hdus[0].header["BUNIT"] = "adu"
hdus[0].header["EXTNAME"] = "BIAS"
for filename in rawfiles:
hdus[0].header["COMMENT"] = "Inc. {}".format(
os.path.basename(filename))
hdus.writeto(outfile, overwrite="True")
log.info("done")
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
from desispec.calibfinder import CalibFinder
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file,
specprod_dir=specprod_dir)
# Load frame meta
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame_meta, flavor=frame_meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
qaprod_dir=qaprod_dir,
specprod_dir=specprod_dir,
outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # For backwards compatability
calib = CalibFinder([frame_meta])
except KeyError:
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera,
specprod_dir=specprod_dir)
else:
fiberflat_fil = os.path.join(os.getenv('DESI_SPECTRO_CALIB'),
calib.data['FIBERFLAT'])
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
# Load if skymodel found
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
#
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write = True
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
try:
cframe = read_frame(cframe_file)
except FileNotFoundError:
warnings.warn(
"cframe file {:s} not found. Skipping..".format(cframe_file))
else:
if qaframe.run_qa('S2N', (cframe, ), clobber=clobber):
write = True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 's2nfit'
s2n_dict['METRICS']['RA'] = frame.fibermap['TARGET_RA'].data
s2n_dict['METRICS']['DEC'] = frame.fibermap['TARGET_DEC'].data
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(
s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_s2n(s2n_dict, qafig)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('fluxcalib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa(
'FLUXCALIB', (frame, fluxcalib),
clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
f, ax = plt.subplots(nrows=6, ncols=1, figsize=(10,10))
plt.subplots_adjust(top=0.97)
for i,cam in enumerate(dic.keys()):
h = fitsio.FITS(dic[cam]['PATH'])
head = h['IMAGE'].read_header()
camName = head['CAMERA'].strip()
d = h['IMAGE'].read()
w = h['MASK'].read()==0.
td = d.copy()
td[~w] = sp.nan
h.close()
if getattr(args,'{}_psf_path'.format(cam)) is None:
cfinder = CalibFinder([head])
p = cfinder.findfile('PSF')
else:
p = getattr(args,'{}_psf_path'.format(cam))
dic[cam]['PSF'] = read_xytraceset(p)
### image of the PSF
xmin = min( dic[cam]['PSF'].x_vs_wave(fmin,dic[cam]['LINE']['LINE']), dic[cam]['PSF'].x_vs_wave(fmax,dic[cam]['LINE']['LINE']) )
xmax = max( dic[cam]['PSF'].x_vs_wave(fmin,dic[cam]['LINE']['LINE']), dic[cam]['PSF'].x_vs_wave(fmax,dic[cam]['LINE']['LINE']) )
ymin = min( dic[cam]['PSF'].y_vs_wave(fmin,dic[cam]['LINE']['LINE']), dic[cam]['PSF'].y_vs_wave(fmax,dic[cam]['LINE']['LINE']) )
ymax = max( dic[cam]['PSF'].y_vs_wave(fmin,dic[cam]['LINE']['LINE']), dic[cam]['PSF'].y_vs_wave(fmax,dic[cam]['LINE']['LINE']) )
ax[2*i].imshow(td,interpolation='nearest',origin='lower',cmap='hot')
ax[2*i].set_xlim(sp.floor(xmin-offset),sp.floor(xmax+offset))
ax[2*i].set_ylim(sp.floor(ymin-offset),sp.floor(ymax+offset))
ax[2*i].set_ylabel(r'$\mathrm{y-axis}$')
def read_raw(filename, camera, fibermapfile=None, **kwargs):
'''
Returns preprocessed raw data from `camera` extension of `filename`
Args:
filename : input fits filename with DESI raw data
camera : camera name (B0,R1, .. Z9) or FITS extension name or number
Options:
fibermapfile : read fibermap from this file; if None create blank fm
Other keyword arguments are passed to desispec.preproc.preproc(),
e.g. bias, pixflat, mask. See preproc() documentation for details.
Returns Image object with member variables pix, ivar, mask, readnoise
'''
log = get_logger()
t0 = time.time()
fx = fits.open(filename, memmap=False)
if camera.upper() not in fx:
raise IOError('Camera {} not in {}'.format(camera, filename))
rawimage = fx[camera.upper()].data
header = fx[camera.upper()].header
hdu=0
while True :
primary_header= fx[hdu].header
if "EXPTIME" in primary_header : break
if len(fx)>hdu+1 :
if hdu > 0:
log.warning("Did not find header keyword EXPTIME in hdu {}, moving to the next".format(hdu))
hdu +=1
else :
log.error("Did not find header keyword EXPTIME in any HDU of {}".format(filename))
raise KeyError("Did not find header keyword EXPTIME in any HDU of {}".format(filename))
#- Check if NIGHT keyword is present and valid; fix if needed
#- e.g. 20210105 have headers with NIGHT='None' instead of YEARMMDD
try:
tmp = int(primary_header['NIGHT'])
except (KeyError, ValueError, TypeError):
primary_header['NIGHT'] = header2night(primary_header)
try:
tmp = int(header['NIGHT'])
except (KeyError, ValueError, TypeError):
try:
header['NIGHT'] = header2night(header)
except (KeyError, ValueError, TypeError):
#- early teststand data only have NIGHT/timestamps in primary hdr
header['NIGHT'] = primary_header['NIGHT']
#- early data (e.g. 20200219/51053) had a mix of int vs. str NIGHT
primary_header['NIGHT'] = int(primary_header['NIGHT'])
header['NIGHT'] = int(header['NIGHT'])
if primary_header['NIGHT'] != header['NIGHT']:
msg = 'primary header NIGHT={} != camera header NIGHT={}'.format(
primary_header['NIGHT'], header['NIGHT'])
log.error(msg)
raise ValueError(msg)
#- early data have >8 char FIBERASSIGN key; rename to match current data
if 'FIBERASSIGN' in primary_header:
log.warning('renaming long header keyword FIBERASSIGN -> FIBASSGN')
primary_header['FIBASSGN'] = primary_header['FIBERASSIGN']
del primary_header['FIBERASSIGN']
if 'FIBERASSIGN' in header:
header['FIBASSGN'] = header['FIBERASSIGN']
del header['FIBERASSIGN']
skipkeys = ["EXTEND","SIMPLE","NAXIS1","NAXIS2","CHECKSUM","DATASUM","XTENSION","EXTNAME","COMMENT"]
if 'INHERIT' in header and header['INHERIT']:
h0 = fx[0].header
for key in h0:
if ( key not in skipkeys ) and ( key not in header ):
header[key] = h0[key]
if "fill_header" in kwargs :
hdus = kwargs["fill_header"]
if hdus is None :
hdus=[0,]
if "PLC" in fx :
hdus.append("PLC")
if hdus is not None :
log.info("will add header keywords from hdus %s"%str(hdus))
for hdu in hdus :
try :
ihdu = int(hdu)
hdu = ihdu
except ValueError:
pass
if hdu in fx :
hdu_header = fx[hdu].header
for key in hdu_header:
if ( key not in skipkeys ) and ( key not in header ) :
log.debug("adding {} = {}".format(key,hdu_header[key]))
header[key] = hdu_header[key]
else :
log.debug("key %s already in header or in skipkeys"%key)
else :
log.warning("warning HDU %s not in fits file"%str(hdu))
kwargs.pop("fill_header")
fx.close()
duration = time.time() - t0
log.info(iotime.format('read', filename, duration))
img = desispec.preproc.preproc(rawimage, header, primary_header, **kwargs)
if fibermapfile is not None and os.path.exists(fibermapfile):
fibermap = desispec.io.read_fibermap(fibermapfile)
else:
log.warning('creating blank fibermap')
fibermap = desispec.io.empty_fibermap(5000)
#- Add image header keywords inherited from raw data to fibermap too
desispec.io.util.addkeys(fibermap.meta, img.meta)
#- Augment the image header with some tile info from fibermap if needed
for key in ['TILEID', 'TILERA', 'TILEDEC']:
if key in fibermap.meta:
if key not in img.meta:
log.info('Updating header from fibermap {}={}'.format(
key, fibermap.meta[key]))
img.meta[key] = fibermap.meta[key]
elif img.meta[key] != fibermap.meta[key]:
#- complain loudly, but don't crash and don't override
log.error('Inconsistent {}: raw header {} != fibermap header {}'.format(key, img.meta[key], fibermap.meta[key]))
#- Trim to matching camera based upon PETAL_LOC, but that requires
#- a mapping prior to 20191211
#- HACK HACK HACK
#- TODO: replace this with a mapping from calibfinder, as soon as
#- that is implemented in calibfinder / desi_spectro_calib
#- HACK HACK HACK
#- From DESI-5286v5 page 3 where sp=sm-1 and
#- "spectro logical number" = petal_loc
spec_to_petal = {4:2, 2:9, 3:0, 5:3, 1:8, 0:4, 6:6, 7:7, 8:5, 9:1}
assert set(spec_to_petal.keys()) == set(range(10))
assert set(spec_to_petal.values()) == set(range(10))
#- Mapping only for dates < 20191211
if "NIGHT" in primary_header:
dateobs = int(primary_header["NIGHT"])
elif "DATE-OBS" in primary_header:
dateobs=parse_date_obs(primary_header["DATE-OBS"])
else:
msg = "Need either NIGHT or DATE-OBS in primary header"
log.error(msg)
raise KeyError(msg)
if dateobs < 20191211 :
petal_loc = spec_to_petal[int(camera[1])]
log.warning('Prior to 20191211, mapping camera {} to PETAL_LOC={}'.format(camera, petal_loc))
else :
petal_loc = int(camera[1])
log.debug('Since 20191211, camera {} is PETAL_LOC={}'.format(camera, petal_loc))
if 'PETAL_LOC' in fibermap.dtype.names : # not the case in early teststand data
ii = (fibermap['PETAL_LOC'] == petal_loc)
fibermap = fibermap[ii]
if 'FIBER' in fibermap.dtype.names : # not the case in early teststand data
## Mask fibers
cfinder = CalibFinder([header,primary_header])
mod_fibers = fibermap['FIBER'].data % 500
## Mask blacklisted fibers
fiberblacklist = cfinder.fiberblacklist()
for fiber in fiberblacklist:
loc = np.where(mod_fibers==fiber)[0]
fibermap['FIBERSTATUS'][loc] |= maskbits.fibermask.BADFIBER
# Mask Fibers that are set to be excluded due to CCD/amp/readout issues
camname = camera.upper()[0]
if camname == 'B':
badamp_bit = maskbits.fibermask.BADAMPB
elif camname == 'R':
badamp_bit = maskbits.fibermask.BADAMPR
else:
#elif camname == 'Z':
badamp_bit = maskbits.fibermask.BADAMPZ
fibers_to_exclude = cfinder.fibers_to_exclude()
for fiber in fibers_to_exclude:
loc = np.where(mod_fibers==fiber)[0]
fibermap['FIBERSTATUS'][loc] |= badamp_bit
img.fibermap = fibermap
return img
def subtract_overscan(fx,camera):
import desispec.preproc
from desispec.calibfinder import parse_date_obs, CalibFinder
rawimage = fx[camera.upper()].data
rawimage = rawimage.astype(np.float64)
header = fx[camera.upper()].header
hdu=0
primary_header= fx[hdu].header
ccd_calibration_filename=None
log=desispec.preproc.get_logger()
cfinder = CalibFinder([header, primary_header], yaml_file=ccd_calibration_filename)
amp_ids = desispec.preproc.get_amp_ids(header)
#######################
use_overscan_row = False
overscan_per_row=False
#######################
# Subtract overscan
for amp in amp_ids:
# Grab the sections
ov_col = desispec.preproc.parse_sec_keyword(header['BIASSEC'+amp])
if 'ORSEC'+amp in header.keys():
ov_row = desispec.preproc.parse_sec_keyword(header['ORSEC'+amp])
elif use_overscan_row:
log.error('No ORSEC{} keyword; not using overscan_row'.format(amp))
use_overscan_row = False
# Generate the overscan images
raw_overscan_col = rawimage[ov_col].copy()
if use_overscan_row:
raw_overscan_row = rawimage[ov_row].copy()
overscan_row = np.zeros_like(raw_overscan_row)
# Remove overscan_col from overscan_row
raw_overscan_squared = rawimage[ov_row[0], ov_col[1]].copy()
for row in range(raw_overscan_row.shape[0]):
o,r = _overscan(raw_overscan_squared[row])
overscan_row[row] = raw_overscan_row[row] - o
# Now remove the overscan_col
nrows=raw_overscan_col.shape[0]
log.info("nrows in overscan=%d"%nrows)
overscan_col = np.zeros(nrows)
rdnoise = np.zeros(nrows)
if (cfinder and cfinder.haskey('OVERSCAN'+amp) and cfinder.value("OVERSCAN"+amp).upper()=="PER_ROW") or overscan_per_row:
log.info("Subtracting overscan per row for amplifier %s of camera %s"%(amp,camera))
for j in range(nrows) :
if np.isnan(np.sum(overscan_col[j])) :
log.warning("NaN values in row %d of overscan of amplifier %s of camera %s"%(j,amp,camera))
continue
o,r = _overscan(raw_overscan_col[j])
#log.info("%d %f %f"%(j,o,r))
overscan_col[j]=o
rdnoise[j]=r
else :
log.info("Subtracting average overscan for amplifier %s of camera %s"%(amp,camera))
o,r = desispec.preproc._overscan(raw_overscan_col)
overscan_col += o
rdnoise += r
#- subtract overscan from data region and apply gain
jj = desispec.preproc.parse_sec_keyword(header['DATASEC'+amp])
kk = desispec.preproc.parse_sec_keyword(header['CCDSEC'+amp])
data = rawimage[jj].copy()
# Subtract columns
for k in range(nrows):
data[k] -= overscan_col[k]
# And now the rows
if use_overscan_row:
# Savgol?
if use_savgol:
log.info("Using savgol")
collapse_oscan_row = np.zeros(overscan_row.shape[1])
for col in range(overscan_row.shape[1]):
o, _ = _overscan(overscan_row[:,col])
collapse_oscan_row[col] = o
oscan_row = _savgol_clipped(collapse_oscan_row, niter=0)
oimg_row = np.outer(np.ones(data.shape[0]), oscan_row)
data -= oimg_row
else:
o,r = _overscan(overscan_row)
data -= o
rawimage[jj]=data
return rawimage
xx=[]
for f,filename in enumerate(filenames) :
print('reading {} hdu={}'.format(filename,args.camera))
try:
pheader=fitsio.read_header(filename)
if "flavor" in pheader :
if pheader["flavor"].lower().strip() != args.flavor :
print("ignore image with flavor='{}'".format(pheader["flavor"]))
continue
else :
print("warning: no flavor keyword in header")
img,header=fitsio.read(filename,args.camera,header=True)
try :
cfinder=CalibFinder([header,pheader])
except Exception as e:
print("warning: could not find calib, error message='{}'".format(e))
cfinder=None
except OSError :
print("# failed to read",filename)
continue
if "EXPID" in pheader :
expid = pheader["EXPID"]
elif "EXPNUM" in pheader : # winlight version ...
expid = pheader["EXPNUM"]
else :
expid = 0.
if "MJD-OBS" in pheader :
mjdobs = pheader["MJD-OBS"]
def process_exposures(night, verbose=False, overwrite=False):
"""Process all exposures and write out cFrame files.
usage: desi_process_exposure [-h] -i INFILE [--fiberflat FIBERFLAT]
[--sky SKY] [--calib CALIB] -o OUTFILE
[--cosmics-nsig COSMICS_NSIG]
[--sky-throughput-correction]
Apply fiberflat, sky subtraction and calibration.
optional arguments:
-h, --help show this help message and exit
-i INFILE, --infile INFILE
path of DESI exposure frame fits file
--fiberflat FIBERFLAT
path of DESI fiberflat fits file
--sky SKY path of DESI sky fits file
--calib CALIB path of DESI calibration fits file
-o OUTFILE, --outfile OUTFILE
path of DESI sky fits file
--cosmics-nsig COSMICS_NSIG
n sigma rejection for cosmics in 1D (default, no
rejection)
--sky-throughput-correction
apply a throughput correction when subtraction the sky
"""
allexpiddir = glob(
os.path.join(outdir, 'exposures', str(night), '????????'))
for expiddir in allexpiddir:
expid = os.path.basename(expiddir)
framefiles = sorted(
glob(
desispec.io.findfile('frame',
night=night,
expid=int(expid),
camera='*',
specprod_dir=outdir)))
for framefile in framefiles:
outframefile = framefile.replace('frame-', 'cframe-')
if os.path.isfile(outframefile) and not overwrite:
print('File exists {}'.format(outframefile))
continue
# Gather the calibration files.
hdr = fitsio.read_header(framefile)
camera = hdr['CAMERA'].strip()
calib = CalibFinder([hdr])
fiberflatfile = os.path.join(os.getenv('DESI_SPECTRO_CALIB'),
calib.data['FIBERFLAT'])
skymodelfile = desispec.io.findfile('sky',
night=night,
expid=int(expid),
camera=camera,
specprod_dir=outdir)
fluxcalibfile = desispec.io.findfile('fluxcalib',
night=night,
expid=int(expid),
camera=camera,
specprod_dir=outdir)
if os.path.isfile(fiberflatfile) and os.path.isfile(
skymodelfile) and os.path.isfile(fluxcalibfile):
cmd = 'desi_process_exposure --infile {framefile} --sky {skymodelfile} '
cmd += '--fiberflat {fiberflatfile} --calib {fluxcalibfile} '
cmd += '--cosmics-nsig 6 --outfile {outframefile} '
cmd = cmd.format(framefile=framefile,
skymodelfile=skymodelfile,
fiberflatfile=fiberflatfile,
fluxcalibfile=fluxcalibfile,
outframefile=outframefile)
#print(cmd)
os.system(cmd)
else:
print('Missing calibration files.')
def preproc(rawimage, header, primary_header, bias=True, dark=True, pixflat=True, mask=True, bkgsub=False, nocosmic=False, cosmics_nsig=6, cosmics_cfudge=3., cosmics_c2fudge=0.5,ccd_calibration_filename=None, nocrosstalk=False, nogain=False):
'''
preprocess image using metadata in header
image = ((rawimage-bias-overscan)*gain)/pixflat
Args:
rawimage : 2D numpy array directly from raw data file
header : dict-like metadata, e.g. from FITS header, with keywords
CAMERA, BIASSECx, DATASECx, CCDSECx
where x = A, B, C, D for each of the 4 amplifiers
(also supports old naming convention 1, 2, 3, 4).
primary_header: dict-like metadata fit keywords EXPTIME, DOSVER
DATE-OBS is also required if bias, pixflat, or mask=True
Optional bias, pixflat, and mask can each be:
False: don't apply that step
True: use default calibration data for that night
ndarray: use that array
filename (str or unicode): read HDU 0 and use that
Optional background subtraction with median filtering if bkgsub=True
Optional disabling of cosmic ray rejection if nocosmic=True
Optional tuning of cosmic ray rejection parameters:
cosmics_nsig: number of sigma above background required
cosmics_cfudge: number of sigma inconsistent with PSF required
cosmics_c2fudge: fudge factor applied to PSF
Returns Image object with member variables:
image : 2D preprocessed image in units of electrons per pixel
ivar : 2D inverse variance of image
mask : 2D mask of image (0=good)
readnoise : 2D per-pixel readnoise of image
meta : metadata dictionary
TODO: define what keywords are included
preprocessing includes the following steps:
- bias image subtraction
- overscan subtraction (from BIASSEC* keyword defined regions)
- readnoise estimation (from BIASSEC* keyword defined regions)
- gain correction (from GAIN* keywords)
- pixel flat correction
- cosmic ray masking
- propagation of input known bad pixel mask
- inverse variance estimation
Notes:
The bias image is subtracted before any other calculation to remove any
non-uniformities in the overscan regions prior to calculating overscan
levels and readnoise.
The readnoise is an image not just one number per amp, because the pixflat
image also affects the interpreted readnoise.
The inverse variance is estimated from the readnoise and the image itself,
and thus is biased.
'''
log=get_logger()
header = header.copy()
cfinder = None
if ccd_calibration_filename is not False :
cfinder = CalibFinder([header, primary_header], yaml_file=ccd_calibration_filename)
#- TODO: Check for required keywords first
#- Subtract bias image
camera = header['CAMERA'].lower()
#- convert rawimage to float64 : this is the output format of read_image
rawimage = rawimage.astype(np.float64)
bias = get_calibration_image(cfinder,"BIAS",bias)
if bias is not False : #- it's an array
if bias.shape == rawimage.shape :
log.info("subtracting bias")
rawimage = rawimage - bias
else:
raise ValueError('shape mismatch bias {} != rawimage {}'.format(bias.shape, rawimage.shape))
if cfinder and cfinder.haskey("AMPLIFIERS") :
amp_ids=list(cfinder.value("AMPLIFIERS"))
else :
amp_ids=['A','B','C','D']
#- check whether it's indeed CCDSECx with x in ['A','B','C','D']
# or older version with x in ['1','2','3','4']
# we can remove this piece of code at later times
has_valid_keywords = True
for amp in amp_ids :
if not 'CCDSEC%s'%amp in header :
log.warning("No CCDSEC%s keyword in header , will look for alternative naming CCDSEC{1,2,3,4} ..."%amp)
has_valid_keywords = False
break
if not has_valid_keywords :
amp_ids=['1','2','3','4']
for amp in ['1','2','3','4'] :
if not 'CCDSEC%s'%amp in header :
log.error("No CCDSEC%s keyword, exit"%amp)
raise KeyError("No CCDSEC%s keyword"%amp)
#- Output arrays
ny=0
nx=0
for amp in amp_ids :
yy, xx = _parse_sec_keyword(header['CCDSEC%s'%amp])
ny=max(ny,yy.stop)
nx=max(nx,xx.stop)
image = np.zeros( (ny,nx) )
readnoise = np.zeros_like(image)
#- Load mask
mask = get_calibration_image(cfinder,"MASK",mask)
if mask is False :
mask = np.zeros(image.shape, dtype=np.int32)
else :
if mask.shape != image.shape :
raise ValueError('shape mismatch mask {} != image {}'.format(mask.shape, image.shape))
#- Load dark
dark = get_calibration_image(cfinder,"DARK",dark)
if dark is not False :
if dark.shape != image.shape :
log.error('shape mismatch dark {} != image {}'.format(dark.shape, image.shape))
raise ValueError('shape mismatch dark {} != image {}'.format(dark.shape, image.shape))
if cfinder and cfinder.haskey("EXPTIMEKEY") :
exptime_key=cfinder.value("EXPTIMEKEY")
log.info("Using exposure time keyword %s for dark normalization"%exptime_key)
else :
exptime_key="EXPTIME"
exptime = primary_header[exptime_key]
log.info("Multiplying dark by exptime %f"%(exptime))
dark *= exptime
for amp in amp_ids :
ii = _parse_sec_keyword(header['BIASSEC'+amp])
if nogain :
gain = 1.
else :
#- Initial teststand data may be missing GAIN* keywords; don't crash
if 'GAIN'+amp in header:
gain = header['GAIN'+amp] #- gain = electrons / ADU
else:
if cfinder and cfinder.haskey('GAIN'+amp) :
gain = float(cfinder.value('GAIN'+amp))
log.info('Using GAIN{}={} from calibration data'.format(amp,gain))
else :
gain = 1.0
log.warning('Missing keyword GAIN{} in header and nothing in calib data; using {}'.format(amp,gain))
#- Add saturation level
if 'SATURLEV'+amp in header:
saturlev = header['SATURLEV'+amp] # in electrons
else:
if cfinder and cfinder.haskey('SATURLEV'+amp) :
saturlev = float(cfinder.value('SATURLEV'+amp))
log.info('Using SATURLEV{}={} from calibration data'.format(amp,saturlev))
else :
saturlev = 200000
log.warning('Missing keyword SATURLEV{} in header and nothing in calib data; using 200000'.format(amp,saturlev))
overscan_image = rawimage[ii].copy()
nrows=overscan_image.shape[0]
log.info("nrows in overscan=%d"%nrows)
overscan = np.zeros(nrows)
rdnoise = np.zeros(nrows)
overscan_per_row = True
if cfinder and cfinder.haskey('OVERSCAN'+amp) and cfinder.value("OVERSCAN"+amp).upper()=="PER_ROW" :
log.info("Subtracting overscan per row for amplifier %s of camera %s"%(amp,camera))
for j in range(nrows) :
if np.isnan(np.sum(overscan_image[j])) :
log.warning("NaN values in row %d of overscan of amplifier %s of camera %s"%(j,amp,camera))
continue
o,r = _overscan(overscan_image[j])
#log.info("%d %f %f"%(j,o,r))
overscan[j]=o
rdnoise[j]=r
else :
log.info("Subtracting average overscan for amplifier %s of camera %s"%(amp,camera))
o,r = _overscan(overscan_image)
overscan += o
rdnoise += r
rdnoise *= gain
median_rdnoise = np.median(rdnoise)
median_overscan = np.median(overscan)
log.info("Median rdnoise and overscan= %f %f"%(median_rdnoise,median_overscan))
kk = _parse_sec_keyword(header['CCDSEC'+amp])
for j in range(nrows) :
readnoise[kk][j] = rdnoise[j]
header['OVERSCN'+amp] = (median_overscan,'ADUs (gain not applied)')
if gain != 1 :
rdnoise_message = 'electrons (gain is applied)'
gain_message = 'e/ADU (gain applied to image)'
else :
rdnoise_message = 'ADUs (gain not applied)'
gain_message = 'gain not applied to image'
header['OBSRDN'+amp] = (median_rdnoise,rdnoise_message)
header['GAIN'+amp] = (gain,gain_message)
#- Warn/error if measured readnoise is very different from expected if exists
if 'RDNOISE'+amp in header:
expected_readnoise = header['RDNOISE'+amp]
if median_rdnoise < 0.5*expected_readnoise:
log.error('Amp {} measured readnoise {:.2f} < 0.5 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
elif median_rdnoise < 0.9*expected_readnoise:
log.warning('Amp {} measured readnoise {:.2f} < 0.9 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 2.0*expected_readnoise:
log.error('Amp {} measured readnoise {:.2f} > 2 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
elif median_rdnoise > 1.2*expected_readnoise:
log.warning('Amp {} measured readnoise {:.2f} > 1.2 * expected readnoise {:.2f}'.format(
amp, median_rdnoise, expected_readnoise))
#else:
# log.warning('Expected readnoise keyword {} missing'.format('RDNOISE'+amp))
log.info("Measured readnoise for AMP %s = %f"%(amp,median_rdnoise))
#- subtract overscan from data region and apply gain
jj = _parse_sec_keyword(header['DATASEC'+amp])
data = rawimage[jj].copy()
for k in range(nrows) :
data[k] -= overscan[k]
#- apply saturlev (defined in ADU), prior to multiplication by gain
saturated = (rawimage[jj]>=saturlev)
mask[kk][saturated] |= ccdmask.SATURATED
#- subtract dark prior to multiplication by gain
if dark is not False :
log.info("subtracting dark for amp %s"%amp)
data -= dark[kk]
image[kk] = data*gain
if not nocrosstalk :
#- apply cross-talk
# the ccd looks like :
# C D
# A B
# for cross talk, we need a symmetric 4x4 flip_matrix
# of coordinates ABCD giving flip of both axis
# when computing crosstalk of
# A B C D
#
# A AA AB AC AD
# B BA BB BC BD
# C CA CB CC CD
# D DA DB DC BB
# orientation_matrix_defines change of orientation
#
fip_axis_0= np.array([[1,1,-1,-1],
[1,1,-1,-1],
[-1,-1,1,1],
[-1,-1,1,1]])
fip_axis_1= np.array([[1,-1,1,-1],
[-1,1,-1,1],
[1,-1,1,-1],
[-1,1,-1,1]])
for a1 in range(len(amp_ids)) :
amp1=amp_ids[a1]
ii1 = _parse_sec_keyword(header['CCDSEC'+amp1])
a1flux=image[ii1]
#a1mask=mask[ii1]
for a2 in range(len(amp_ids)) :
if a1==a2 :
continue
amp2=amp_ids[a2]
if cfinder is None : continue
if not cfinder.haskey("CROSSTALK%s%s"%(amp1,amp2)) : continue
crosstalk=cfinder.value("CROSSTALK%s%s"%(amp1,amp2))
if crosstalk==0. : continue
log.info("Correct for crosstalk=%f from AMP %s into %s"%(crosstalk,amp1,amp2))
a12flux=crosstalk*a1flux.copy()
#a12mask=a1mask.copy()
if fip_axis_0[a1,a2]==-1 :
a12flux=a12flux[::-1]
#a12mask=a12mask[::-1]
if fip_axis_1[a1,a2]==-1 :
a12flux=a12flux[:,::-1]
#a12mask=a12mask[:,::-1]
ii2 = _parse_sec_keyword(header['CCDSEC'+amp2])
image[ii2] -= a12flux
# mask[ii2] |= a12mask (not sure we really need to propagate the mask)
#- Divide by pixflat image
pixflat = get_calibration_image(cfinder,"PIXFLAT",pixflat)
if pixflat is not False :
if pixflat.shape != image.shape:
raise ValueError('shape mismatch pixflat {} != image {}'.format(pixflat.shape, image.shape))
if np.all(pixflat != 0.0):
image /= pixflat
readnoise /= pixflat
else:
good = (pixflat != 0.0)
image[good] /= pixflat[good]
readnoise[good] /= pixflat[good]
mask[~good] |= ccdmask.PIXFLATZERO
lowpixflat = (0 < pixflat) & (pixflat < 0.1)
if np.any(lowpixflat):
mask[lowpixflat] |= ccdmask.PIXFLATLOW
#- Inverse variance, estimated directly from the data (BEWARE: biased!)
var = image.clip(0) + readnoise**2
ivar = np.zeros(var.shape)
ivar[var>0] = 1.0 / var[var>0]
if bkgsub :
bkg = _background(image,header)
image -= bkg
img = Image(image, ivar=ivar, mask=mask, meta=header, readnoise=readnoise, camera=camera)
#- update img.mask to mask cosmic rays
if not nocosmic :
cosmics.reject_cosmic_rays(img,nsig=cosmics_nsig,cfudge=cosmics_cfudge,c2fudge=cosmics_c2fudge)
return img